From 5e79e8ba653781d962dcb71e5d47b8465e6d9a77 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 22 Aug 2023 16:01:50 -0400 Subject: [PATCH 01/26] Started to setup for codebase validation tests. Fleshed out initial version of AST::is_equal( AST* ) Setup the test directory with initial files for each major validation test. --- .gitignore | 6 + project/components/ast.cpp | 590 +++++++++++++++++- project/components/ast_types.hpp | 11 +- project/components/interface.cpp | 2 + test/GamePipeline/Readme.md | 13 + test/GamePipeline/validate.gamepipeline.cpp | 0 test/GamePipeline/validate.ps1 | 0 test/Godot/Readme.md | 15 + test/Godot/validate.godot.cpp | 0 test/Godot/validate.ps1 | 0 test/Unreal/Readme.md | 25 + test/Unreal/validate.ps1 | 0 test/Unreal/validate.unreal.cpp | 0 test/ZPL-C/Readme.md | 18 + test/ZPL-C/validate.ps1 | 0 test/ZPL-C/validate.zpl-c.cpp | 0 test/{ => _old}/parsed/Array.Parsed.hpp | 0 test/{ => _old}/parsed/Buffer.Parsed.hpp | 0 test/{ => _old}/parsed/HashTable.Parsed.hpp | 0 test/{ => _old}/parsed/Ring.Parsed.hpp | 0 test/{ => _old}/parsed/Sanity.Parsed.hpp | 0 test/{ => _old}/parsed/test.parsing.cpp | 0 test/{ => _old}/upfront/Array.Upfront.hpp | 0 test/{ => _old}/upfront/Buffer.Upfront.hpp | 0 test/{ => _old}/upfront/HashTable.Upfront.hpp | 0 test/{ => _old}/upfront/Ring.Upfront.hpp | 0 test/{ => _old}/upfront/Sanity.Upfront.hpp | 0 test/{ => _old}/upfront/test.upfront.cpp | 0 test/parsing.hpp | 21 + test/sanity.cpp | 5 + test/test.cpp | 5 +- test/upfront.hpp | 2 +- ...e_bootstrap.cpp => validate.bootstrap.cpp} | 0 ...ader_ast.cpp => validate.singleheader.cpp} | 0 test/validate_singleheader.cpp | 3 - 35 files changed, 697 insertions(+), 19 deletions(-) create mode 100644 test/GamePipeline/Readme.md create mode 100644 test/GamePipeline/validate.gamepipeline.cpp create mode 100644 test/GamePipeline/validate.ps1 create mode 100644 test/Godot/Readme.md create mode 100644 test/Godot/validate.godot.cpp create mode 100644 test/Godot/validate.ps1 create mode 100644 test/Unreal/Readme.md create mode 100644 test/Unreal/validate.ps1 create mode 100644 test/Unreal/validate.unreal.cpp create mode 100644 test/ZPL-C/Readme.md create mode 100644 test/ZPL-C/validate.ps1 create mode 100644 test/ZPL-C/validate.zpl-c.cpp rename test/{ => _old}/parsed/Array.Parsed.hpp (100%) rename test/{ => _old}/parsed/Buffer.Parsed.hpp (100%) rename test/{ => _old}/parsed/HashTable.Parsed.hpp (100%) rename test/{ => _old}/parsed/Ring.Parsed.hpp (100%) rename test/{ => _old}/parsed/Sanity.Parsed.hpp (100%) rename test/{ => _old}/parsed/test.parsing.cpp (100%) rename test/{ => _old}/upfront/Array.Upfront.hpp (100%) rename test/{ => _old}/upfront/Buffer.Upfront.hpp (100%) rename test/{ => _old}/upfront/HashTable.Upfront.hpp (100%) rename test/{ => _old}/upfront/Ring.Upfront.hpp (100%) rename test/{ => _old}/upfront/Sanity.Upfront.hpp (100%) rename test/{ => _old}/upfront/test.upfront.cpp (100%) rename test/{validate_bootstrap.cpp => validate.bootstrap.cpp} (100%) rename test/{test.singleheader_ast.cpp => validate.singleheader.cpp} (100%) delete mode 100644 test/validate_singleheader.cpp diff --git a/.gitignore b/.gitignore index 64e3c07..566e67e 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,9 @@ bld/ [Ll]og/ [Ll]ogs/ vc140.pdb + + +# Unreal +**/Unreal/*.h +**/Unreal/*.cpp +! **/Unreal/validate.unreal.cpp diff --git a/project/components/ast.cpp b/project/components/ast.cpp index c3193b8..be87278 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -872,23 +872,603 @@ String AST::to_string() bool AST::is_equal( AST* other ) { +/* + AST values are either some u32 value, a cached string, or a pointer to another AST. + + u32 values are compared by value. + Cached strings are compared by pointer. + AST nodes are compared with AST::is_equal. +*/ + if ( other == nullptr ) + return false; + if ( Type != other->Type ) return false; switch ( Type ) { - case ECode::Typedef: - case ECode::Typename: + using namespace ECode; + + case NewLine: + case Access_Public: + case Access_Protected: + case Access_Private: + case Preprocess_Else: + case Preprocess_EndIf: + return true; + + case Comment: + case Execution: + case PlatformAttributes: + case Untyped: + { + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return Content == other->Content; + } + + case Class_Fwd: + case Struct_Fwd: { if ( Name != other->Name ) return false; + if ( ParentType != other->ParentType ) + return false; + if ( ParentAccess != other->ParentAccess ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; return true; } - } - if ( Name != other->Name ) - return false; + case Class: + case Struct: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( ParentType != other->ParentType ) + return false; + if ( ParentAccess != other->ParentAccess ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + + return true; + } + + case Constructor: + { + if ( InitializerList ? ! InitializerList->is_equal( other->InitializerList ) : other->InitializerList != nullptr ) + return false; + if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Constructor_Fwd: + { + if ( InitializerList ? ! InitializerList->is_equal( other->InitializerList ) : other->InitializerList != nullptr ) + return false; + if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Destructor: + { + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Destructor_Fwd: + { + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Enum: + case Enum_Class: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( UnderlyingType ? ! UnderlyingType->is_equal( other->UnderlyingType ) : other->UnderlyingType != nullptr ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Enum_Fwd: + case Enum_Class_Fwd: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( UnderlyingType ? ! UnderlyingType->is_equal( other->UnderlyingType ) : other->UnderlyingType != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Extern_Linkage: + { + if ( Name != other->Name ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Friend: + { + if ( Name != other->Name ) + return false; + if ( Declaration ? ! Declaration->is_equal( other->Declaration ) : other->Declaration != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Function: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( ReturnType ? ! ReturnType->is_equal( other->ReturnType ) : other->ReturnType != nullptr ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Function_Fwd: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( ReturnType ? ! ReturnType->is_equal( other->ReturnType ) : other->ReturnType != nullptr ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Module: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Namespace: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Operator: + case Operator_Member: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( ReturnType ? ! ReturnType->is_equal( other->ReturnType ) : other->ReturnType != nullptr ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Operator_Fwd: + case Operator_Member_Fwd: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( ReturnType ? ! ReturnType->is_equal( other->ReturnType ) : other->ReturnType != nullptr ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Operator_Cast: + { + if ( Name != other->Name ) + return false; + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( ValueType ? ! ValueType->is_equal( other->ValueType ) : other->ValueType != nullptr ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Operator_Cast_Fwd: + { + if ( Name != other->Name ) + return false; + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( ValueType ? ! ValueType->is_equal( other->ValueType ) : other->ValueType != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Parameters: + { + if ( NumEntries > 1 ) + { + if ( NumEntries != other->NumEntries ) + return false; + + AST* curr = this; + AST* curr_other = other; + while ( curr != nullptr ) + { + if ( ! curr->is_equal( curr_other ) ) + return false; + + curr = curr->Next; + curr_other = curr_other->Next; + } + + return true; + } + + if ( Name != other->Name ) + return false; + if ( ValueType ? ! ValueType->is_equal( other->ValueType ) : other->ValueType != nullptr ) + return false; + if ( Value ? ! Value->is_equal( other->Value ) : other->Value != nullptr ) + return false; + if ( ArrExpr ? ! ArrExpr->is_equal( other->ArrExpr ) : other->ArrExpr != nullptr ) + return false; + + return true; + } + + case Preprocess_Define: + { + if ( Name != other->Name ) + return false; + if ( Content != other->Content ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Preprocess_If: + case Preprocess_IfDef: + case Preprocess_IfNotDef: + case Preprocess_ElIf: + { + if ( Content != other->Content ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Preprocess_Include: + case Preprocess_Pragma: + { + if ( Content != other->Content ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Specifiers: + { + if ( NumEntries != other->NumEntries ) + return false; + if ( Name != other->Name ) + return false; + for ( s32 idx = 0; idx < NumEntries; ++idx ) + { + if ( ArrSpecs[ idx ] != other->ArrSpecs[ idx ] ) + return false; + } + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Template: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) + return false; + if ( Declaration ? ! Declaration->is_equal( other->Declaration ) : other->Declaration != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Typedef: + { + if ( IsFunction != other->IsFunction ) + return false; + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( UnderlyingType != other->UnderlyingType ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + case Typename: + { + if ( Name != other->Name ) + return false; + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( ArrExpr ? ! ArrExpr->is_equal( other->ArrExpr ) : other->ArrExpr != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Union: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Using: + case Using_Namespace: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( UnderlyingType ? ! UnderlyingType->is_equal( other->UnderlyingType ) : other->UnderlyingType != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Variable: + { + if ( ModuleFlags != other->ModuleFlags ) + return false; + if ( Name != other->Name ) + return false; + if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) + return false; + if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) + return false; + if ( ValueType ? ! ValueType->is_equal( other->ValueType ) : other->ValueType != nullptr ) + return false; + if ( BitfieldSize ? ! BitfieldSize->is_equal( other->BitfieldSize ) : other->BitfieldSize != nullptr ) + return false; + if ( Value ? ! Value->is_equal( other->Value ) : other->Value != nullptr ) + return false; + if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) + return false; + if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) + return false; + + return true; + } + + case Class_Body: + case Enum_Body: + case Export_Body: + case Global_Body: + case Namespace_Body: + case Struct_Body: + case Union_Body: + { + if ( NumEntries != other->NumEntries ) + return false; + + if ( Front != other->Front ) + return false; + + if ( Back != other->Back ) + return false; + + AST* curr = Front; + AST* curr_other = other->Front; + while ( curr != nullptr ) + { + if ( ! curr->is_equal( curr_other ) ) + return false; + + curr = curr->Next; + curr_other = curr_other->Next; + } + + return true; + } + } return true; } diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 20d2f86..9a215be 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -89,9 +89,9 @@ struct AST_Constructor Code Prev; Code Next; Code Parent; - StringCached Name; + char _PAD_NAME_[ sizeof(StringCached) ]; CodeT Type; - char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; }; static_assert( sizeof(AST_Constructor) == sizeof(AST), "ERROR: AST_Constructor is not the same size as AST"); @@ -125,7 +125,7 @@ struct AST_Destructor Code Prev; Code Next; Code Parent; - StringCached Name; + char _PAD_NAME_[ sizeof(StringCached) ]; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; }; @@ -158,10 +158,7 @@ struct AST_Exec { union { char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; - struct - { - char _PAD_PROPERTIES_[ sizeof(AST*) * 5 ]; - }; + StringCached Content; }; Code Prev; Code Next; diff --git a/project/components/interface.cpp b/project/components/interface.cpp index f3f3ae1..c29c805 100644 --- a/project/components/interface.cpp +++ b/project/components/interface.cpp @@ -413,6 +413,8 @@ Code make_code() } Code result { rcast( AST*, alloc( * allocator, sizeof(AST) )) }; + // mem_set( result.ast, 0, sizeof(AST) ); + result->Type = ECode::Invalid; result->Content = { nullptr }; result->Prev = { nullptr }; diff --git a/test/GamePipeline/Readme.md b/test/GamePipeline/Readme.md new file mode 100644 index 0000000..70971ed --- /dev/null +++ b/test/GamePipeline/Readme.md @@ -0,0 +1,13 @@ +# Test : Eskil Steenberg's Game Pipeline + +***Note: This validation test has not been implemented yet.*** + +Repo : https://github.com/valiet/quel_solaar + +This is a AST reconstruction test of the gamepipeline library. + +1. Download the library +2. Grab all header and source file paths +3. Generate an ast for each file and serialize it to a file called .gen. +4. Reconstruct the ast from the generated file +5. Compare the original ast to the reconstructed ast diff --git a/test/GamePipeline/validate.gamepipeline.cpp b/test/GamePipeline/validate.gamepipeline.cpp new file mode 100644 index 0000000..e69de29 diff --git a/test/GamePipeline/validate.ps1 b/test/GamePipeline/validate.ps1 new file mode 100644 index 0000000..e69de29 diff --git a/test/Godot/Readme.md b/test/Godot/Readme.md new file mode 100644 index 0000000..aeeecfd --- /dev/null +++ b/test/Godot/Readme.md @@ -0,0 +1,15 @@ +# Test : Godot full AST reconstruction and compile validation + +***Note: This validation test has not been implemented yet.*** + +Repo : https://github.com/godotengine/godot + +* Download the Unreal source code +* Find paths of every header and source file +* Generate an ast for each file and serialize it to a file called `.gen.` +* Reconstruct the ast from the generated file +* Compare the original ast to the reconstructed ast +* If all ASTs are considered valid, overwrite the original files with the generated files +* Compile the engine. + +Currently the most involved test planned for the library. diff --git a/test/Godot/validate.godot.cpp b/test/Godot/validate.godot.cpp new file mode 100644 index 0000000..e69de29 diff --git a/test/Godot/validate.ps1 b/test/Godot/validate.ps1 new file mode 100644 index 0000000..e69de29 diff --git a/test/Unreal/Readme.md b/test/Unreal/Readme.md new file mode 100644 index 0000000..882f0ef --- /dev/null +++ b/test/Unreal/Readme.md @@ -0,0 +1,25 @@ +# Unreal Header & Source reconstruction tests + +***Note: This validation test has not been implemented yet.*** + +Will test the following modules + plugins: + +* Kismet +* Slate +* RTTI Bases +* Gameframework +* Actor & Component Bases +* Lyra + +In the future I could attempt to do a similar test to that of the godot engine full compilation test. + +For now it just does the following: + +* Download the Unreal source code +* For each module + 1. Grab all header and source file paths + 2. Generate an ast for each file and serialize it to a file called `.gen.` + 3. Reconstruct the ast from the generated file + 4. Compare the original ast to the reconstructed ast + +This wil most likely be the most difficult test along-side godot's full compilation test. diff --git a/test/Unreal/validate.ps1 b/test/Unreal/validate.ps1 new file mode 100644 index 0000000..e69de29 diff --git a/test/Unreal/validate.unreal.cpp b/test/Unreal/validate.unreal.cpp new file mode 100644 index 0000000..e69de29 diff --git a/test/ZPL-C/Readme.md b/test/ZPL-C/Readme.md new file mode 100644 index 0000000..89b41ea --- /dev/null +++ b/test/ZPL-C/Readme.md @@ -0,0 +1,18 @@ +# Test : ZPL-C Reconstruction + +***Note: This validation test has not been implemented yet.*** + +Repo : https://github.com/zpl-c/zpl + +This is a AST reconstruction test of the ZPL-C library. + +Much of the dependency code used in gencpp is derived from the ZPL-C library. +In the future I could directly generate that related code from the ZPL-C library. + +For now it just does the following: + +1. Download the ZPL-C library +2. Grab all header and source file paths +3. Generate an ast for each file and serialize it to a file called .gen. +4. Reconstruct the ast from the generated file +5. Compare the original ast to the reconstructed ast diff --git a/test/ZPL-C/validate.ps1 b/test/ZPL-C/validate.ps1 new file mode 100644 index 0000000..e69de29 diff --git a/test/ZPL-C/validate.zpl-c.cpp b/test/ZPL-C/validate.zpl-c.cpp new file mode 100644 index 0000000..e69de29 diff --git a/test/parsed/Array.Parsed.hpp b/test/_old/parsed/Array.Parsed.hpp similarity index 100% rename from test/parsed/Array.Parsed.hpp rename to test/_old/parsed/Array.Parsed.hpp diff --git a/test/parsed/Buffer.Parsed.hpp b/test/_old/parsed/Buffer.Parsed.hpp similarity index 100% rename from test/parsed/Buffer.Parsed.hpp rename to test/_old/parsed/Buffer.Parsed.hpp diff --git a/test/parsed/HashTable.Parsed.hpp b/test/_old/parsed/HashTable.Parsed.hpp similarity index 100% rename from test/parsed/HashTable.Parsed.hpp rename to test/_old/parsed/HashTable.Parsed.hpp diff --git a/test/parsed/Ring.Parsed.hpp b/test/_old/parsed/Ring.Parsed.hpp similarity index 100% rename from test/parsed/Ring.Parsed.hpp rename to test/_old/parsed/Ring.Parsed.hpp diff --git a/test/parsed/Sanity.Parsed.hpp b/test/_old/parsed/Sanity.Parsed.hpp similarity index 100% rename from test/parsed/Sanity.Parsed.hpp rename to test/_old/parsed/Sanity.Parsed.hpp diff --git a/test/parsed/test.parsing.cpp b/test/_old/parsed/test.parsing.cpp similarity index 100% rename from test/parsed/test.parsing.cpp rename to test/_old/parsed/test.parsing.cpp diff --git a/test/upfront/Array.Upfront.hpp b/test/_old/upfront/Array.Upfront.hpp similarity index 100% rename from test/upfront/Array.Upfront.hpp rename to test/_old/upfront/Array.Upfront.hpp diff --git a/test/upfront/Buffer.Upfront.hpp b/test/_old/upfront/Buffer.Upfront.hpp similarity index 100% rename from test/upfront/Buffer.Upfront.hpp rename to test/_old/upfront/Buffer.Upfront.hpp diff --git a/test/upfront/HashTable.Upfront.hpp b/test/_old/upfront/HashTable.Upfront.hpp similarity index 100% rename from test/upfront/HashTable.Upfront.hpp rename to test/_old/upfront/HashTable.Upfront.hpp diff --git a/test/upfront/Ring.Upfront.hpp b/test/_old/upfront/Ring.Upfront.hpp similarity index 100% rename from test/upfront/Ring.Upfront.hpp rename to test/_old/upfront/Ring.Upfront.hpp diff --git a/test/upfront/Sanity.Upfront.hpp b/test/_old/upfront/Sanity.Upfront.hpp similarity index 100% rename from test/upfront/Sanity.Upfront.hpp rename to test/_old/upfront/Sanity.Upfront.hpp diff --git a/test/upfront/test.upfront.cpp b/test/_old/upfront/test.upfront.cpp similarity index 100% rename from test/upfront/test.upfront.cpp rename to test/_old/upfront/test.upfront.cpp diff --git a/test/parsing.hpp b/test/parsing.hpp index e69de29..671f058 100644 --- a/test/parsing.hpp +++ b/test/parsing.hpp @@ -0,0 +1,21 @@ +#ifdef GEN_TIME +#define GEN_FEATURE_PARSING +#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS +#define GEN_ENFORCE_STRONG_CODE_TYPES +#define GEN_EXPOSE_BACKEND +#define GEN_BENCHMARK +#include "gen.hpp" + +void check_parsing() +{ + using namespace gen; + log_fmt("\nupfront: "); + gen::init(); + + // TODO + + gen::deinit(); + log_fmt("Passed!\n"); +} + +#endif \ No newline at end of file diff --git a/test/sanity.cpp b/test/sanity.cpp index dc63eac..aee4483 100644 --- a/test/sanity.cpp +++ b/test/sanity.cpp @@ -4,6 +4,11 @@ #define GEN_ENFORCE_STRONG_CODE_TYPES #define GEN_EXPOSE_BACKEND #define GEN_BENCHMARK + +#define GEN_GLOBAL_BUCKET_SIZE megabytes(10) +#define GEN_CODE_POOL_BLOCK_SIZE megabytes(32) +#define GEN_STRING_ARENA_SIZE megabytes(1) + #include "gen.hpp" #include "gen.builder.hpp" diff --git a/test/test.cpp b/test/test.cpp index 5362de4..47b36f5 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -7,7 +7,7 @@ #include "gen.builder.cpp" #include "sanity.cpp" #include "SOA.cpp" -#include "test.singleheader_ast.cpp" +#include "test.singleheader.cpp" int gen_main() { @@ -18,8 +18,7 @@ int gen_main() // check_SOA(); - check_singleheader_ast(); - + // check_singleheader_ast(); return 0; } #endif diff --git a/test/upfront.hpp b/test/upfront.hpp index 28d02dc..9220752 100644 --- a/test/upfront.hpp +++ b/test/upfront.hpp @@ -12,7 +12,7 @@ void check_upfront() log_fmt("\nupfront: "); gen::init(); - + // TODO gen::deinit(); log_fmt("Passed!\n"); diff --git a/test/validate_bootstrap.cpp b/test/validate.bootstrap.cpp similarity index 100% rename from test/validate_bootstrap.cpp rename to test/validate.bootstrap.cpp diff --git a/test/test.singleheader_ast.cpp b/test/validate.singleheader.cpp similarity index 100% rename from test/test.singleheader_ast.cpp rename to test/validate.singleheader.cpp diff --git a/test/validate_singleheader.cpp b/test/validate_singleheader.cpp deleted file mode 100644 index fcd9752..0000000 --- a/test/validate_singleheader.cpp +++ /dev/null @@ -1,3 +0,0 @@ -// Constructs an AST from the singlheader generated gen files, then serializes it to a set of files. -// Using the new set of serialized files, reconstructs the AST and then serializes it again (to different set of files). -// The two sets of serialized files should be identical. (Verified by comparing the file hashes) From c97762ac162d682da46ce26c2f7b95b18e1a4526 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 23 Aug 2023 00:05:58 -0400 Subject: [PATCH 02/26] Added support for inline comments Also now doing comment serialization on def_comment directly as parse_comment doesn't need it. Essentially comment ast types serialize the same way s untyped and execution ASTs --- project/bootstrap.cpp | 2 +- project/components/ast.cpp | 134 +++++++++++--- project/components/ast.hpp | 10 +- project/components/ast_types.hpp | 34 ++-- project/components/gen/ast_inlines.hpp | 2 +- project/components/gen/ecode.hpp | 2 +- project/components/gen/eoperator.hpp | 2 +- project/components/gen/especifier.hpp | 2 +- project/components/gen/etoktype.cpp | 4 +- project/components/interface.parsing.cpp | 215 +++++++++++++++++------ project/components/interface.upfront.cpp | 32 +++- project/enums/ETokType.csv | 2 +- singleheader/singleheader.cpp | 2 +- test/test.cpp | 4 +- 14 files changed, 344 insertions(+), 103 deletions(-) diff --git a/project/bootstrap.cpp b/project/bootstrap.cpp index 229b888..2efad37 100644 --- a/project/bootstrap.cpp +++ b/project/bootstrap.cpp @@ -17,7 +17,7 @@ GEN_NS_END using namespace gen; constexpr char const* generation_notice = -"// This file was generated automatially by gen.bootstrap.cpp " +"// This file was generated automatially by gencpp's bootstrap.cpp " "(See: https://github.com/Ed94/gencpp)\n\n"; constexpr bool DontSkipInitialDirectives = false; diff --git a/project/components/ast.cpp b/project/components/ast.cpp index be87278..9836fd3 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -43,6 +43,11 @@ String AST::to_string() case Comment: { + // TODO : Move this formmating process to def_comment, + // Were going to preserve as much of the original formatting as possible + // so that the parsed comments don't have any artifacts. + // Just doing what untyped and execution do +#if 0 if ( Prev && Prev->Type != Comment && Prev->Type != NewLine ) result.append( "\n" ); @@ -72,6 +77,9 @@ String AST::to_string() if ( result.back() != '\n' ) result.append( "\n" ); +#else + result.append( Content ); +#endif } break; @@ -140,9 +148,15 @@ String AST::to_string() result.append_fmt( "class %S %S", Attributes->to_string(), Name ); else result.append_fmt( "class %S", Name ); - + + // Check if it can have an end-statement if ( Parent == nullptr || ( Parent->Type != ECode::Typedef && Parent->Type != ECode::Variable ) ) - result.append(";\n"); + { + if ( InlineCmt ) + result.append_fmt( "; // %S\n", InlineCmt->Content ); + else + result.append(";\n"); + } } break; @@ -169,7 +183,12 @@ String AST::to_string() if ( Params ) result.append_fmt( "( %S )", Params->to_string() ); else - result.append( "(void);\n" ); + { + if ( InlineCmt ) + result.append_fmt( "(void); // %S\n", InlineCmt->Content ); + else + result.append( "(void);\n" ); + } } break; @@ -203,10 +222,15 @@ String AST::to_string() result.append_fmt( "~%S()", Parent->Name ); if ( specs.has( ESpecifier::Pure ) ) - result.append( " = 0;\n" ); + result.append( " = 0;" ); } else - result.append_fmt( "~%S();\n", Parent->Name ); + result.append_fmt( "~%S();", Parent->Name ); + + if ( InlineCmt ) + result.append_fmt( " %S", InlineCmt->Content ); + else + result.append("\n"); } break; @@ -249,7 +273,12 @@ String AST::to_string() result.append_fmt( "enum %S : %S", Name, UnderlyingType->to_string() ); if ( Parent == nullptr || ( Parent->Type != ECode::Typedef && Parent->Type != ECode::Variable ) ) - result.append(";\n"); + { + if ( InlineCmt ) + result.append_fmt("; %S", InlineCmt->Content ); + else + result.append(";\n"); + } } break; @@ -299,7 +328,12 @@ String AST::to_string() result.append_fmt( "%S : %S", Name, UnderlyingType->to_string() ); if ( Parent == nullptr || ( Parent->Type != ECode::Typedef && Parent->Type != ECode::Variable ) ) - result.append(";\n"); + { + if ( InlineCmt ) + result.append_fmt("; %S", InlineCmt->Content ); + else + result.append(";\n"); + } } break; @@ -327,7 +361,14 @@ String AST::to_string() result.append_fmt( "friend %S", Declaration->to_string() ); if ( result[ result.length() -1 ] != ';' ) - result.append( ";\n" ); + { + result.append( ";" ); + } + + if ( InlineCmt ) + result.append_fmt(" %S", InlineCmt->Content ); + else + result.append("\n"); break; case Function: @@ -404,7 +445,10 @@ String AST::to_string() } } - result.append( ";\n" ); + if ( InlineCmt ) + result.append_fmt( "; %S", InlineCmt->Content ); + else + result.append( ";\n" ); } break; @@ -495,8 +539,11 @@ String AST::to_string() } } } - - result.append( ";\n" ); + + if ( InlineCmt ) + result.append_fmt( "; %S", InlineCmt->Content ); + else + result.append( ";\n" ); } break; @@ -504,6 +551,8 @@ String AST::to_string() { if ( Specs ) { + // TODO : Add support for specifies before the operator keyword + if ( Name && Name.length() ) result.append_fmt( "%Soperator %S()", Name, ValueType->to_string() ); else @@ -532,6 +581,8 @@ String AST::to_string() case Operator_Cast_Fwd: if ( Specs ) { + // TODO : Add support for specifies before the operator keyword + result.append_fmt( "operator %S()", ValueType->to_string() ); for ( SpecifierT spec : Specs->cast() ) @@ -543,11 +594,17 @@ String AST::to_string() } } - result.append( ";" ); + if ( InlineCmt ) + result.append_fmt( "; %S", InlineCmt->Content ); + else + result.append( ";\n" ); break; } - result.append_fmt("operator %S();\n", ValueType->to_string() ); + if ( InlineCmt ) + result.append_fmt("operator %S(); %S", ValueType->to_string() ); + else + result.append_fmt("operator %S();\n", ValueType->to_string() ); break; case Parameters: @@ -680,7 +737,12 @@ String AST::to_string() } if ( Parent == nullptr || ( Parent->Type != ECode::Typedef && Parent->Type != ECode::Variable ) ) - result.append(";\n"); + { + if ( InlineCmt ) + result.append_fmt("; %S", InlineCmt->Content ); + else + result.append(";\n"); + } } break; @@ -695,7 +757,12 @@ String AST::to_string() else result.append_fmt( "struct %S", Name ); if ( Parent == nullptr || ( Parent->Type != ECode::Typedef && Parent->Type != ECode::Variable ) ) - result.append(";\n"); + { + if ( InlineCmt ) + result.append_fmt("; %S", InlineCmt->Content ); + else + result.append(";\n"); + } } break; @@ -726,8 +793,13 @@ String AST::to_string() } else { - result.append( ";\n" ); + result.append( ";" ); } + + if ( InlineCmt ) + result.append_fmt(" %S", InlineCmt->Content); + else + result.append("\n"); } break; @@ -796,15 +868,23 @@ String AST::to_string() if ( UnderlyingType->ArrExpr ) result.append_fmt( "[%S]", UnderlyingType->ArrExpr->to_string() ); - result.append( ";\n" ); + result.append( ";" ); } else - result.append_fmt( "using %S;\n", Name ); + result.append_fmt( "using %S;", Name ); + + if ( InlineCmt ) + result.append_fmt(" %S\n", InlineCmt->Content ); + else + result.append("\n"); } break; case Using_Namespace: - result.append_fmt( "using namespace %s;\n", Name ); + if ( InlineCmt ) + result.append_fmt( "using namespace $S; %S", Name, InlineCmt->Content ); + else + result.append_fmt( "using namespace %s;\n", Name ); break; case Variable: @@ -831,19 +911,27 @@ String AST::to_string() if ( Value ) result.append_fmt( " = %S", Value->to_string() ); - result.append( ";\n" ); + if ( InlineCmt ) + result.append_fmt("; %S", InlineCmt->Content); + else + result.append( ";\n" ); break; } if ( BitfieldSize ) - result.append_fmt( "%S %S : %S;\n", ValueType->to_string(), Name, BitfieldSize->to_string() ); + result.append_fmt( "%S %S : %S;", ValueType->to_string(), Name, BitfieldSize->to_string() ); else if ( UnderlyingType->ArrExpr ) - result.append_fmt( "%S %S[%S];\n", UnderlyingType->to_string(), Name, UnderlyingType->ArrExpr->to_string() ); + result.append_fmt( "%S %S[%S];", UnderlyingType->to_string(), Name, UnderlyingType->ArrExpr->to_string() ); else - result.append_fmt( "%S %S;\n", UnderlyingType->to_string(), Name ); + result.append_fmt( "%S %S;", UnderlyingType->to_string(), Name ); + + if ( InlineCmt ) + result.append_fmt(" %S", InlineCmt->Content); + else + result.append("\n"); } break; diff --git a/project/components/ast.hpp b/project/components/ast.hpp index 7292e25..07f669a 100644 --- a/project/components/ast.hpp +++ b/project/components/ast.hpp @@ -226,10 +226,11 @@ struct AST union { struct { + AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable - AST* Specs; // Function, Operator, Type symbol, Variable + AST* Specs; // Destructor, Function, Operator, Type symbol, Variable union { - AST* InitializerList; // Constructor, Destructor + AST* InitializerList; // Constructor AST* ParentType; // Class, Struct AST* ReturnType; // Function, Operator AST* UnderlyingType; // Enum, Typedef @@ -237,7 +238,7 @@ struct AST }; union { AST* BitfieldSize; // Varaiable (Class/Struct Data Member) - AST* Params; // Function, Operator, Template + AST* Params; // Constructor, Function, Operator, Template }; union { AST* ArrExpr; // Type Symbol @@ -275,10 +276,11 @@ struct AST_POD union { struct { + AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable AST* Attributes; // Class, Enum, Function, Struct, Typename, Union, Using, Variable AST* Specs; // Function, Operator, Type symbol, Variable union { - AST* InitializerList; // Constructor, Destructor + AST* InitializerList; // Constructor AST* ParentType; // Class, Struct AST* ReturnType; // Function, Operator AST* UnderlyingType; // Enum, Typedef diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 9a215be..860feab 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -57,6 +57,7 @@ struct AST_Class char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; CodeAttributes Attributes; char _PAD_SPECS_ [ sizeof(AST*) ]; CodeType ParentType; @@ -80,10 +81,11 @@ struct AST_Constructor char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - char _PAD_PROPERTIES_ [ sizeof(AST*) * 3 ]; - Code InitializerList; - CodeParam Params; - Code Body; + CodeComment InlineCmt; + char _PAD_PROPERTIES_ [ sizeof(AST*) * 3 ]; + Code InitializerList; + CodeParam Params; + Code Body; }; }; Code Prev; @@ -116,6 +118,7 @@ struct AST_Destructor char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; char _PAD_PROPERTIES_ [ sizeof(AST*) * 1 ]; CodeSpecifiers Specs; char _PAD_PROPERTIES_2_ [ sizeof(AST*) * 2 ]; @@ -137,6 +140,7 @@ struct AST_Enum char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; CodeAttributes Attributes; char _PAD_SPEC_ [ sizeof(AST*) ]; CodeType UnderlyingType; @@ -175,7 +179,7 @@ struct AST_Extern char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - char _PAD_PROPERTIES_[ sizeof(AST*) * 4 ]; + char _PAD_PROPERTIES_[ sizeof(AST*) * 5 ]; CodeBody Body; }; }; @@ -209,8 +213,9 @@ struct AST_Friend char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - char _PAD_PROPERTIES_[ sizeof(AST*) * 4 ]; - Code Declaration; + CodeComment InlineCmt; + char _PAD_PROPERTIES_[ sizeof(AST*) * 4 ]; + Code Declaration; }; }; Code Prev; @@ -228,6 +233,7 @@ struct AST_Fn char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; CodeAttributes Attributes; CodeSpecifiers Specs; CodeType ReturnType; @@ -263,7 +269,7 @@ struct AST_NS union { char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - char _PAD_PROPERTIES_[ sizeof(AST*) * 4 ]; + char _PAD_PROPERTIES_[ sizeof(AST*) * 5 ]; CodeBody Body; }; }; @@ -283,6 +289,7 @@ struct AST_Operator char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; CodeAttributes Attributes; CodeSpecifiers Specs; CodeType ReturnType; @@ -306,6 +313,7 @@ struct AST_OpCast char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; char _PAD_PROPERTIES_[ sizeof(AST*) ]; CodeSpecifiers Specs; CodeType ValueType; @@ -328,7 +336,7 @@ struct AST_Param char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - char _PAD_PROPERTIES_2_[ sizeof(AST*) * 2 ]; + char _PAD_PROPERTIES_2_[ sizeof(AST*) * 3 ]; CodeType ValueType; char _PAD_PROPERTIES_[ sizeof(AST*) ]; Code Value; @@ -393,6 +401,7 @@ struct AST_Struct char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; CodeAttributes Attributes; char _PAD_SPECS_ [ sizeof(AST*) ]; CodeType ParentType; @@ -416,7 +425,7 @@ struct AST_Template char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - char _PAD_PROPERTIES_[ sizeof(AST*) * 3 ]; + char _PAD_PROPERTIES_[ sizeof(AST*) * 4 ]; CodeParam Params; Code Declaration; }; @@ -437,6 +446,7 @@ struct AST_Type char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + char _PAD_CMT_[ sizeof(AST*) ]; CodeAttributes Attributes; CodeSpecifiers Specs; char _PAD_PROPERTIES_[ sizeof(AST*) * 2 ]; @@ -458,6 +468,7 @@ struct AST_Typedef char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; char _PAD_PROPERTIES_[ sizeof(AST*) * 2 ]; Code UnderlyingType; char _PAD_PROPERTIES_2_[ sizeof(AST*) * 2 ]; @@ -479,6 +490,7 @@ struct AST_Union char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + char _PAD_INLINE_CMT_[ sizeof(AST*) ]; CodeAttributes Attributes; char _PAD_PROPERTIES_[ sizeof(AST*) * 3 ]; CodeBody Body; @@ -500,6 +512,7 @@ struct AST_Using char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; CodeAttributes Attributes; char _PAD_SPECS_ [ sizeof(AST*) ]; CodeType UnderlyingType; @@ -522,6 +535,7 @@ struct AST_Var char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { + CodeComment InlineCmt; CodeAttributes Attributes; CodeSpecifiers Specs; CodeType ValueType; diff --git a/project/components/gen/ast_inlines.hpp b/project/components/gen/ast_inlines.hpp index aaed685..f4bb2ab 100644 --- a/project/components/gen/ast_inlines.hpp +++ b/project/components/gen/ast_inlines.hpp @@ -1,5 +1,5 @@ #pragma once -// This file was generated automatially by gen.bootstrap.cpp (See: https://github.com/Ed94/gencpp) +// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) #pragma region generated code inline implementation diff --git a/project/components/gen/ecode.hpp b/project/components/gen/ecode.hpp index a1701b9..a2e61dc 100644 --- a/project/components/gen/ecode.hpp +++ b/project/components/gen/ecode.hpp @@ -1,6 +1,6 @@ #pragma once -// This file was generated automatially by gen.bootstrap.cpp (See: https://github.com/Ed94/gencpp) +// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) namespace ECode { diff --git a/project/components/gen/eoperator.hpp b/project/components/gen/eoperator.hpp index dd40844..cea26b1 100644 --- a/project/components/gen/eoperator.hpp +++ b/project/components/gen/eoperator.hpp @@ -1,6 +1,6 @@ #pragma once -// This file was generated automatially by gen.bootstrap.cpp (See: https://github.com/Ed94/gencpp) +// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) namespace EOperator { diff --git a/project/components/gen/especifier.hpp b/project/components/gen/especifier.hpp index b5f9d5e..4dedcf7 100644 --- a/project/components/gen/especifier.hpp +++ b/project/components/gen/especifier.hpp @@ -1,6 +1,6 @@ #pragma once -// This file was generated automatially by gen.bootstrap.cpp (See: https://github.com/Ed94/gencpp) +// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) namespace ESpecifier { diff --git a/project/components/gen/etoktype.cpp b/project/components/gen/etoktype.cpp index 0ef4f7a..db84d4c 100644 --- a/project/components/gen/etoktype.cpp +++ b/project/components/gen/etoktype.cpp @@ -1,4 +1,4 @@ -// This file was generated automatially by gen.bootstrap.cpp (See: https://github.com/Ed94/gencpp) +// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) #pragma once @@ -128,7 +128,7 @@ namespace Parser { sizeof( "]" ), "]" }, { sizeof( "(" ), "(" }, { sizeof( ")" ), ")" }, - { sizeof( "__comemnt__" ), "__comemnt__" }, + { sizeof( "__comment__" ), "__comment__" }, { sizeof( "__comment_end__" ), "__comment_end__" }, { sizeof( "__comment_start__" ), "__comment_start__" }, { sizeof( "__character__" ), "__character__" }, diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index f9eea83..3f2d357 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -67,22 +67,22 @@ namespace Parser bool __eat( TokType type ); - Token& current( bool skip_new_lines = true ) + Token& current( bool skip_formatting = true ) { - if ( skip_new_lines ) + if ( skip_formatting ) { - while ( Arr[Idx].Type == TokType::NewLine ) + while ( Arr[Idx].Type == TokType::NewLine || Arr[Idx].Type == TokType::Comment ) Idx++; } return Arr[Idx]; } - Token& previous( bool skip_new_lines = false ) + Token& previous( bool skip_formatting = false ) { s32 idx = this->Idx; - if ( skip_new_lines ) + if ( skip_formatting ) { while ( Arr[idx].Type == TokType::NewLine ) idx--; @@ -93,11 +93,11 @@ namespace Parser return Arr[idx - 1]; } - Token& next( bool skip_new_lines = false ) + Token& next( bool skip_formatting = false ) { s32 idx = this->Idx; - if ( skip_new_lines ) + if ( skip_formatting ) { while ( Arr[idx].Type == TokType::NewLine ) idx++; @@ -114,7 +114,7 @@ namespace Parser } }; - constexpr bool dont_skip_new_lines = false; + constexpr bool dont_skip_formatting = false; struct StackNode { @@ -200,7 +200,8 @@ namespace Parser return false; } - if ( Arr[Idx].Type == TokType::NewLine && type != TokType::NewLine ) + if ( Arr[ Idx ].Type == TokType::NewLine && type != TokType::NewLine + || Arr[ Idx ].Type == TokType::Comment && type != TokType::Comment ) { Idx++; } @@ -879,57 +880,53 @@ namespace Parser if ( current == '/' ) { - token.Type = TokType::Comment_Start; + token.Type = TokType::Comment; token.Length = 2; - Tokens.append( token ); move_forward(); - Token content = { scanner, 1, TokType::Comment, line, column, false }; + token.Length++; while ( left && current != '\n' && current != '\r' ) { move_forward(); - content.Length++; + token.Length++; } - Tokens.append( content ); if ( current == '\r' ) { move_forward(); + token.Length++; } if ( current == '\n' ) { move_forward(); + token.Length++; } + Tokens.append( token ); continue; } else if ( current == '*' ) { - token.Type = TokType::Comment_Start; + token.Type = TokType::Comment; token.Length = 2; - Tokens.append( token ); - - Token content = { token.Text, 0, TokType::Comment, line, column, false }; + move_forward(); - content.Length++; + token.Length++; - bool star = current == '*'; + bool star = current == '*'; bool slash = scanner[1] == '/'; bool at_end = star && slash; while ( left && ! at_end ) { move_forward(); - content.Length++; + token.Length++; - star = current == '*'; + star = current == '*'; slash = scanner[1] == '/'; at_end = star && slash; } - content.Length += 3; - Tokens.append( content ); - - Token end = { scanner, 2, TokType::Comment_End, line, column, false }; - Tokens.append( end ); + token.Length += 3; + Tokens.append( token ); move_forward(); move_forward(); @@ -1141,7 +1138,7 @@ if ( def.Ptr == nullptr ) \ return CodeInvalid; \ } -# define currtok_noskip Context.Tokens.current( dont_skip_new_lines ) +# define currtok_noskip Context.Tokens.current( dont_skip_formatting ) # define currtok Context.Tokens.current() # define prevtok Context.Tokens.previous() # define nexttok Context.Tokens.next() @@ -1185,20 +1182,16 @@ internal CodeComment parse_comment() { using namespace Parser; - push_scope(); - - eat( TokType::Comment_Start ); - + StackNode scope { nullptr, currtok_noskip, NullToken, txt( __func__ ) }; + Context.push( & scope ); + CodeComment result = (CodeComment) make_code(); result->Type = ECode::Comment; - result->Content = get_cached_string( currtok ); + result->Content = get_cached_string( currtok_noskip ); result->Name = result->Content; eat( TokType::Comment ); - if ( check( TokType::Comment_End ) ) - eat( TokType::Comment_End ); - Context.pop(); return result; } @@ -1804,7 +1797,8 @@ CodeFn parse_function_after_name( eat( currtok.Type ); } - CodeBody body = { nullptr }; + CodeBody body = NoCode; + CodeComment inline_cmt = NoCode; if ( check( TokType::BraceCurly_Open ) ) { body = parse_function_body(); @@ -1816,7 +1810,11 @@ CodeFn parse_function_after_name( } else { + Token stmt_end = currtok; eat( TokType::Statement_End ); + + if ( currtok_noskip.Type && TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); } using namespace ECode; @@ -1857,6 +1855,9 @@ CodeFn parse_function_after_name( if ( params ) result->Params = params; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; Context.pop(); return result; @@ -2126,7 +2127,8 @@ CodeOperator parse_operator_after_ret_type( } // Parse Body - CodeBody body = { nullptr }; + CodeBody body = { nullptr }; + CodeComment inline_cmt = NoCode; if ( check( TokType::BraceCurly_Open ) ) { body = parse_function_body(); @@ -2138,11 +2140,19 @@ CodeOperator parse_operator_after_ret_type( } else { + Token stmt_end = currtok; eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); } // OpValidateResult check_result = operator__validate( op, params, ret_type, specifiers ); CodeOperator result = def_operator( op, nspace, params, ret_type, body, specifiers, attributes, mflags ); + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; + Context.pop(); return result; } @@ -2230,8 +2240,18 @@ CodeVar parse_variable_after_name( expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text; bitfield_expr = untyped_str( expr_tok ); } - + + Token stmt_end = currtok; eat( TokType::Statement_End ); + + // Check for inline comment : = ; // + CodeComment inline_cmt = NoCode; + if ( left + && ( currtok_noskip.Type == TokType::Comment ) + && currtok_noskip.Line == stmt_end.Line ) + { + inline_cmt = parse_comment(); + } using namespace ECode; @@ -2257,6 +2277,9 @@ CodeVar parse_variable_after_name( if ( expr ) result->Value = expr; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; Context.pop(); return result; @@ -2294,7 +2317,11 @@ Code parse_simple_preprocess( Parser::TokType which ) { if ( check( TokType::Statement_End )) { + Token stmt_end = currtok; eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + eat( TokType::Comment ); } } @@ -2306,7 +2333,11 @@ Code parse_simple_preprocess( Parser::TokType which ) { if ( check( TokType::Statement_End )) { + Token stmt_end = currtok; eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + eat( TokType::Comment ); } } @@ -2382,7 +2413,6 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes if ( check( TokType::Capture_Start) ) { // Dealing with a function - result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, type, name ); } else @@ -2575,7 +2605,7 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::Token name = Pa eat( TokType::NewLine ); break; - case TokType::Comment_Start: + case TokType::Comment: member = parse_comment(); break; @@ -2880,14 +2910,24 @@ Code parse_class_struct( Parser::TokType which, bool inplace_def = false ) body = parse_class_struct_body( which, name ); } + CodeComment inline_cmt = NoCode; if ( ! inplace_def ) + { + Token stmt_end = currtok; eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); + } if ( which == TokType::Decl_Class ) result = def_class( name, body, parent, access, attributes, mflags ); else result = def_struct( name, body, (CodeType)parent, access, attributes, mflags ); + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; interfaces.free(); return result; @@ -2972,7 +3012,7 @@ CodeBody parse_global_nspace( CodeT which ) eat( TokType::NewLine ); break; - case TokType::Comment_Start: + case TokType::Comment: member = parse_comment(); break; @@ -3232,10 +3272,11 @@ CodeConstructor parse_constructor() using namespace Parser; push_scope(); - Token identifier = parse_identifier(); - CodeParam params = parse_params(); - Code initializer_list = { nullptr }; - CodeBody body = { nullptr }; + Token identifier = parse_identifier(); + CodeParam params = parse_params(); + Code initializer_list = NoCode; + CodeBody body = NoCode; + CodeComment inline_cmt = NoCode; if ( check( TokType::Assign_Classifer ) ) { @@ -3263,6 +3304,14 @@ CodeConstructor parse_constructor() { body = parse_function_body(); } + else + { + Token stmt_end = currtok; + eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); + } CodeConstructor result = (CodeConstructor) make_code(); @@ -3279,6 +3328,9 @@ CodeConstructor parse_constructor() } else result->Type = ECode::Constructor_Fwd; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; Context.pop(); return result; @@ -3321,8 +3373,8 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers ) return CodeInvalid; } - Token identifier = parse_identifier(); - CodeBody body = { nullptr }; + Token identifier = parse_identifier(); + CodeBody body = { nullptr }; eat( TokType::Capture_Start ); eat( TokType::Capture_End ); @@ -3343,9 +3395,19 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers ) return CodeInvalid; } } + + CodeComment inline_cmt = NoCode; if ( check( TokType::BraceCurly_Open ) ) body = parse_function_body(); + else + { + Token stmt_end = currtok; + eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); + } CodeDestructor result = (CodeDestructor) make_code(); @@ -3359,6 +3421,9 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers ) } else result->Type = ECode::Destructor_Fwd; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; Context.pop(); return result; @@ -3452,7 +3517,7 @@ CodeEnum parse_enum( bool inplace_def ) eat( TokType::NewLine ); break; - case TokType::Comment_Start: + case TokType::Comment: member = parse_comment(); break; @@ -3528,8 +3593,16 @@ CodeEnum parse_enum( bool inplace_def ) eat( TokType::BraceCurly_Close ); } + CodeComment inline_cmt = NoCode; + if ( ! inplace_def ) + { + Token stmt_end = currtok; eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); + } using namespace ECode; @@ -3553,6 +3626,9 @@ CodeEnum parse_enum( bool inplace_def ) if ( type ) result->UnderlyingType = type; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; Context.pop(); return result; @@ -3690,8 +3766,13 @@ CodeFriend parse_friend() if ( params ) function->Params = params; } - + + Token stmt_end = currtok; eat( TokType::Statement_End ); + + CodeComment inline_cmt = NoCode; + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); CodeFriend result = (CodeFriend) make_code(); @@ -3702,6 +3783,9 @@ CodeFriend parse_friend() else result->Declaration = type; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; Context.pop(); return result; @@ -3949,7 +4033,7 @@ CodeOpCast parse_operator_cast( CodeSpecifiers specifiers ) using namespace Parser; push_scope(); - // Specifiers attributed to the cast + // TODO : Specifiers attributed to the cast // Operator's namespace if not within same class. Token name = NullToken; @@ -3987,7 +4071,8 @@ CodeOpCast parse_operator_cast( CodeSpecifiers specifiers ) eat( TokType::Spec_Const ); } - Code body = { nullptr }; + Code body = NoCode; + CodeComment inline_cmt = NoCode; if ( check( TokType::BraceCurly_Open) ) { @@ -4014,7 +4099,11 @@ CodeOpCast parse_operator_cast( CodeSpecifiers specifiers ) } else { + Token stmt_end = currtok; eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); } CodeOpCast result = (CodeOpCast) make_code(); @@ -4542,8 +4631,13 @@ CodeTypedef parse_typedef() } array_expr = parse_array_decl(); - + + Token stmt_end = currtok; eat( TokType::Statement_End ); + + CodeComment inline_cmt = NoCode; + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); using namespace ECode; @@ -4567,6 +4661,9 @@ CodeTypedef parse_typedef() if ( type->Type == Typename && array_expr && array_expr->Type != Invalid ) type.cast()->ArrExpr = array_expr; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; Context.pop(); return result; @@ -4633,7 +4730,7 @@ CodeUnion parse_union( bool inplace_def ) eat( TokType::NewLine ); break; - case TokType::Comment_Start: + case TokType::Comment: member = parse_comment(); break; @@ -4778,7 +4875,14 @@ CodeUsing parse_using() array_expr = parse_array_decl(); + Token stmt_end = currtok; eat( TokType::Statement_End ); + + CodeComment inline_cmt = NoCode; + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + { + inline_cmt = parse_comment(); + } using namespace ECode; @@ -4803,6 +4907,9 @@ CodeUsing parse_using() if ( attributes ) result->Attributes = attributes; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; } Context.pop(); diff --git a/project/components/interface.upfront.cpp b/project/components/interface.upfront.cpp index a69b7f7..764c92a 100644 --- a/project/components/interface.upfront.cpp +++ b/project/components/interface.upfront.cpp @@ -415,11 +415,41 @@ CodeComment def_comment( StrC content ) return CodeInvalid; } + static char line[ MaxCommentLineLength ]; + + String cmt_formatted = String::make_reserve( GlobalAllocator, kilobytes(1) ); + char const* end = content.Ptr + content.Len; + char const* scanner = content.Ptr; + s32 curr = 0; + do + { + char const* next = scanner; + s32 length = 0; + while ( next != end && scanner[ length ] != '\n' ) + { + next = scanner + length; + length++; + } + length++; + + str_copy( line, scanner, length ); + cmt_formatted.append_fmt( "//%.*s", length, line ); + mem_set( line, 0, MaxCommentLineLength ); + + scanner += length; + } + while ( scanner <= end ); + + if ( cmt_formatted.back() != '\n' ) + cmt_formatted.append( "\n" ); + Code result = make_code(); result->Type = ECode::Comment; - result->Name = get_cached_string( content ); + result->Name = get_cached_string( cmt_formatted ); result->Content = result->Name; + + cmt_formatted.free(); return (CodeComment) result; } diff --git a/project/enums/ETokType.csv b/project/enums/ETokType.csv index 49a2fe3..608d053 100644 --- a/project/enums/ETokType.csv +++ b/project/enums/ETokType.csv @@ -15,7 +15,7 @@ BraceSquare_Open, "[" BraceSquare_Close, "]" Capture_Start, "(" Capture_End, ")" -Comment, "__comemnt__" +Comment, "__comment__" Comment_End, "__comment_end__" Comment_Start, "__comment_start__" Char, "__character__" diff --git a/singleheader/singleheader.cpp b/singleheader/singleheader.cpp index a699613..f79c017 100644 --- a/singleheader/singleheader.cpp +++ b/singleheader/singleheader.cpp @@ -17,7 +17,7 @@ GEN_NS_END using namespace gen; constexpr char const* generation_notice = -"// This file was generated automatially by gen.bootstrap.cpp " +"// This file was generated automatially by gencpp's singleheader.cpp" "(See: https://github.com/Ed94/gencpp)\n\n"; constexpr StrC implementation_guard_start = txt(R"( diff --git a/test/test.cpp b/test/test.cpp index 47b36f5..405b160 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -7,7 +7,7 @@ #include "gen.builder.cpp" #include "sanity.cpp" #include "SOA.cpp" -#include "test.singleheader.cpp" +#include "validate.singleheader.cpp" int gen_main() { @@ -18,7 +18,7 @@ int gen_main() // check_SOA(); - // check_singleheader_ast(); + check_singleheader_ast(); return 0; } #endif From c81f4b34ee93a49b5b77e8a1327cdbdc602bb711 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 23 Aug 2023 02:17:47 -0400 Subject: [PATCH 03/26] Cleanup and doc updates --- docs/ASTs.md | 684 +++++++++++++++++++++++ docs/Parsing.md | 16 +- project/components/ast.cpp | 59 +- project/components/ast_types.hpp | 13 +- project/components/interface.parsing.cpp | 32 +- 5 files changed, 733 insertions(+), 71 deletions(-) create mode 100644 docs/ASTs.md diff --git a/docs/ASTs.md b/docs/ASTs.md new file mode 100644 index 0000000..f44f258 --- /dev/null +++ b/docs/ASTs.md @@ -0,0 +1,684 @@ +# ASTs Documentation + +While the Readme for docs covers the data layout per AST, this will focus on the AST types avaialble, and their nuances. + +## Body + +These are containers representing a scope body of a definition that can be of the following `ECode` type: + +* Class_Body +* Enum_Body +* Export_Body +* Extern_Linkage_Body +* Function_Body +* Global_Body +* Namespace_Body +* Struct_Body +* Union_Body + +Fields: + +```cpp +Code Front; +Code Back; +Code Parent; +StringCached Name; +CodeT Type; +s32 NumEntries; +``` + +The `Front` member represents the start of the link list and `Back` the end. +NumEntries is the number of entries in the body. + +Parent should have a compatible ECode type for the type of defintion used. + +Serialization: + +Will output only the entries, the braces are handled by the parent. + +```cpp +... + +``` + +## Attributes + +Represent standard or vendor specific C/C++ attributes. + +Fields: + +```cpp +StringCached Content; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp + +``` + +While the parser supports the `__declspec` and `__attribute__` syntax, the upfront constructor ( def_attributes ) must have the user specify the entire attribute, including the `[[]]`, `__declspec` or `__attribute__` parts. + +## Comment + +Stores a comment. + +Fields: + +```cpp +StringCached Content; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp + +``` + +The parser will perserve comments found if residing with a body or in accepted inline-to-definition locations. +Otherwise they will be skipped by the TokArray::__eat and TokArray::current( skip foramtting enabled ) functions. + +The upfront constructor: `def_comment` expects to recieve a comment without the `//` or `/* */` parts. It will add them during construction. + +## Class & Struct + +Fields: + +```cpp +CodeComment InlineCmt; // Only supported by forward declarations +CodeAttributes Attributes; +CodeType ParentType; +CodeBody Body; +CodeType Last; // Used to store references to interfaces +CodeType Next; // Used to store references to interfaces +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +AccessSpec ParentAccess; +``` + +Serialization: + +```cpp +// Class_Fwd + ; + +// Class + : , public , ... +{ + +}; +``` + +You'll notice that only one parent type is supported only with parent access. This library only supports single inheritance, the rest must be done through interfaces. + +## Constructor + +Fields: + +```cpp +CodeComment InlineCmt; // Only supported by forward declarations +Code InitializerList; +CodeParam Params; +Code Body; +Code Prev; +Code Next; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +// Constructor_Fwd + Name>( ); + +// Constructor + Name>( ): +{ + +} +``` + +## Define + +Represents a preprocessor define + +Fields: + +```cpp +StringCached Content; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp +#define +``` + +## Destructor + +Fields: + +```cpp +CodeComment InlineCmt; +CodeSpecifiers Specs; +Code Body; +Code Prev; +Code Next; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +// Destructor_Fwd + ~Name>( ) ; + +// Destructor + ~Name>( ) +{ + +} +``` + +## Enum + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeType UnderlyingType; +CodeBody Body; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp +// Enum_Fwd + enum class : ; + +// Enum + : +{ + +}; +``` + +## Execution + +Just represents an execution body. Equivalent to an untyped body. +Will be obsolute when function body parsing is implemented. + +Fields: + +```cpp +StringCached Content; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp + +``` + +## External Linkage + +Fields: + +```cpp +CodeBody Body; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp +extern "" +{ + +} +``` + +## Include + +Fields: + +```cpp +StringCached Content; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp +#include +``` + +## Friend + +This library (until its necessary become some third-party library to do otherwise) does not support friend declarations with in-statment function definitions. + +Fields: + +```cpp +CodeComment InlineCmt; +Code Declaration; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp +friend ; +``` + +## Function + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeSpecifiers Specs; +CodeType ReturnType; +CodeParam Params; +CodeBody Body; +Code Prev; +Code Parent; +Code Next; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp +// Function_Fwd + ( ) ; + +// Function + ( ) +{ + +} +``` + +## Module + +Fields: + +```cpp +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp + module ; +``` + +## Namespace + +Fields: + +```cpp +CodeBody Body; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp + namespace +{ + +} +``` + +## Operator Overload + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeSpecifiers Specs; +CodeType ReturnType; +CodeParam Params; +CodeBody Body; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +OperatorT Op; +``` + +Serialization: + +```cpp +// Operator_Fwd + operator ( ) ; + +// Operator + operator ( ) +{ + +} +``` + +## Operator Cast Overload ( User-Defined Type Conversion ) + +Fields: + +```cpp +CodeComment InlineCmt; +CodeSpecifiers Specs; +CodeType ValueType; +CodeBody Body; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp +// Operator_Cast_Fwd + operator () ; + +// Operator_Cast + operator () +{ + +} +``` + +## Parameters + +Fields: + +```cpp +CodeType ValueType; +Code Value; +CodeParam Last; +CodeParam Next; +Code Parent; +StringCached Name; +CodeT Type; +s32 NumEntries; +``` + +Serialization: + +```cpp + , ... +``` + +## Pragma + +Fields: + +```cpp +StringCached Content; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp +#pragma +``` + +## Preprocessor Conditional + +Fields: + +```cpp +StringCached Content; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp +# +``` + +## Specifiers + +Fields: + +```cpp +SpecifierT ArrSpecs[ AST::ArrSpecs_Cap ]; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +s32 NumEntries; +``` + +Serialization: + +```cpp +, ... +``` + +## Template + +Fields: + +```cpp +CodeParam Params; +Code Declaration; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp + +template< > + +``` + +## Typename + +Typenames represent the type "symbol". + +Fields: + +```cpp +CodeAttributes Attributes; +CodeSpecifiers Specs; +Code ArrExpr; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +``` + +Serialization: + +```cpp + +``` + +## Typedef + +Behave as usual except function or macro typedefs. +Those don't use the underlying type field as everything was serialized under the Name field. + +Fields: + +```cpp +CodeComment InlineCmt; +Code UnderlyingType; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +b32 IsFunction; +``` + +Serialization: + +```cpp +// Regular + typedef ; + +// Functions + typedef ; +``` + +## Union + +Fields: + +```cpp +CodeAttributes Attributes; +CodeBody Body; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp + union +{ + +} +``` + +## Using + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeType UnderlyingType; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp +// Regular + using = ; + +// Namespace + using namespace ; +``` + +## Variable + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeSpecifiers Specs; +CodeType ValueType; +Code BitfieldSize; +Code Value; +Code Prev; +Code Next; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp +// Regular + = ; + +// Bitfield + : = ; +``` diff --git a/docs/Parsing.md b/docs/Parsing.md index 9a538bd..12f189b 100644 --- a/docs/Parsing.md +++ b/docs/Parsing.md @@ -1,7 +1,9 @@ # Parsing The library features a naive parser tailored for only what the library needs to construct the supported syntax of C++ into its AST. -This parser does not, and should not do the compiler's job. By only supporting this minimal set of features, the parser is kept (so far) under 5000 loc. +This parser does not, and should not do the compiler's job. By only supporting this minimal set of features, the parser is kept (so far) around 5000 loc. + +You can think of this parser of a frontend parser vs a semantic parser. Its intuitively similar to WYSIWYG. What you precerive as the syntax from the user-side before the compiler gets a hold of it, is what you get. The parsing implementation supports the following for the user: @@ -27,10 +29,12 @@ CodeUsing parse_using ( StrC using_def ); CodeVar parse_variable ( StrC var_def ); ``` +To parse file buffers, use the `parse_global_body` function. + ***Parsing will aggregate any tokens within a function body or expression statement to an untyped Code AST.*** Everything is done in one pass for both the preprocessor directives and the rest of the language. -The parser performs no macro expansion as the scope of gencpp feature-set is to only support the preprocessor for the goal of having rudimentary awareness of preprocessor ***conditionals***, ***defines***, and ***includes***, and ***pragmas***. +The parser performs no macro expansion as the scope of gencpp feature-set is to only support the preprocessor for the goal of having rudimentary awareness of preprocessor ***conditionals***, ***defines***, ***includes***, and ***pragmas***. The keywords supported for the preprocessor are: @@ -51,10 +55,17 @@ Any preprocessor definition abuse that changes the syntax of the core language i Exceptions: * function signatures are allowed for a preprocessed macro: `neverinline MACRO() { ... }` + * Disable with: `#define GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES` * typedefs allow for a preprocessed macro: `typedef MACRO();` + * Disable with: `#define GEN_PARSER_DISABLE_MACRO_TYPEDEF` *(See functions `parse_operator_function_or_variable` and `parse_typedef` )* +Adding your own exceptions is possible by simply modifying the parser to allow for the syntax you need. + +*Note: You could interpret this strictness as a feature. This would allow the user to see if their codebase or a third-party's codebase some some egregious preprocessor abuse.* + + The lexing and parsing takes shortcuts from whats expected in the standard. * Numeric literals are not checked for validity. @@ -69,3 +80,4 @@ The lexing and parsing takes shortcuts from whats expected in the standard. * Parsing attributes can be extended to support user defined macros by defining `GEN_DEFINE_ATTRIBUTE_TOKENS` (see `gen.hpp` for the formatting) Empty lines used throughout the file are preserved for formatting purposes during ast serialization. + diff --git a/project/components/ast.cpp b/project/components/ast.cpp index 9836fd3..86055e4 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -38,49 +38,8 @@ String AST::to_string() case Untyped: case Execution: - result.append( Content ); - break; - case Comment: - { - // TODO : Move this formmating process to def_comment, - // Were going to preserve as much of the original formatting as possible - // so that the parsed comments don't have any artifacts. - // Just doing what untyped and execution do -#if 0 - if ( Prev && Prev->Type != Comment && Prev->Type != NewLine ) - result.append( "\n" ); - - static char line[ MaxCommentLineLength ]; - - char const* end = & scast(String, Content).back(); - char* scanner = Content.Data; - s32 curr = 0; - do - { - char const* next = scanner; - s32 length = 0; - while ( next != end && scanner[ length ] != '\n' ) - { - next = scanner + length; - length++; - } - length++; - - str_copy( line, scanner, length ); - result.append_fmt( "//%.*s", length, line ); - mem_set( line, 0, MaxCommentLineLength ); - - scanner += length; - } - while ( scanner <= end ); - - if ( result.back() != '\n' ) - result.append( "\n" ); -#else result.append( Content ); -#endif - } break; case Access_Private: @@ -148,7 +107,7 @@ String AST::to_string() result.append_fmt( "class %S %S", Attributes->to_string(), Name ); else result.append_fmt( "class %S", Name ); - + // Check if it can have an end-statement if ( Parent == nullptr || ( Parent->Type != ECode::Typedef && Parent->Type != ECode::Variable ) ) { @@ -226,7 +185,7 @@ String AST::to_string() } else result.append_fmt( "~%S();", Parent->Name ); - + if ( InlineCmt ) result.append_fmt( " %S", InlineCmt->Content ); else @@ -364,7 +323,7 @@ String AST::to_string() { result.append( ";" ); } - + if ( InlineCmt ) result.append_fmt(" %S", InlineCmt->Content ); else @@ -539,7 +498,7 @@ String AST::to_string() } } } - + if ( InlineCmt ) result.append_fmt( "; %S", InlineCmt->Content ); else @@ -552,7 +511,7 @@ String AST::to_string() if ( Specs ) { // TODO : Add support for specifies before the operator keyword - + if ( Name && Name.length() ) result.append_fmt( "%Soperator %S()", Name, ValueType->to_string() ); else @@ -582,7 +541,7 @@ String AST::to_string() if ( Specs ) { // TODO : Add support for specifies before the operator keyword - + result.append_fmt( "operator %S()", ValueType->to_string() ); for ( SpecifierT spec : Specs->cast() ) @@ -795,7 +754,7 @@ String AST::to_string() { result.append( ";" ); } - + if ( InlineCmt ) result.append_fmt(" %S", InlineCmt->Content); else @@ -872,7 +831,7 @@ String AST::to_string() } else result.append_fmt( "using %S;", Name ); - + if ( InlineCmt ) result.append_fmt(" %S\n", InlineCmt->Content ); else @@ -927,7 +886,7 @@ String AST::to_string() else result.append_fmt( "%S %S;", UnderlyingType->to_string(), Name ); - + if ( InlineCmt ) result.append_fmt(" %S", InlineCmt->Content); else diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 860feab..15710e2 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -57,7 +57,7 @@ struct AST_Class char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - CodeComment InlineCmt; + CodeComment InlineCmt; // Only supported by forward declarations CodeAttributes Attributes; char _PAD_SPECS_ [ sizeof(AST*) ]; CodeType ParentType; @@ -81,11 +81,12 @@ struct AST_Constructor char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - CodeComment InlineCmt; - char _PAD_PROPERTIES_ [ sizeof(AST*) * 3 ]; - Code InitializerList; - CodeParam Params; - Code Body; + CodeComment InlineCmt; // Only supported by forward declarations + char _PAD_PROPERTIES_ [ sizeof(AST*) * 1 ]; + CodeSpecifiers Specs; + Code InitializerList; + CodeParam Params; + Code Body; }; }; Code Prev; diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 3f2d357..caf14dd 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -1184,7 +1184,7 @@ CodeComment parse_comment() using namespace Parser; StackNode scope { nullptr, currtok_noskip, NullToken, txt( __func__ ) }; Context.push( & scope ); - + CodeComment result = (CodeComment) make_code(); result->Type = ECode::Comment; @@ -1812,7 +1812,7 @@ CodeFn parse_function_after_name( { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type && TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); } @@ -1855,7 +1855,7 @@ CodeFn parse_function_after_name( if ( params ) result->Params = params; - + if ( inline_cmt ) result->InlineCmt = inline_cmt; @@ -2142,7 +2142,7 @@ CodeOperator parse_operator_after_ret_type( { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); } @@ -2152,7 +2152,7 @@ CodeOperator parse_operator_after_ret_type( if ( inline_cmt ) result->InlineCmt = inline_cmt; - + Context.pop(); return result; } @@ -2319,7 +2319,7 @@ Code parse_simple_preprocess( Parser::TokType which ) { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) eat( TokType::Comment ); } @@ -2335,7 +2335,7 @@ Code parse_simple_preprocess( Parser::TokType which ) { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) eat( TokType::Comment ); } @@ -2361,6 +2361,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes Code result = CodeInvalid; +#ifndef GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES if ( currtok.Type == TokType::Preprocess_Macro ) { // Were dealing with a macro after attributes/specifiers. @@ -2368,6 +2369,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes Context.pop(); return result; } +#endif CodeType type = parse_type(); @@ -3594,12 +3596,12 @@ CodeEnum parse_enum( bool inplace_def ) } CodeComment inline_cmt = NoCode; - + if ( ! inplace_def ) { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); } @@ -3626,7 +3628,7 @@ CodeEnum parse_enum( bool inplace_def ) if ( type ) result->UnderlyingType = type; - + if ( inline_cmt ) result->InlineCmt = inline_cmt; @@ -4518,7 +4520,11 @@ CodeTypedef parse_typedef() constexpr bool from_typedef = true; +#if GEN_PARSER_DISABLE_MACRO_TYPEDEF + if ( false ) +#else if ( check( TokType::Preprocess_Macro )) +#endif { type = t_empty; name = currtok; @@ -4631,10 +4637,10 @@ CodeTypedef parse_typedef() } array_expr = parse_array_decl(); - + Token stmt_end = currtok; eat( TokType::Statement_End ); - + CodeComment inline_cmt = NoCode; if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); @@ -4661,7 +4667,7 @@ CodeTypedef parse_typedef() if ( type->Type == Typename && array_expr && array_expr->Type != Invalid ) type.cast()->ArrExpr = array_expr; - + if ( inline_cmt ) result->InlineCmt = inline_cmt; From f9117a23532564f252437f6502a887ba048ed463 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 23 Aug 2023 11:05:49 -0400 Subject: [PATCH 04/26] Added zpl's ivrtual memory to dependencies (unused for now) --- gencpp.10x | 1 + project/components/ast.cpp | 3 + project/components/ast.hpp | 6 +- project/components/ast_types.hpp | 3 +- project/components/interface.parsing.cpp | 10 ++ project/dependencies/memory.cpp | 126 +++++++++++++++++++++++ project/dependencies/memory.hpp | 27 +++++ 7 files changed, 173 insertions(+), 3 deletions(-) diff --git a/gencpp.10x b/gencpp.10x index 8cb3c94..c8d06e3 100644 --- a/gencpp.10x +++ b/gencpp.10x @@ -40,6 +40,7 @@ GEN_TIME + GEN_SYSTEM_WINDOWS diff --git a/project/components/ast.cpp b/project/components/ast.cpp index 86055e4..2fb0ff3 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -779,6 +779,9 @@ String AST::to_string() { result.append_fmt( "%S", Name ); } + + if ( IsParamPack ) + result.append("..."); } break; diff --git a/project/components/ast.hpp b/project/components/ast.hpp index 07f669a..96040ca 100644 --- a/project/components/ast.hpp +++ b/project/components/ast.hpp @@ -264,7 +264,8 @@ struct AST CodeT Type; ModuleFlag ModuleFlags; union { - b32 IsFunction; // Used by typedef to not serialize the name field. + b32 IsFunction; // Used by typedef to not serialize the name field. + b32 IsParamPack; // Used by typename to know if type should be considered a parameter pack. OperatorT Op; AccessSpec ParentAccess; s32 NumEntries; @@ -314,7 +315,8 @@ struct AST_POD CodeT Type; ModuleFlag ModuleFlags; union { - b32 IsFunction; // Used by typedef to not serialize the name field. + b32 IsFunction; // Used by typedef to not serialize the name field. + b32 IsParamPack; // Used by typename to know if type should be considered a parameter pack. OperatorT Op; AccessSpec ParentAccess; s32 NumEntries; diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 15710e2..00ec5ad 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -459,7 +459,8 @@ struct AST_Type Code Parent; StringCached Name; CodeT Type; - char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + char _PAD_UNUSED_[ sizeof(ModuleFlag) ]; + b32 IsParamPack; }; static_assert( sizeof(AST_Type) == sizeof(AST), "ERROR: AST_Type is not the same size as AST"); diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index caf14dd..c65ef79 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -4454,6 +4454,13 @@ CodeType parse_type( bool* is_function ) brute_sig.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)brute_sig.Text; is_first_capture = false; } + + bool is_param_pack = false; + if ( check(TokType::Varadic_Argument) ) + { + is_param_pack = true; + eat( TokType::Varadic_Argument ); + } using namespace ECode; @@ -4479,6 +4486,9 @@ CodeType parse_type( bool* is_function ) if ( attributes ) result->Attributes = attributes; + + if ( is_param_pack ) + result->IsParamPack = true; Context.pop(); return result; diff --git a/project/dependencies/memory.cpp b/project/dependencies/memory.cpp index 9d7a17c..e9bdc53 100644 --- a/project/dependencies/memory.cpp +++ b/project/dependencies/memory.cpp @@ -206,6 +206,132 @@ void* heap_allocator_proc( void* allocator_data, AllocType type, sw size, sw ali return ptr; } +#pragma region VirtualMemory +VirtualMemory vm_from_memory( void* data, sw size ) +{ + VirtualMemory vm; + vm.data = data; + vm.size = size; + return vm; +} + +#if defined( GEN_SYSTEM_WINDOWS ) +VirtualMemory vm_alloc( void* addr, sw size ) +{ + VirtualMemory vm; + GEN_ASSERT( size > 0 ); + vm.data = VirtualAlloc( addr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE ); + vm.size = size; + return vm; +} + +b32 vm_free( VirtualMemory vm ) +{ + MEMORY_BASIC_INFORMATION info; + while ( vm.size > 0 ) + { + if ( VirtualQuery( vm.data, &info, size_of( info ) ) == 0 ) + return false; + if ( info.BaseAddress != vm.data || info.AllocationBase != vm.data || info.State != MEM_COMMIT || info.RegionSize > zpl_cast( uw ) vm.size ) + { + return false; + } + if ( VirtualFree( vm.data, 0, MEM_RELEASE ) == 0 ) + return false; + vm.data = pointer_add( vm.data, info.RegionSize ); + vm.size -= info.RegionSize; + } + return true; +} + +VirtualMemory vm_trim( VirtualMemory vm, sw lead_size, sw size ) +{ + VirtualMemory new_vm = { 0 }; + void* ptr; + GEN_ASSERT( vm.size >= lead_size + size ); + + ptr = pointer_add( vm.data, lead_size ); + + vm_free( vm ); + new_vm = vm_alloc( ptr, size ); + if ( new_vm.data == ptr ) + return new_vm; + if ( new_vm.data ) + vm_free( new_vm ); + return new_vm; +} + +b32 vm_purge( VirtualMemory vm ) +{ + VirtualAlloc( vm.data, vm.size, MEM_RESET, PAGE_READWRITE ); + // NOTE: Can this really fail? + return true; +} + +sw virtual_memory_page_size( sw* alignment_out ) +{ + SYSTEM_INFO info; + GetSystemInfo( &info ); + if ( alignment_out ) + *alignment_out = info.dwAllocationGranularity; + return info.dwPageSize; +} + +#else +# include + +# ifndef MAP_ANONYMOUS +# define MAP_ANONYMOUS MAP_ANON +# endif +VirtualMemory vm_alloc( void* addr, sw size ) +{ + VirtualMemory vm; + GEN_ASSERT( size > 0 ); + vm.data = mmap( addr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0 ); + vm.size = size; + return vm; +} + +b32 vm_free( VirtualMemory vm ) +{ + munmap( vm.data, vm.size ); + return true; +} + +VirtualMemory vm_trim( VirtualMemory vm, sw lead_size, sw size ) +{ + void* ptr; + sw trail_size; + GEN_ASSERT( vm.size >= lead_size + size ); + + ptr = pointer_add( vm.data, lead_size ); + trail_size = vm.size - lead_size - size; + + if ( lead_size != 0 ) + vm_free( vm_from_memory(( vm.data, lead_size ) ); + if ( trail_size != 0 ) + vm_free( vm_from_memory( ptr, trail_size ) ); + return vm_from_memory( ptr, size ); +} + +b32 vm_purge( VirtualMemory vm ) +{ + int err = madvise( vm.data, vm.size, MADV_DONTNEED ); + return err != 0; +} + +sw virtual_memory_page_size( sw* alignment_out ) +{ + // TODO: Is this always true? + sw result = zpl_cast( sw ) sysconf( _SC_PAGE_SIZE ); + if ( alignment_out ) + *alignment_out = result; + return result; +} +#endif + +#pragma endregion VirtualMemory + void* Arena::allocator_proc( void* allocator_data, AllocType type, sw size, sw alignment, void* old_memory, sw old_size, u64 flags ) { Arena* arena = rcast(Arena*, allocator_data); diff --git a/project/dependencies/memory.hpp b/project/dependencies/memory.hpp index d8335db..f87b68a 100644 --- a/project/dependencies/memory.hpp +++ b/project/dependencies/memory.hpp @@ -363,6 +363,33 @@ GEN_IMPL_INLINE void zero_size( void* ptr, sw size ) mem_set( ptr, 0, size ); } +struct VirtualMemory +{ + void* data; + sw size; +}; + +//! Initialize virtual memory from existing data. +VirtualMemory vm_from_memory( void* data, sw size ); + +//! Allocate virtual memory at address with size. + +//! @param addr The starting address of the region to reserve. If NULL, it lets operating system to decide where to allocate it. +//! @param size The size to serve. +VirtualMemory vm_alloc( void* addr, sw size ); + +//! Release the virtual memory. +b32 vm_free( VirtualMemory vm ); + +//! Trim virtual memory. +VirtualMemory vm_trim( VirtualMemory vm, sw lead_size, sw size ); + +//! Purge virtual memory. +b32 gen_vm_purge( VirtualMemory vm ); + +//! Retrieve VM's page size and alignment. +sw gen_virtual_memory_page_size( sw* alignment_out ); + struct Arena { static From 30eec99628ad3d93ef75e7b9d3116d5ac49a9309 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 23 Aug 2023 13:17:22 -0400 Subject: [PATCH 05/26] Changes to include usage, starting to attempt singleheader automated verification --- project/components/ast.cpp | 2 +- project/components/interface.hpp | 2 +- project/components/interface.parsing.cpp | 43 +++++++++++++++++++----- project/components/interface.upfront.cpp | 14 +++++--- test/validate.singleheader.cpp | 19 ++++++++++- 5 files changed, 64 insertions(+), 16 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index 2fb0ff3..b0a3cc4 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -609,7 +609,7 @@ String AST::to_string() break; case Preprocess_Include: - result.append_fmt( "#include \"%S\"\n", Content ); + result.append_fmt( "#include %S\n", Content ); break; case Preprocess_ElIf: diff --git a/project/components/interface.hpp b/project/components/interface.hpp index 4b90046..91ecab9 100644 --- a/project/components/interface.hpp +++ b/project/components/interface.hpp @@ -68,7 +68,7 @@ CodeFn def_function( StrC name , CodeSpecifiers specifiers = NoCode, CodeAttributes attributes = NoCode , ModuleFlag mflags = ModuleFlag::None ); -CodeInclude def_include ( StrC content ); +CodeInclude def_include ( StrC content, bool foreign = false ); CodeModule def_module ( StrC name, ModuleFlag mflags = ModuleFlag::None ); CodeNS def_namespace( StrC name, Code body, ModuleFlag mflags = ModuleFlag::None ); diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index c65ef79..b20e8e6 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -56,6 +56,21 @@ namespace Parser { return scast(AccessSpec, Type); } + + String to_string() + { + String result = String::make_reserve( GlobalAllocator, kilobytes(4) ); + + StrC type_str = ETokType::to_str( Type ); + + result.append_fmt( "Line: %d Column: %d, Type: %.*s Content: %.*s" + , Line, Column + , type_str.Len, type_str.Ptr + , Length, Text + ); + + return result; + } }; constexpr Token NullToken { nullptr, 0, TokType::Invalid, false, 0, 0 }; @@ -303,6 +318,13 @@ namespace Parser while (left ) { + #if 0 + if (Tokens.num()) + { + log_fmt("\nLastTok: %S", Tokens.back().to_string()); + } + #endif + Token token = { scanner, 0, TokType::Invalid, line, column, false }; bool is_define = false; @@ -873,18 +895,15 @@ namespace Parser token.Text = scanner; token.Length = 1; token.Type = TokType::Operator; + move_forward(); if ( left ) { - move_forward(); - if ( current == '/' ) { token.Type = TokType::Comment; token.Length = 2; - move_forward(); - token.Length++; while ( left && current != '\n' && current != '\r' ) { @@ -909,9 +928,7 @@ namespace Parser { token.Type = TokType::Comment; token.Length = 2; - move_forward(); - token.Length++; bool star = current == '*'; bool slash = scanner[1] == '/'; @@ -925,11 +942,21 @@ namespace Parser slash = scanner[1] == '/'; at_end = star && slash; } - token.Length += 3; - Tokens.append( token ); + token.Length += 2; move_forward(); move_forward(); + if ( current == '\r' ) + { + move_forward(); + token.Length++; + } + if ( current == '\n' ) + { + move_forward(); + token.Length++; + } + Tokens.append( token ); end_line(); continue; } diff --git a/project/components/interface.upfront.cpp b/project/components/interface.upfront.cpp index 764c92a..af42b74 100644 --- a/project/components/interface.upfront.cpp +++ b/project/components/interface.upfront.cpp @@ -416,7 +416,7 @@ CodeComment def_comment( StrC content ) } static char line[ MaxCommentLineLength ]; - + String cmt_formatted = String::make_reserve( GlobalAllocator, kilobytes(1) ); char const* end = content.Ptr + content.Len; char const* scanner = content.Ptr; @@ -442,13 +442,13 @@ CodeComment def_comment( StrC content ) if ( cmt_formatted.back() != '\n' ) cmt_formatted.append( "\n" ); - + Code result = make_code(); result->Type = ECode::Comment; result->Name = get_cached_string( cmt_formatted ); result->Content = result->Name; - + cmt_formatted.free(); return (CodeComment) result; @@ -850,7 +850,7 @@ CodeFn def_function( StrC name return result; } -CodeInclude def_include ( StrC path ) +CodeInclude def_include( StrC path, bool foreign ) { if ( path.Len <= 0 || path.Ptr == nullptr ) { @@ -858,10 +858,14 @@ CodeInclude def_include ( StrC path ) return CodeInvalid; } + StrC content = foreign ? + to_str( str_fmt_buf( "<%.*s>\n", path.Len, path.Ptr )) + : to_str( str_fmt_buf( "\"%.*s\"\n", path.Len, path.Ptr )); + Code result = make_code(); result->Type = ECode::Preprocess_Include; - result->Name = get_cached_string( path ); + result->Name = get_cached_string( content ); result->Content = result->Name; return (CodeInclude) result; diff --git a/test/validate.singleheader.cpp b/test/validate.singleheader.cpp index e9d9388..6104cc4 100644 --- a/test/validate.singleheader.cpp +++ b/test/validate.singleheader.cpp @@ -27,7 +27,24 @@ void check_singleheader_ast() builder.print( ast ); builder.write(); - log_fmt("passed!! Time taken: %llu ms\n", time_rel_ms() - time_start); + log_fmt("Serialized. Time taken: %llu ms\n", time_rel_ms() - time_start); + + FileContents file_gen = file_read_contents( GlobalAllocator, true, "gen/singleheader_copy.gen.hpp" ); + + log_fmt("Reconstructing from generated file:\n"); + time_start = time_rel_ms(); + CodeBody ast_gen = parse_global_body( { file_gen.size, (char const*)file_gen.data } ); + + log_fmt("\nAst generated. Time taken: %llu ms\n", time_rel_ms() - time_start); + + time_start = time_rel_ms(); + + if ( ast.is_equal( ast_gen ) ) + log_fmt( "Passed!: AST passed validation!\n" ); + else + log_fmt( "Failed: AST did not pass validation\n" ); + + log_fmt( "Time taken: %llu ms\n", time_rel_ms() - time_start ); gen::deinit(); } From 8635b0fd1bf5800dc04d65184bdd742dd6ec002b Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 23 Aug 2023 13:17:30 -0400 Subject: [PATCH 06/26] doc update for parampack typename --- docs/ASTs.md | 3 ++- test/validate.singleheader.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/ASTs.md b/docs/ASTs.md index f44f258..81e19ba 100644 --- a/docs/ASTs.md +++ b/docs/ASTs.md @@ -567,12 +567,13 @@ Code Next; Code Parent; StringCached Name; CodeT Type; +b32 IsParamPack; ``` Serialization: ```cpp - + ``` ## Typedef diff --git a/test/validate.singleheader.cpp b/test/validate.singleheader.cpp index 6104cc4..331d942 100644 --- a/test/validate.singleheader.cpp +++ b/test/validate.singleheader.cpp @@ -31,7 +31,7 @@ void check_singleheader_ast() FileContents file_gen = file_read_contents( GlobalAllocator, true, "gen/singleheader_copy.gen.hpp" ); - log_fmt("Reconstructing from generated file:\n"); + log_fmt("\nReconstructing from generated file:\n"); time_start = time_rel_ms(); CodeBody ast_gen = parse_global_body( { file_gen.size, (char const*)file_gen.data } ); From a6766cf0b1ffcb828c9a7a901aca017497d8e326 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 23 Aug 2023 18:16:45 -0400 Subject: [PATCH 07/26] Singleheader validation test got through ast reconstruction, failed to validate the reconstructed AST. --- project/components/ast.cpp | 653 +++++++++++------------ project/components/gen/ast_inlines.hpp | 382 ++++++------- project/components/interface.parsing.cpp | 49 +- test/validate.singleheader.cpp | 4 +- 4 files changed, 535 insertions(+), 553 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index b0a3cc4..d4c964d 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -351,7 +351,7 @@ String AST::to_string() result.append_fmt( "%S)", Params->to_string() ); else - result.append( "void)" ); + result.append( ")" ); if ( Specs ) { @@ -390,7 +390,7 @@ String AST::to_string() result.append_fmt( "%S)", Params->to_string() ); else - result.append( "void)" ); + result.append( ")" ); if ( Specs ) { @@ -447,7 +447,7 @@ String AST::to_string() result.append_fmt( "%S)", Params->to_string() ); else - result.append( "void)" ); + result.append( ")" ); if ( Specs ) { @@ -485,7 +485,7 @@ String AST::to_string() result.append_fmt( "%S)", Params->to_string() ); else - result.append_fmt( "void)" ); + result.append_fmt( ")" ); if ( Specs ) { @@ -779,7 +779,7 @@ String AST::to_string() { result.append_fmt( "%S", Name ); } - + if ( IsParamPack ) result.append("..."); } @@ -930,15 +930,87 @@ bool AST::is_equal( AST* other ) AST nodes are compared with AST::is_equal. */ if ( other == nullptr ) + { + log_fmt( "AST::is_equal: other is null\nAST: %S", debug_str() ); return false; + } if ( Type != other->Type ) + { + log_fmt("AST::is_equal: Type check failure with other\nAST: %S\nOther: %S" + , debug_str() + , other->debug_str() + ); + return false; + } switch ( Type ) { using namespace ECode; + #define check_member_val( val ) \ + if ( val != other->val ) \ + { \ + log_fmt("AST::is_equal: Member - " #val "\n failed" \ + "AST : %S\n" \ + "Other: %S\n" \ + "For val member: " #val \ + , debug_str() \ + , other->debug_str() \ + ); \ + \ + return false; \ + } + + #define check_member_str( str ) \ + if ( str != other->str ) \ + { \ + log_fmt("AST::is_equal: Member string check failure with other\n" \ + "AST : %S\n" \ + "Other: %S\n" \ + "For str member: " #str \ + , debug_str() \ + , other->debug_str() \ + ); \ + \ + return false; \ + } + + #define check_member_ast( ast ) \ + if ( ast ) \ + { \ + if ( other->ast == nullptr ) \ + { \ + log_fmt("AST::is_equal: Failed for member " #ast " other equivalent param is null\n" \ + "AST : %S\n" \ + "Other: %S\n" \ + "For ast member: %S\n" \ + , debug_str() \ + , other->debug_str() \ + , ast->debug_str() \ + ); \ + \ + return false; \ + } \ + \ + if ( ! ast->is_equal( other->ast ) ) \ + { \ + log_fmt( "AST::is_equal: Failed for " #ast"\n" \ + "AST : %S\n" \ + "Other: %S\n" \ + "For ast member: %S\n" \ + "other's ast member: %S\n" \ + , debug_str() \ + , other->debug_str() \ + , ast->debug_str() \ + , other->ast->debug_str() \ + ); \ + } \ + \ + return false; \ + } + case NewLine: case Access_Public: case Access_Protected: @@ -952,29 +1024,20 @@ bool AST::is_equal( AST* other ) case PlatformAttributes: case Untyped: { - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; - - return Content == other->Content; + check_member_str( Content ); + check_member_ast( Prev ); + check_member_ast( Next ); } case Class_Fwd: case Struct_Fwd: { - if ( Name != other->Name ) - return false; - if ( ParentType != other->ParentType ) - return false; - if ( ParentAccess != other->ParentAccess ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_str( Name ); + check_member_ast( ParentType ); + check_member_val( ParentAccess ); + check_member_ast( Attributes ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -982,76 +1045,54 @@ bool AST::is_equal( AST* other ) case Class: case Struct: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( ParentType != other->ParentType ) - return false; - if ( ParentAccess != other->ParentAccess ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( ParentType ); + check_member_val( ParentAccess ); + check_member_ast( Attributes ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Constructor: { - if ( InitializerList ? ! InitializerList->is_equal( other->InitializerList ) : other->InitializerList != nullptr ) - return false; - if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_ast( InitializerList ); + check_member_ast( Params ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Constructor_Fwd: { - if ( InitializerList ? ! InitializerList->is_equal( other->InitializerList ) : other->InitializerList != nullptr ) - return false; - if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_ast( InitializerList ); + check_member_ast( Params ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Destructor: { - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_ast( Specs ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Destructor_Fwd: { - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_ast( Specs ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1059,20 +1100,13 @@ bool AST::is_equal( AST* other ) case Enum: case Enum_Class: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( UnderlyingType ? ! UnderlyingType->is_equal( other->UnderlyingType ) : other->UnderlyingType != nullptr ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( Attributes ); + check_member_ast( UnderlyingType ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1080,122 +1114,82 @@ bool AST::is_equal( AST* other ) case Enum_Fwd: case Enum_Class_Fwd: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( UnderlyingType ? ! UnderlyingType->is_equal( other->UnderlyingType ) : other->UnderlyingType != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( Attributes ); + check_member_ast( UnderlyingType ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Extern_Linkage: { - if ( Name != other->Name ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_str( Name ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Friend: { - if ( Name != other->Name ) - return false; - if ( Declaration ? ! Declaration->is_equal( other->Declaration ) : other->Declaration != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_str( Name ); + check_member_ast( Declaration ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Function: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( ReturnType ? ! ReturnType->is_equal( other->ReturnType ) : other->ReturnType != nullptr ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( ReturnType ); + check_member_ast( Attributes ); + check_member_ast( Specs ); + check_member_ast( Params ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Function_Fwd: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( ReturnType ? ! ReturnType->is_equal( other->ReturnType ) : other->ReturnType != nullptr ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( ReturnType ); + check_member_ast( Attributes ); + check_member_ast( Specs ); + check_member_ast( Params ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Module: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Namespace: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1203,24 +1197,15 @@ bool AST::is_equal( AST* other ) case Operator: case Operator_Member: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( ReturnType ? ! ReturnType->is_equal( other->ReturnType ) : other->ReturnType != nullptr ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( ReturnType ); + check_member_ast( Attributes ); + check_member_ast( Specs ); + check_member_ast( Params ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1228,56 +1213,37 @@ bool AST::is_equal( AST* other ) case Operator_Fwd: case Operator_Member_Fwd: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( ReturnType ? ! ReturnType->is_equal( other->ReturnType ) : other->ReturnType != nullptr ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( ReturnType ); + check_member_ast( Attributes ); + check_member_ast( Specs ); + check_member_ast( Params ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Operator_Cast: { - if ( Name != other->Name ) - return false; - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( ValueType ? ! ValueType->is_equal( other->ValueType ) : other->ValueType != nullptr ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_str( Name ); + check_member_ast( Specs ); + check_member_ast( ValueType ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Operator_Cast_Fwd: { - if ( Name != other->Name ) - return false; - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( ValueType ? ! ValueType->is_equal( other->ValueType ) : other->ValueType != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_str( Name ); + check_member_ast( Specs ); + check_member_ast( ValueType ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1286,15 +1252,42 @@ bool AST::is_equal( AST* other ) { if ( NumEntries > 1 ) { - if ( NumEntries != other->NumEntries ) - return false; + check_member_val( NumEntries ); AST* curr = this; AST* curr_other = other; while ( curr != nullptr ) { - if ( ! curr->is_equal( curr_other ) ) + if ( curr ) + { + if ( curr_other == nullptr ) + { + log_fmt("AST::is_equal: Failed for parameter, other equivalent param is null\n" + "AST : %S\n" + "Other: %S\n" + "For ast member: %S\n" + , curr->debug_str() + ); + + return false; + } + + if ( ! curr->is_equal( curr_other ) ) + { + log_fmt( "AST::is_equal: Failed for parameter\n" + "AST : %S\n" + "Other: %S\n" + "For ast member: %S\n" + "other's ast member: %S\n" + , debug_str() + , other->debug_str() + , curr->debug_str() + , curr_other->debug_str() + ); + } + return false; + } curr = curr->Next; curr_other = curr_other->Next; @@ -1303,28 +1296,20 @@ bool AST::is_equal( AST* other ) return true; } - if ( Name != other->Name ) - return false; - if ( ValueType ? ! ValueType->is_equal( other->ValueType ) : other->ValueType != nullptr ) - return false; - if ( Value ? ! Value->is_equal( other->Value ) : other->Value != nullptr ) - return false; - if ( ArrExpr ? ! ArrExpr->is_equal( other->ArrExpr ) : other->ArrExpr != nullptr ) - return false; + check_member_str( Name ); + check_member_ast( ValueType ); + check_member_ast( Value ); + check_member_ast( ArrExpr ); return true; } case Preprocess_Define: { - if ( Name != other->Name ) - return false; - if ( Content != other->Content ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_str( Name ); + check_member_str( Content ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1334,12 +1319,9 @@ bool AST::is_equal( AST* other ) case Preprocess_IfNotDef: case Preprocess_ElIf: { - if ( Content != other->Content ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_str( Content ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1347,100 +1329,70 @@ bool AST::is_equal( AST* other ) case Preprocess_Include: case Preprocess_Pragma: { - if ( Content != other->Content ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_str( Content ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Specifiers: { - if ( NumEntries != other->NumEntries ) - return false; - if ( Name != other->Name ) - return false; + check_member_val( NumEntries ); + check_member_str( Name ); for ( s32 idx = 0; idx < NumEntries; ++idx ) { - if ( ArrSpecs[ idx ] != other->ArrSpecs[ idx ] ) - return false; + check_member_val( ArrSpecs[ idx ] ); } - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; - + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Template: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( Params ? ! Params->is_equal( other->Params ) : other->Params != nullptr ) - return false; - if ( Declaration ? ! Declaration->is_equal( other->Declaration ) : other->Declaration != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( Params ); + check_member_ast( Declaration ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Typedef: { - if ( IsFunction != other->IsFunction ) - return false; - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( UnderlyingType != other->UnderlyingType ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( IsFunction ); + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( Specs ); + check_member_ast( UnderlyingType ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Typename: { - if ( Name != other->Name ) - return false; - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( ArrExpr ? ! ArrExpr->is_equal( other->ArrExpr ) : other->ArrExpr != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( IsParamPack ); + check_member_str( Name ); + check_member_ast( Specs ); + check_member_ast( ArrExpr ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Union: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( Body ? ! Body->is_equal( other->Body ) : other->Body != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( Attributes ); + check_member_ast( Body ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1448,42 +1400,27 @@ bool AST::is_equal( AST* other ) case Using: case Using_Namespace: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( UnderlyingType ? ! UnderlyingType->is_equal( other->UnderlyingType ) : other->UnderlyingType != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( UnderlyingType ); + check_member_ast( Attributes ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } case Variable: { - if ( ModuleFlags != other->ModuleFlags ) - return false; - if ( Name != other->Name ) - return false; - if ( Attributes ? ! Attributes->is_equal( other->Attributes ) : other->Attributes != nullptr ) - return false; - if ( Specs ? ! Specs->is_equal( other->Specs ) : other->Specs != nullptr ) - return false; - if ( ValueType ? ! ValueType->is_equal( other->ValueType ) : other->ValueType != nullptr ) - return false; - if ( BitfieldSize ? ! BitfieldSize->is_equal( other->BitfieldSize ) : other->BitfieldSize != nullptr ) - return false; - if ( Value ? ! Value->is_equal( other->Value ) : other->Value != nullptr ) - return false; - if ( Prev ? ! Prev->is_equal( other->Prev ) : other->Prev != nullptr ) - return false; - if ( Next ? ! Next->is_equal( other->Next ) : other->Next != nullptr ) - return false; + check_member_val( ModuleFlags ); + check_member_str( Name ); + check_member_ast( ValueType ); + check_member_ast( BitfieldSize ); + check_member_ast( Value ); + check_member_ast( Attributes ); + check_member_ast( Specs ); + check_member_ast( Prev ); + check_member_ast( Next ); return true; } @@ -1496,21 +1433,41 @@ bool AST::is_equal( AST* other ) case Struct_Body: case Union_Body: { - if ( NumEntries != other->NumEntries ) - return false; - - if ( Front != other->Front ) - return false; - - if ( Back != other->Back ) - return false; + check_member_val( NumEntries ); + check_member_ast( Front ); + check_member_ast( Back ); AST* curr = Front; AST* curr_other = other->Front; while ( curr != nullptr ) { - if ( ! curr->is_equal( curr_other ) ) + if ( curr_other == nullptr ) + { + log_fmt("AST::is_equal: Failed for body, other equivalent param is null\n" + "AST : %S\n" + "Other: %S\n" + "For ast member: %S\n" + , curr->debug_str() + ); + return false; + } + + if ( ! curr->is_equal( curr_other ) ) + { + log_fmt( "AST::is_equal: Failed for body\n" + "AST : %S\n" + "Other: %S\n" + "For ast member: %S\n" + "other's ast member: %S\n" + , debug_str() + , other->debug_str() + , curr->debug_str() + , curr_other->debug_str() + ); + + return false; + } curr = curr->Next; curr_other = curr_other->Next; @@ -1518,6 +1475,10 @@ bool AST::is_equal( AST* other ) return true; } + + #undef check_member_val + #undef check_member_str + #undef check_member_ast } return true; diff --git a/project/components/gen/ast_inlines.hpp b/project/components/gen/ast_inlines.hpp index f4bb2ab..7cb3fad 100644 --- a/project/components/gen/ast_inlines.hpp +++ b/project/components/gen/ast_inlines.hpp @@ -3,14 +3,14 @@ #pragma region generated code inline implementation -char const* Code::debug_str( void ) +char const* Code::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code Code::duplicate( void ) +Code Code::duplicate() { if ( ast == nullptr ) { @@ -30,12 +30,12 @@ bool Code::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool Code::is_valid( void ) +bool Code::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void Code::set_global( void ) +void Code::set_global() { if ( ast == nullptr ) { @@ -45,7 +45,7 @@ void Code::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String Code::to_string( void ) +String Code::to_string() { if ( ast == nullptr ) { @@ -81,14 +81,14 @@ Code::operator bool() return ast != nullptr; } -char const* CodeBody::debug_str( void ) +char const* CodeBody::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeBody::duplicate( void ) +Code CodeBody::duplicate() { if ( ast == nullptr ) { @@ -108,12 +108,12 @@ bool CodeBody::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeBody::is_valid( void ) +bool CodeBody::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeBody::set_global( void ) +void CodeBody::set_global() { if ( ast == nullptr ) { @@ -123,7 +123,7 @@ void CodeBody::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeBody::to_string( void ) +String CodeBody::to_string() { if ( ast == nullptr ) { @@ -159,14 +159,14 @@ CodeBody::operator bool() return ast != nullptr; } -char const* CodeAttributes::debug_str( void ) +char const* CodeAttributes::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeAttributes::duplicate( void ) +Code CodeAttributes::duplicate() { if ( ast == nullptr ) { @@ -186,12 +186,12 @@ bool CodeAttributes::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeAttributes::is_valid( void ) +bool CodeAttributes::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeAttributes::set_global( void ) +void CodeAttributes::set_global() { if ( ast == nullptr ) { @@ -201,7 +201,7 @@ void CodeAttributes::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeAttributes::to_string( void ) +String CodeAttributes::to_string() { if ( ast == nullptr ) { @@ -237,7 +237,7 @@ CodeAttributes::operator bool() return ast != nullptr; } -AST* CodeAttributes::raw( void ) +AST* CodeAttributes::raw() { return rcast( AST*, ast ); } @@ -247,7 +247,7 @@ CodeAttributes::operator Code() return *rcast( Code*, this ); } -AST_Attributes* CodeAttributes::operator->( void ) +AST_Attributes* CodeAttributes::operator->() { if ( ast == nullptr ) { @@ -257,14 +257,14 @@ AST_Attributes* CodeAttributes::operator->( void ) return ast; } -char const* CodeComment::debug_str( void ) +char const* CodeComment::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeComment::duplicate( void ) +Code CodeComment::duplicate() { if ( ast == nullptr ) { @@ -284,12 +284,12 @@ bool CodeComment::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeComment::is_valid( void ) +bool CodeComment::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeComment::set_global( void ) +void CodeComment::set_global() { if ( ast == nullptr ) { @@ -299,7 +299,7 @@ void CodeComment::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeComment::to_string( void ) +String CodeComment::to_string() { if ( ast == nullptr ) { @@ -335,7 +335,7 @@ CodeComment::operator bool() return ast != nullptr; } -AST* CodeComment::raw( void ) +AST* CodeComment::raw() { return rcast( AST*, ast ); } @@ -345,7 +345,7 @@ CodeComment::operator Code() return *rcast( Code*, this ); } -AST_Comment* CodeComment::operator->( void ) +AST_Comment* CodeComment::operator->() { if ( ast == nullptr ) { @@ -355,14 +355,14 @@ AST_Comment* CodeComment::operator->( void ) return ast; } -char const* CodeConstructor::debug_str( void ) +char const* CodeConstructor::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeConstructor::duplicate( void ) +Code CodeConstructor::duplicate() { if ( ast == nullptr ) { @@ -382,12 +382,12 @@ bool CodeConstructor::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeConstructor::is_valid( void ) +bool CodeConstructor::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeConstructor::set_global( void ) +void CodeConstructor::set_global() { if ( ast == nullptr ) { @@ -397,7 +397,7 @@ void CodeConstructor::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeConstructor::to_string( void ) +String CodeConstructor::to_string() { if ( ast == nullptr ) { @@ -433,7 +433,7 @@ CodeConstructor::operator bool() return ast != nullptr; } -AST* CodeConstructor::raw( void ) +AST* CodeConstructor::raw() { return rcast( AST*, ast ); } @@ -443,7 +443,7 @@ CodeConstructor::operator Code() return *rcast( Code*, this ); } -AST_Constructor* CodeConstructor::operator->( void ) +AST_Constructor* CodeConstructor::operator->() { if ( ast == nullptr ) { @@ -453,14 +453,14 @@ AST_Constructor* CodeConstructor::operator->( void ) return ast; } -char const* CodeClass::debug_str( void ) +char const* CodeClass::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeClass::duplicate( void ) +Code CodeClass::duplicate() { if ( ast == nullptr ) { @@ -480,12 +480,12 @@ bool CodeClass::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeClass::is_valid( void ) +bool CodeClass::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeClass::set_global( void ) +void CodeClass::set_global() { if ( ast == nullptr ) { @@ -495,7 +495,7 @@ void CodeClass::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeClass::to_string( void ) +String CodeClass::to_string() { if ( ast == nullptr ) { @@ -531,14 +531,14 @@ CodeClass::operator bool() return ast != nullptr; } -char const* CodeDefine::debug_str( void ) +char const* CodeDefine::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeDefine::duplicate( void ) +Code CodeDefine::duplicate() { if ( ast == nullptr ) { @@ -558,12 +558,12 @@ bool CodeDefine::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeDefine::is_valid( void ) +bool CodeDefine::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeDefine::set_global( void ) +void CodeDefine::set_global() { if ( ast == nullptr ) { @@ -573,7 +573,7 @@ void CodeDefine::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeDefine::to_string( void ) +String CodeDefine::to_string() { if ( ast == nullptr ) { @@ -609,7 +609,7 @@ CodeDefine::operator bool() return ast != nullptr; } -AST* CodeDefine::raw( void ) +AST* CodeDefine::raw() { return rcast( AST*, ast ); } @@ -619,7 +619,7 @@ CodeDefine::operator Code() return *rcast( Code*, this ); } -AST_Define* CodeDefine::operator->( void ) +AST_Define* CodeDefine::operator->() { if ( ast == nullptr ) { @@ -629,14 +629,14 @@ AST_Define* CodeDefine::operator->( void ) return ast; } -char const* CodeDestructor::debug_str( void ) +char const* CodeDestructor::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeDestructor::duplicate( void ) +Code CodeDestructor::duplicate() { if ( ast == nullptr ) { @@ -656,12 +656,12 @@ bool CodeDestructor::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeDestructor::is_valid( void ) +bool CodeDestructor::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeDestructor::set_global( void ) +void CodeDestructor::set_global() { if ( ast == nullptr ) { @@ -671,7 +671,7 @@ void CodeDestructor::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeDestructor::to_string( void ) +String CodeDestructor::to_string() { if ( ast == nullptr ) { @@ -707,7 +707,7 @@ CodeDestructor::operator bool() return ast != nullptr; } -AST* CodeDestructor::raw( void ) +AST* CodeDestructor::raw() { return rcast( AST*, ast ); } @@ -717,7 +717,7 @@ CodeDestructor::operator Code() return *rcast( Code*, this ); } -AST_Destructor* CodeDestructor::operator->( void ) +AST_Destructor* CodeDestructor::operator->() { if ( ast == nullptr ) { @@ -727,14 +727,14 @@ AST_Destructor* CodeDestructor::operator->( void ) return ast; } -char const* CodeEnum::debug_str( void ) +char const* CodeEnum::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeEnum::duplicate( void ) +Code CodeEnum::duplicate() { if ( ast == nullptr ) { @@ -754,12 +754,12 @@ bool CodeEnum::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeEnum::is_valid( void ) +bool CodeEnum::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeEnum::set_global( void ) +void CodeEnum::set_global() { if ( ast == nullptr ) { @@ -769,7 +769,7 @@ void CodeEnum::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeEnum::to_string( void ) +String CodeEnum::to_string() { if ( ast == nullptr ) { @@ -805,7 +805,7 @@ CodeEnum::operator bool() return ast != nullptr; } -AST* CodeEnum::raw( void ) +AST* CodeEnum::raw() { return rcast( AST*, ast ); } @@ -815,7 +815,7 @@ CodeEnum::operator Code() return *rcast( Code*, this ); } -AST_Enum* CodeEnum::operator->( void ) +AST_Enum* CodeEnum::operator->() { if ( ast == nullptr ) { @@ -825,14 +825,14 @@ AST_Enum* CodeEnum::operator->( void ) return ast; } -char const* CodeExec::debug_str( void ) +char const* CodeExec::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeExec::duplicate( void ) +Code CodeExec::duplicate() { if ( ast == nullptr ) { @@ -852,12 +852,12 @@ bool CodeExec::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeExec::is_valid( void ) +bool CodeExec::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeExec::set_global( void ) +void CodeExec::set_global() { if ( ast == nullptr ) { @@ -867,7 +867,7 @@ void CodeExec::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeExec::to_string( void ) +String CodeExec::to_string() { if ( ast == nullptr ) { @@ -903,7 +903,7 @@ CodeExec::operator bool() return ast != nullptr; } -AST* CodeExec::raw( void ) +AST* CodeExec::raw() { return rcast( AST*, ast ); } @@ -913,7 +913,7 @@ CodeExec::operator Code() return *rcast( Code*, this ); } -AST_Exec* CodeExec::operator->( void ) +AST_Exec* CodeExec::operator->() { if ( ast == nullptr ) { @@ -923,14 +923,14 @@ AST_Exec* CodeExec::operator->( void ) return ast; } -char const* CodeExtern::debug_str( void ) +char const* CodeExtern::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeExtern::duplicate( void ) +Code CodeExtern::duplicate() { if ( ast == nullptr ) { @@ -950,12 +950,12 @@ bool CodeExtern::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeExtern::is_valid( void ) +bool CodeExtern::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeExtern::set_global( void ) +void CodeExtern::set_global() { if ( ast == nullptr ) { @@ -965,7 +965,7 @@ void CodeExtern::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeExtern::to_string( void ) +String CodeExtern::to_string() { if ( ast == nullptr ) { @@ -1001,7 +1001,7 @@ CodeExtern::operator bool() return ast != nullptr; } -AST* CodeExtern::raw( void ) +AST* CodeExtern::raw() { return rcast( AST*, ast ); } @@ -1011,7 +1011,7 @@ CodeExtern::operator Code() return *rcast( Code*, this ); } -AST_Extern* CodeExtern::operator->( void ) +AST_Extern* CodeExtern::operator->() { if ( ast == nullptr ) { @@ -1021,14 +1021,14 @@ AST_Extern* CodeExtern::operator->( void ) return ast; } -char const* CodeFriend::debug_str( void ) +char const* CodeFriend::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeFriend::duplicate( void ) +Code CodeFriend::duplicate() { if ( ast == nullptr ) { @@ -1048,12 +1048,12 @@ bool CodeFriend::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeFriend::is_valid( void ) +bool CodeFriend::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeFriend::set_global( void ) +void CodeFriend::set_global() { if ( ast == nullptr ) { @@ -1063,7 +1063,7 @@ void CodeFriend::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeFriend::to_string( void ) +String CodeFriend::to_string() { if ( ast == nullptr ) { @@ -1099,7 +1099,7 @@ CodeFriend::operator bool() return ast != nullptr; } -AST* CodeFriend::raw( void ) +AST* CodeFriend::raw() { return rcast( AST*, ast ); } @@ -1109,7 +1109,7 @@ CodeFriend::operator Code() return *rcast( Code*, this ); } -AST_Friend* CodeFriend::operator->( void ) +AST_Friend* CodeFriend::operator->() { if ( ast == nullptr ) { @@ -1119,14 +1119,14 @@ AST_Friend* CodeFriend::operator->( void ) return ast; } -char const* CodeFn::debug_str( void ) +char const* CodeFn::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeFn::duplicate( void ) +Code CodeFn::duplicate() { if ( ast == nullptr ) { @@ -1146,12 +1146,12 @@ bool CodeFn::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeFn::is_valid( void ) +bool CodeFn::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeFn::set_global( void ) +void CodeFn::set_global() { if ( ast == nullptr ) { @@ -1161,7 +1161,7 @@ void CodeFn::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeFn::to_string( void ) +String CodeFn::to_string() { if ( ast == nullptr ) { @@ -1197,7 +1197,7 @@ CodeFn::operator bool() return ast != nullptr; } -AST* CodeFn::raw( void ) +AST* CodeFn::raw() { return rcast( AST*, ast ); } @@ -1207,7 +1207,7 @@ CodeFn::operator Code() return *rcast( Code*, this ); } -AST_Fn* CodeFn::operator->( void ) +AST_Fn* CodeFn::operator->() { if ( ast == nullptr ) { @@ -1217,14 +1217,14 @@ AST_Fn* CodeFn::operator->( void ) return ast; } -char const* CodeInclude::debug_str( void ) +char const* CodeInclude::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeInclude::duplicate( void ) +Code CodeInclude::duplicate() { if ( ast == nullptr ) { @@ -1244,12 +1244,12 @@ bool CodeInclude::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeInclude::is_valid( void ) +bool CodeInclude::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeInclude::set_global( void ) +void CodeInclude::set_global() { if ( ast == nullptr ) { @@ -1259,7 +1259,7 @@ void CodeInclude::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeInclude::to_string( void ) +String CodeInclude::to_string() { if ( ast == nullptr ) { @@ -1295,7 +1295,7 @@ CodeInclude::operator bool() return ast != nullptr; } -AST* CodeInclude::raw( void ) +AST* CodeInclude::raw() { return rcast( AST*, ast ); } @@ -1305,7 +1305,7 @@ CodeInclude::operator Code() return *rcast( Code*, this ); } -AST_Include* CodeInclude::operator->( void ) +AST_Include* CodeInclude::operator->() { if ( ast == nullptr ) { @@ -1315,14 +1315,14 @@ AST_Include* CodeInclude::operator->( void ) return ast; } -char const* CodeModule::debug_str( void ) +char const* CodeModule::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeModule::duplicate( void ) +Code CodeModule::duplicate() { if ( ast == nullptr ) { @@ -1342,12 +1342,12 @@ bool CodeModule::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeModule::is_valid( void ) +bool CodeModule::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeModule::set_global( void ) +void CodeModule::set_global() { if ( ast == nullptr ) { @@ -1357,7 +1357,7 @@ void CodeModule::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeModule::to_string( void ) +String CodeModule::to_string() { if ( ast == nullptr ) { @@ -1393,7 +1393,7 @@ CodeModule::operator bool() return ast != nullptr; } -AST* CodeModule::raw( void ) +AST* CodeModule::raw() { return rcast( AST*, ast ); } @@ -1403,7 +1403,7 @@ CodeModule::operator Code() return *rcast( Code*, this ); } -AST_Module* CodeModule::operator->( void ) +AST_Module* CodeModule::operator->() { if ( ast == nullptr ) { @@ -1413,14 +1413,14 @@ AST_Module* CodeModule::operator->( void ) return ast; } -char const* CodeNS::debug_str( void ) +char const* CodeNS::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeNS::duplicate( void ) +Code CodeNS::duplicate() { if ( ast == nullptr ) { @@ -1440,12 +1440,12 @@ bool CodeNS::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeNS::is_valid( void ) +bool CodeNS::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeNS::set_global( void ) +void CodeNS::set_global() { if ( ast == nullptr ) { @@ -1455,7 +1455,7 @@ void CodeNS::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeNS::to_string( void ) +String CodeNS::to_string() { if ( ast == nullptr ) { @@ -1491,7 +1491,7 @@ CodeNS::operator bool() return ast != nullptr; } -AST* CodeNS::raw( void ) +AST* CodeNS::raw() { return rcast( AST*, ast ); } @@ -1501,7 +1501,7 @@ CodeNS::operator Code() return *rcast( Code*, this ); } -AST_NS* CodeNS::operator->( void ) +AST_NS* CodeNS::operator->() { if ( ast == nullptr ) { @@ -1511,14 +1511,14 @@ AST_NS* CodeNS::operator->( void ) return ast; } -char const* CodeOperator::debug_str( void ) +char const* CodeOperator::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeOperator::duplicate( void ) +Code CodeOperator::duplicate() { if ( ast == nullptr ) { @@ -1538,12 +1538,12 @@ bool CodeOperator::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeOperator::is_valid( void ) +bool CodeOperator::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeOperator::set_global( void ) +void CodeOperator::set_global() { if ( ast == nullptr ) { @@ -1553,7 +1553,7 @@ void CodeOperator::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeOperator::to_string( void ) +String CodeOperator::to_string() { if ( ast == nullptr ) { @@ -1589,7 +1589,7 @@ CodeOperator::operator bool() return ast != nullptr; } -AST* CodeOperator::raw( void ) +AST* CodeOperator::raw() { return rcast( AST*, ast ); } @@ -1599,7 +1599,7 @@ CodeOperator::operator Code() return *rcast( Code*, this ); } -AST_Operator* CodeOperator::operator->( void ) +AST_Operator* CodeOperator::operator->() { if ( ast == nullptr ) { @@ -1609,14 +1609,14 @@ AST_Operator* CodeOperator::operator->( void ) return ast; } -char const* CodeOpCast::debug_str( void ) +char const* CodeOpCast::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeOpCast::duplicate( void ) +Code CodeOpCast::duplicate() { if ( ast == nullptr ) { @@ -1636,12 +1636,12 @@ bool CodeOpCast::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeOpCast::is_valid( void ) +bool CodeOpCast::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeOpCast::set_global( void ) +void CodeOpCast::set_global() { if ( ast == nullptr ) { @@ -1651,7 +1651,7 @@ void CodeOpCast::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeOpCast::to_string( void ) +String CodeOpCast::to_string() { if ( ast == nullptr ) { @@ -1687,7 +1687,7 @@ CodeOpCast::operator bool() return ast != nullptr; } -AST* CodeOpCast::raw( void ) +AST* CodeOpCast::raw() { return rcast( AST*, ast ); } @@ -1697,7 +1697,7 @@ CodeOpCast::operator Code() return *rcast( Code*, this ); } -AST_OpCast* CodeOpCast::operator->( void ) +AST_OpCast* CodeOpCast::operator->() { if ( ast == nullptr ) { @@ -1707,14 +1707,14 @@ AST_OpCast* CodeOpCast::operator->( void ) return ast; } -char const* CodeParam::debug_str( void ) +char const* CodeParam::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeParam::duplicate( void ) +Code CodeParam::duplicate() { if ( ast == nullptr ) { @@ -1734,12 +1734,12 @@ bool CodeParam::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeParam::is_valid( void ) +bool CodeParam::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeParam::set_global( void ) +void CodeParam::set_global() { if ( ast == nullptr ) { @@ -1749,7 +1749,7 @@ void CodeParam::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeParam::to_string( void ) +String CodeParam::to_string() { if ( ast == nullptr ) { @@ -1785,14 +1785,14 @@ CodeParam::operator bool() return ast != nullptr; } -char const* CodePragma::debug_str( void ) +char const* CodePragma::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodePragma::duplicate( void ) +Code CodePragma::duplicate() { if ( ast == nullptr ) { @@ -1812,12 +1812,12 @@ bool CodePragma::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodePragma::is_valid( void ) +bool CodePragma::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodePragma::set_global( void ) +void CodePragma::set_global() { if ( ast == nullptr ) { @@ -1827,7 +1827,7 @@ void CodePragma::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodePragma::to_string( void ) +String CodePragma::to_string() { if ( ast == nullptr ) { @@ -1863,7 +1863,7 @@ CodePragma::operator bool() return ast != nullptr; } -AST* CodePragma::raw( void ) +AST* CodePragma::raw() { return rcast( AST*, ast ); } @@ -1873,7 +1873,7 @@ CodePragma::operator Code() return *rcast( Code*, this ); } -AST_Pragma* CodePragma::operator->( void ) +AST_Pragma* CodePragma::operator->() { if ( ast == nullptr ) { @@ -1883,14 +1883,14 @@ AST_Pragma* CodePragma::operator->( void ) return ast; } -char const* CodePreprocessCond::debug_str( void ) +char const* CodePreprocessCond::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodePreprocessCond::duplicate( void ) +Code CodePreprocessCond::duplicate() { if ( ast == nullptr ) { @@ -1910,12 +1910,12 @@ bool CodePreprocessCond::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodePreprocessCond::is_valid( void ) +bool CodePreprocessCond::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodePreprocessCond::set_global( void ) +void CodePreprocessCond::set_global() { if ( ast == nullptr ) { @@ -1925,7 +1925,7 @@ void CodePreprocessCond::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodePreprocessCond::to_string( void ) +String CodePreprocessCond::to_string() { if ( ast == nullptr ) { @@ -1961,7 +1961,7 @@ CodePreprocessCond::operator bool() return ast != nullptr; } -AST* CodePreprocessCond::raw( void ) +AST* CodePreprocessCond::raw() { return rcast( AST*, ast ); } @@ -1971,7 +1971,7 @@ CodePreprocessCond::operator Code() return *rcast( Code*, this ); } -AST_PreprocessCond* CodePreprocessCond::operator->( void ) +AST_PreprocessCond* CodePreprocessCond::operator->() { if ( ast == nullptr ) { @@ -1981,14 +1981,14 @@ AST_PreprocessCond* CodePreprocessCond::operator->( void ) return ast; } -char const* CodeSpecifiers::debug_str( void ) +char const* CodeSpecifiers::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeSpecifiers::duplicate( void ) +Code CodeSpecifiers::duplicate() { if ( ast == nullptr ) { @@ -2008,12 +2008,12 @@ bool CodeSpecifiers::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeSpecifiers::is_valid( void ) +bool CodeSpecifiers::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeSpecifiers::set_global( void ) +void CodeSpecifiers::set_global() { if ( ast == nullptr ) { @@ -2023,7 +2023,7 @@ void CodeSpecifiers::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeSpecifiers::to_string( void ) +String CodeSpecifiers::to_string() { if ( ast == nullptr ) { @@ -2059,14 +2059,14 @@ CodeSpecifiers::operator bool() return ast != nullptr; } -char const* CodeStruct::debug_str( void ) +char const* CodeStruct::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeStruct::duplicate( void ) +Code CodeStruct::duplicate() { if ( ast == nullptr ) { @@ -2086,12 +2086,12 @@ bool CodeStruct::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeStruct::is_valid( void ) +bool CodeStruct::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeStruct::set_global( void ) +void CodeStruct::set_global() { if ( ast == nullptr ) { @@ -2101,7 +2101,7 @@ void CodeStruct::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeStruct::to_string( void ) +String CodeStruct::to_string() { if ( ast == nullptr ) { @@ -2137,14 +2137,14 @@ CodeStruct::operator bool() return ast != nullptr; } -char const* CodeTemplate::debug_str( void ) +char const* CodeTemplate::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeTemplate::duplicate( void ) +Code CodeTemplate::duplicate() { if ( ast == nullptr ) { @@ -2164,12 +2164,12 @@ bool CodeTemplate::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeTemplate::is_valid( void ) +bool CodeTemplate::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeTemplate::set_global( void ) +void CodeTemplate::set_global() { if ( ast == nullptr ) { @@ -2179,7 +2179,7 @@ void CodeTemplate::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeTemplate::to_string( void ) +String CodeTemplate::to_string() { if ( ast == nullptr ) { @@ -2215,7 +2215,7 @@ CodeTemplate::operator bool() return ast != nullptr; } -AST* CodeTemplate::raw( void ) +AST* CodeTemplate::raw() { return rcast( AST*, ast ); } @@ -2225,7 +2225,7 @@ CodeTemplate::operator Code() return *rcast( Code*, this ); } -AST_Template* CodeTemplate::operator->( void ) +AST_Template* CodeTemplate::operator->() { if ( ast == nullptr ) { @@ -2235,14 +2235,14 @@ AST_Template* CodeTemplate::operator->( void ) return ast; } -char const* CodeType::debug_str( void ) +char const* CodeType::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeType::duplicate( void ) +Code CodeType::duplicate() { if ( ast == nullptr ) { @@ -2262,12 +2262,12 @@ bool CodeType::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeType::is_valid( void ) +bool CodeType::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeType::set_global( void ) +void CodeType::set_global() { if ( ast == nullptr ) { @@ -2277,7 +2277,7 @@ void CodeType::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeType::to_string( void ) +String CodeType::to_string() { if ( ast == nullptr ) { @@ -2313,7 +2313,7 @@ CodeType::operator bool() return ast != nullptr; } -AST* CodeType::raw( void ) +AST* CodeType::raw() { return rcast( AST*, ast ); } @@ -2323,7 +2323,7 @@ CodeType::operator Code() return *rcast( Code*, this ); } -AST_Type* CodeType::operator->( void ) +AST_Type* CodeType::operator->() { if ( ast == nullptr ) { @@ -2333,14 +2333,14 @@ AST_Type* CodeType::operator->( void ) return ast; } -char const* CodeTypedef::debug_str( void ) +char const* CodeTypedef::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeTypedef::duplicate( void ) +Code CodeTypedef::duplicate() { if ( ast == nullptr ) { @@ -2360,12 +2360,12 @@ bool CodeTypedef::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeTypedef::is_valid( void ) +bool CodeTypedef::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeTypedef::set_global( void ) +void CodeTypedef::set_global() { if ( ast == nullptr ) { @@ -2375,7 +2375,7 @@ void CodeTypedef::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeTypedef::to_string( void ) +String CodeTypedef::to_string() { if ( ast == nullptr ) { @@ -2411,7 +2411,7 @@ CodeTypedef::operator bool() return ast != nullptr; } -AST* CodeTypedef::raw( void ) +AST* CodeTypedef::raw() { return rcast( AST*, ast ); } @@ -2421,7 +2421,7 @@ CodeTypedef::operator Code() return *rcast( Code*, this ); } -AST_Typedef* CodeTypedef::operator->( void ) +AST_Typedef* CodeTypedef::operator->() { if ( ast == nullptr ) { @@ -2431,14 +2431,14 @@ AST_Typedef* CodeTypedef::operator->( void ) return ast; } -char const* CodeUnion::debug_str( void ) +char const* CodeUnion::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeUnion::duplicate( void ) +Code CodeUnion::duplicate() { if ( ast == nullptr ) { @@ -2458,12 +2458,12 @@ bool CodeUnion::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeUnion::is_valid( void ) +bool CodeUnion::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeUnion::set_global( void ) +void CodeUnion::set_global() { if ( ast == nullptr ) { @@ -2473,7 +2473,7 @@ void CodeUnion::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeUnion::to_string( void ) +String CodeUnion::to_string() { if ( ast == nullptr ) { @@ -2509,7 +2509,7 @@ CodeUnion::operator bool() return ast != nullptr; } -AST* CodeUnion::raw( void ) +AST* CodeUnion::raw() { return rcast( AST*, ast ); } @@ -2519,7 +2519,7 @@ CodeUnion::operator Code() return *rcast( Code*, this ); } -AST_Union* CodeUnion::operator->( void ) +AST_Union* CodeUnion::operator->() { if ( ast == nullptr ) { @@ -2529,14 +2529,14 @@ AST_Union* CodeUnion::operator->( void ) return ast; } -char const* CodeUsing::debug_str( void ) +char const* CodeUsing::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeUsing::duplicate( void ) +Code CodeUsing::duplicate() { if ( ast == nullptr ) { @@ -2556,12 +2556,12 @@ bool CodeUsing::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeUsing::is_valid( void ) +bool CodeUsing::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeUsing::set_global( void ) +void CodeUsing::set_global() { if ( ast == nullptr ) { @@ -2571,7 +2571,7 @@ void CodeUsing::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeUsing::to_string( void ) +String CodeUsing::to_string() { if ( ast == nullptr ) { @@ -2607,7 +2607,7 @@ CodeUsing::operator bool() return ast != nullptr; } -AST* CodeUsing::raw( void ) +AST* CodeUsing::raw() { return rcast( AST*, ast ); } @@ -2617,7 +2617,7 @@ CodeUsing::operator Code() return *rcast( Code*, this ); } -AST_Using* CodeUsing::operator->( void ) +AST_Using* CodeUsing::operator->() { if ( ast == nullptr ) { @@ -2627,14 +2627,14 @@ AST_Using* CodeUsing::operator->( void ) return ast; } -char const* CodeVar::debug_str( void ) +char const* CodeVar::debug_str() { if ( ast == nullptr ) return "Code::debug_str: AST is null!"; return rcast( AST*, ast )->debug_str(); } -Code CodeVar::duplicate( void ) +Code CodeVar::duplicate() { if ( ast == nullptr ) { @@ -2654,12 +2654,12 @@ bool CodeVar::is_equal( Code other ) return rcast( AST*, ast )->is_equal( other.ast ); } -bool CodeVar::is_valid( void ) +bool CodeVar::is_valid() { return ( AST* )ast != nullptr && rcast( AST*, ast )->Type != CodeT::Invalid; } -void CodeVar::set_global( void ) +void CodeVar::set_global() { if ( ast == nullptr ) { @@ -2669,7 +2669,7 @@ void CodeVar::set_global( void ) rcast( AST*, ast )->Parent = Code::Global.ast; } -String CodeVar::to_string( void ) +String CodeVar::to_string() { if ( ast == nullptr ) { @@ -2705,7 +2705,7 @@ CodeVar::operator bool() return ast != nullptr; } -AST* CodeVar::raw( void ) +AST* CodeVar::raw() { return rcast( AST*, ast ); } @@ -2715,7 +2715,7 @@ CodeVar::operator Code() return *rcast( Code*, this ); } -AST_Var* CodeVar::operator->( void ) +AST_Var* CodeVar::operator->() { if ( ast == nullptr ) { diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index b20e8e6..8245673 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -149,10 +149,17 @@ namespace Parser { node->Prev = Scope; Scope = node; + + #if 0 && Build_Debug + log_fmt("\tEntering Context: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); + #endif } void pop() { + #if 0 && Build_Debug + log_fmt("\tPopping Context: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); + #endif Scope = Scope->Prev; } @@ -234,6 +241,10 @@ namespace Parser return false; } + #if 0 && Build_Debug + log_fmt("Ate: %S\n", Arr[Idx].to_string() ); + #endif + Idx++; return true; } @@ -1027,8 +1038,18 @@ namespace Parser } else { - String context_str = String::fmt_buf( GlobalAllocator, "%.*s", min( 100, left ), scanner ); + s32 start = max( 0, Tokens.num() - 100 ); + log_fmt("\n%d\n", start); + for ( s32 idx = start; idx < Tokens.num(); idx++ ) + { + log_fmt( "Token %d Type: %s : %.*s\n" + , idx + , ETokType::to_str( Tokens[ idx ].Type ).Ptr + , Tokens[ idx ].Length, Tokens[ idx ].Text + ); + } + String context_str = String::fmt_buf( GlobalAllocator, "%.*s", min( 100, left ), scanner ); log_failure( "Failed to lex token '%c' (%d, %d)\n%s", current, line, column, context_str ); // Skip to next whitespace since we can't know if anything else is valid until then. @@ -1175,7 +1196,7 @@ if ( def.Ptr == nullptr ) \ # define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ ) # define check( Type_ ) ( left && currtok.Type == Type_ ) -# define push_scope() \ +# define push_scope() \ StackNode scope { nullptr, currtok, NullToken, txt( __func__ ) }; \ Context.push( & scope ) @@ -2267,13 +2288,13 @@ CodeVar parse_variable_after_name( expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text; bitfield_expr = untyped_str( expr_tok ); } - + Token stmt_end = currtok; eat( TokType::Statement_End ); - + // Check for inline comment : = ; // CodeComment inline_cmt = NoCode; - if ( left + if ( left && ( currtok_noskip.Type == TokType::Comment ) && currtok_noskip.Line == stmt_end.Line ) { @@ -2304,7 +2325,7 @@ CodeVar parse_variable_after_name( if ( expr ) result->Value = expr; - + if ( inline_cmt ) result->InlineCmt = inline_cmt; @@ -2944,7 +2965,7 @@ Code parse_class_struct( Parser::TokType which, bool inplace_def = false ) { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); } @@ -2954,7 +2975,7 @@ Code parse_class_struct( Parser::TokType which, bool inplace_def = false ) else result = def_struct( name, body, (CodeType)parent, access, attributes, mflags ); - + if ( inline_cmt ) result->InlineCmt = inline_cmt; @@ -3337,7 +3358,7 @@ CodeConstructor parse_constructor() { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); } @@ -3357,7 +3378,7 @@ CodeConstructor parse_constructor() } else result->Type = ECode::Constructor_Fwd; - + if ( inline_cmt ) result->InlineCmt = inline_cmt; @@ -3795,10 +3816,10 @@ CodeFriend parse_friend() if ( params ) function->Params = params; } - + Token stmt_end = currtok; eat( TokType::Statement_End ); - + CodeComment inline_cmt = NoCode; if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); @@ -3812,7 +3833,7 @@ CodeFriend parse_friend() else result->Declaration = type; - + if ( inline_cmt ) result->InlineCmt = inline_cmt; @@ -4761,7 +4782,7 @@ CodeUnion parse_union( bool inplace_def ) while ( ! check_noskip( TokType::BraceCurly_Close ) ) { - if ( currtok.Type == TokType::Preprocess_Hash ) + if ( currtok_noskip.Type == TokType::Preprocess_Hash ) eat( TokType::Preprocess_Hash ); Code member = { nullptr }; diff --git a/test/validate.singleheader.cpp b/test/validate.singleheader.cpp index 331d942..f92130c 100644 --- a/test/validate.singleheader.cpp +++ b/test/validate.singleheader.cpp @@ -40,9 +40,9 @@ void check_singleheader_ast() time_start = time_rel_ms(); if ( ast.is_equal( ast_gen ) ) - log_fmt( "Passed!: AST passed validation!\n" ); + log_fmt( "\nPassed!: AST passed validation!\n" ); else - log_fmt( "Failed: AST did not pass validation\n" ); + log_fmt( "\nFailed: AST did not pass validation\n" ); log_fmt( "Time taken: %llu ms\n", time_rel_ms() - time_start ); From 9edcbad907115124283e1ffaeaa58ae39a012789 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 23 Aug 2023 21:19:31 -0400 Subject: [PATCH 08/26] Fixes to parsing marco content, progress on validation test (single-header) --- project/components/ast.cpp | 64 +++++++++++++++++------- project/components/inlines.hpp | 37 -------------- project/components/interface.parsing.cpp | 4 +- test/validate.singleheader.cpp | 2 +- 4 files changed, 49 insertions(+), 58 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index d4c964d..703fde2 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -4,6 +4,36 @@ Code Code::Global; Code Code::Invalid; +char const* AST::debug_str() +{ + if ( Parent ) + { + String + result = String::make_reserve( GlobalAllocator, kilobytes(1) ); + result.append_fmt( + "\nType : %s" + "\nParent : %s %s" + "\nName : %s" + , type_str() + , Parent->type_str() + , Parent->Name, Name ? Name : "" + ); + + return result; + } + + String + result = String::make_reserve( GlobalAllocator, kilobytes(1) ); + result.append_fmt( + "\nType : %s" + "\nName : %s" + , type_str() + , Name ? Name : "" + ); + + return result; +} + AST* AST::duplicate() { using namespace ECode; @@ -952,10 +982,9 @@ bool AST::is_equal( AST* other ) #define check_member_val( val ) \ if ( val != other->val ) \ { \ - log_fmt("AST::is_equal: Member - " #val "\n failed" \ + log_fmt("AST::is_equal: Member - " #val " failed\n" \ "AST : %S\n" \ "Other: %S\n" \ - "For val member: " #val \ , debug_str() \ , other->debug_str() \ ); \ @@ -963,18 +992,17 @@ bool AST::is_equal( AST* other ) return false; \ } - #define check_member_str( str ) \ - if ( str != other->str ) \ - { \ - log_fmt("AST::is_equal: Member string check failure with other\n" \ - "AST : %S\n" \ - "Other: %S\n" \ - "For str member: " #str \ - , debug_str() \ - , other->debug_str() \ - ); \ - \ - return false; \ + #define check_member_str( str ) \ + if ( str != other->str ) \ + { \ + log_fmt("AST::is_equal: Member string - "#str " failed\n" \ + "AST : %S\n" \ + "Other: %S\n" \ + , debug_str() \ + , other->debug_str() \ + ); \ + \ + return false; \ } #define check_member_ast( ast ) \ @@ -983,9 +1011,9 @@ bool AST::is_equal( AST* other ) if ( other->ast == nullptr ) \ { \ log_fmt("AST::is_equal: Failed for member " #ast " other equivalent param is null\n" \ - "AST : %S\n" \ - "Other: %S\n" \ - "For ast member: %S\n" \ + "AST : %s\n" \ + "Other: %s\n" \ + "For ast member: %s\n" \ , debug_str() \ , other->debug_str() \ , ast->debug_str() \ @@ -1000,7 +1028,7 @@ bool AST::is_equal( AST* other ) "AST : %S\n" \ "Other: %S\n" \ "For ast member: %S\n" \ - "other's ast member: %S\n" \ + "other ast member: %S\n" \ , debug_str() \ , other->debug_str() \ , ast->debug_str() \ diff --git a/project/components/inlines.hpp b/project/components/inlines.hpp index 06b0d51..20aac3d 100644 --- a/project/components/inlines.hpp +++ b/project/components/inlines.hpp @@ -25,43 +25,6 @@ void AST::append( AST* other ) NumEntries++; } -char const* AST::debug_str() -{ - if ( Parent ) - { - char const* fmt = stringize( - \nType : %s - \nParent : %s %s - \nName : %s - ); - - // These should be used immediately in a log. - // Thus if its desired to keep the debug str - // for multiple calls to bprintf, - // allocate this to proper string. - return str_fmt_buf( fmt - , type_str() - , Parent->Name - , Parent->type_str() - , Name ? Name : "" - ); - } - - char const* fmt = stringize( - \nType : %s - \nName : %s - ); - - // These should be used immediately in a log. - // Thus if its desired to keep the debug str - // for multiple calls to bprintf, - // allocate this to proper string. - return str_fmt_buf( fmt - , type_str() - , Name ? Name : "" - ); -} - Code& AST::entry( u32 idx ) { AST** current = & Front; diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 8245673..720ade6 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -523,10 +523,10 @@ namespace Parser s32 within_char = false; while ( left ) { - if ( current == '"' ) + if ( current == '"' && ! within_char ) within_string ^= true; - if ( current == '\'' ) + if ( current == '\'' && ! within_string ) within_char ^= true; if ( current == '\\' && ! within_string && ! within_char ) diff --git a/test/validate.singleheader.cpp b/test/validate.singleheader.cpp index f92130c..fbfef34 100644 --- a/test/validate.singleheader.cpp +++ b/test/validate.singleheader.cpp @@ -35,7 +35,7 @@ void check_singleheader_ast() time_start = time_rel_ms(); CodeBody ast_gen = parse_global_body( { file_gen.size, (char const*)file_gen.data } ); - log_fmt("\nAst generated. Time taken: %llu ms\n", time_rel_ms() - time_start); + log_fmt("\nAst generated. Time taken: %llu ms\n\n", time_rel_ms() - time_start); time_start = time_rel_ms(); From 9b6dc3cbd8173b651b1618308269c274805ff5a3 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 25 Aug 2023 18:40:13 -0400 Subject: [PATCH 09/26] Work on AST::is_equal. The NumEntries checks need to be deferred until the end as a final unresolved check on valdiation. As if there really is a discrepancy of entires it should be revealed by the specific entry failing. Right now the latest failure with the single header check involves a define directive specifically the define does omit whitespace properly and so the check interprets the different cached content to be non-equivalent. This will happen with all unvalidated aspects of the AST ( expressions, function bodies, etc ) There are two ways to resolve, either make an AST that can tokenize all items (not realistic), or I need to strip non-syntax important whitespace and then cache the string. This would mean removing everything but a single whitespace for all content within a content string. Otherwise, I would have to somehow make sure the content of the string has the exact formatting between both files for the definitions that matter. AST types with this issue: * Define Directive * Pragma Directive * Comment * Execution * Platform Attributes * Untyped Comments can technically be left unverified as they do not matter semantically. When the serialization is first emitted, the content these strings should for the most part be equivalent. However I do see some possible failures for that if a different style of bracket placment is used (between the serialization). At that point what I could do is just leave those unverified and just emit the content to the user as warning that the ast and the other compared could not be verified. Those technically can be handled on a per-eye basis, and worst case the tests with the compiler will in the determine if any critical defintions are missing for the user. --- project/components/ast.cpp | 138 +++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 66 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index 703fde2..3abecb2 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -1028,17 +1028,23 @@ bool AST::is_equal( AST* other ) "AST : %S\n" \ "Other: %S\n" \ "For ast member: %S\n" \ - "other ast member: %S\n" \ + "other's ast member: %S\n" \ , debug_str() \ , other->debug_str() \ , ast->debug_str() \ , other->ast->debug_str() \ ); \ - } \ \ - return false; \ + return false; \ + } \ } + // Need to check to make sure the prev->is_equal wont lead to a recursion. + // #define check_member_prev() \ + // if ( Prev ) \ + // { \ + + case NewLine: case Access_Public: case Access_Protected: @@ -1053,8 +1059,8 @@ bool AST::is_equal( AST* other ) case Untyped: { check_member_str( Content ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); } case Class_Fwd: @@ -1064,8 +1070,8 @@ bool AST::is_equal( AST* other ) check_member_ast( ParentType ); check_member_val( ParentAccess ); check_member_ast( Attributes ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1079,8 +1085,8 @@ bool AST::is_equal( AST* other ) check_member_val( ParentAccess ); check_member_ast( Attributes ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1090,8 +1096,8 @@ bool AST::is_equal( AST* other ) check_member_ast( InitializerList ); check_member_ast( Params ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1100,8 +1106,8 @@ bool AST::is_equal( AST* other ) { check_member_ast( InitializerList ); check_member_ast( Params ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1110,8 +1116,8 @@ bool AST::is_equal( AST* other ) { check_member_ast( Specs ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1119,8 +1125,8 @@ bool AST::is_equal( AST* other ) case Destructor_Fwd: { check_member_ast( Specs ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1133,8 +1139,8 @@ bool AST::is_equal( AST* other ) check_member_ast( Attributes ); check_member_ast( UnderlyingType ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1146,8 +1152,8 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Attributes ); check_member_ast( UnderlyingType ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1156,8 +1162,8 @@ bool AST::is_equal( AST* other ) { check_member_str( Name ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1166,8 +1172,8 @@ bool AST::is_equal( AST* other ) { check_member_str( Name ); check_member_ast( Declaration ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1181,8 +1187,8 @@ bool AST::is_equal( AST* other ) check_member_ast( Specs ); check_member_ast( Params ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1195,8 +1201,8 @@ bool AST::is_equal( AST* other ) check_member_ast( Attributes ); check_member_ast( Specs ); check_member_ast( Params ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1205,8 +1211,8 @@ bool AST::is_equal( AST* other ) { check_member_val( ModuleFlags ); check_member_str( Name ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1216,8 +1222,8 @@ bool AST::is_equal( AST* other ) check_member_val( ModuleFlags ); check_member_str( Name ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1232,8 +1238,8 @@ bool AST::is_equal( AST* other ) check_member_ast( Specs ); check_member_ast( Params ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1247,8 +1253,8 @@ bool AST::is_equal( AST* other ) check_member_ast( Attributes ); check_member_ast( Specs ); check_member_ast( Params ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1259,8 +1265,8 @@ bool AST::is_equal( AST* other ) check_member_ast( Specs ); check_member_ast( ValueType ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1270,8 +1276,8 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Specs ); check_member_ast( ValueType ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1280,8 +1286,6 @@ bool AST::is_equal( AST* other ) { if ( NumEntries > 1 ) { - check_member_val( NumEntries ); - AST* curr = this; AST* curr_other = other; while ( curr != nullptr ) @@ -1312,15 +1316,16 @@ bool AST::is_equal( AST* other ) , curr->debug_str() , curr_other->debug_str() ); + return false; } - - return false; } curr = curr->Next; curr_other = curr_other->Next; } + check_member_val( NumEntries ); + return true; } @@ -1336,8 +1341,8 @@ bool AST::is_equal( AST* other ) { check_member_str( Name ); check_member_str( Content ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1348,8 +1353,8 @@ bool AST::is_equal( AST* other ) case Preprocess_ElIf: { check_member_str( Content ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1358,8 +1363,8 @@ bool AST::is_equal( AST* other ) case Preprocess_Pragma: { check_member_str( Content ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1372,8 +1377,8 @@ bool AST::is_equal( AST* other ) { check_member_val( ArrSpecs[ idx ] ); } - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1383,8 +1388,8 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Params ); check_member_ast( Declaration ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1396,8 +1401,8 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Specs ); check_member_ast( UnderlyingType ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1407,8 +1412,8 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Specs ); check_member_ast( ArrExpr ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1419,8 +1424,8 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Attributes ); check_member_ast( Body ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1432,8 +1437,8 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( UnderlyingType ); check_member_ast( Attributes ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1447,8 +1452,8 @@ bool AST::is_equal( AST* other ) check_member_ast( Value ); check_member_ast( Attributes ); check_member_ast( Specs ); - check_member_ast( Prev ); - check_member_ast( Next ); + // check_member_ast( Prev ); + // check_member_ast( Next ); return true; } @@ -1461,7 +1466,6 @@ bool AST::is_equal( AST* other ) case Struct_Body: case Union_Body: { - check_member_val( NumEntries ); check_member_ast( Front ); check_member_ast( Back ); @@ -1501,6 +1505,8 @@ bool AST::is_equal( AST* other ) curr_other = curr_other->Next; } + check_member_val( NumEntries ); + return true; } From abf51e4aa9eebb2e71384d595dea473fa6a232ab Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 25 Aug 2023 18:57:53 -0400 Subject: [PATCH 10/26] Adjustment to AST::is_equal based on issues found with last CS. Defined check_member_content, will spit out the strings if they aren't equivalent so the user can verify for themselves if its correct. --- project/components/ast.cpp | 33 +++++++++++++++++------- project/components/interface.parsing.cpp | 4 +-- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index 3abecb2..d8e6ff9 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -1005,6 +1005,25 @@ bool AST::is_equal( AST* other ) return false; \ } + #define check_member_content( content ) \ + if ( content != other->content ) \ + { \ + log_fmt("AST::is_equal: Member content - "#content " failed\n" \ + "AST : %S\n" \ + "Other: %S\n" \ + , debug_str() \ + , other->debug_str() \ + ); \ + \ + log_fmt("Content cannot be trusted to be unique with this check " \ + "so it must be verified by eye for now\n" \ + "AST Content:\n%S\n" \ + "Other Content:\n%S\n" \ + , content \ + , other->content \ + ); \ + } + #define check_member_ast( ast ) \ if ( ast ) \ { \ @@ -1039,12 +1058,6 @@ bool AST::is_equal( AST* other ) } \ } - // Need to check to make sure the prev->is_equal wont lead to a recursion. - // #define check_member_prev() \ - // if ( Prev ) \ - // { \ - - case NewLine: case Access_Public: case Access_Protected: @@ -1058,7 +1071,7 @@ bool AST::is_equal( AST* other ) case PlatformAttributes: case Untyped: { - check_member_str( Content ); + check_member_content( Content ); // check_member_ast( Prev ); // check_member_ast( Next ); } @@ -1340,7 +1353,7 @@ bool AST::is_equal( AST* other ) case Preprocess_Define: { check_member_str( Name ); - check_member_str( Content ); + check_member_content( Content ); // check_member_ast( Prev ); // check_member_ast( Next ); @@ -1352,7 +1365,7 @@ bool AST::is_equal( AST* other ) case Preprocess_IfNotDef: case Preprocess_ElIf: { - check_member_str( Content ); + check_member_content( Content ); // check_member_ast( Prev ); // check_member_ast( Next ); @@ -1362,7 +1375,7 @@ bool AST::is_equal( AST* other ) case Preprocess_Include: case Preprocess_Pragma: { - check_member_str( Content ); + check_member_content( Content ); // check_member_ast( Prev ); // check_member_ast( Next ); diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 720ade6..c568273 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -4502,7 +4502,7 @@ CodeType parse_type( bool* is_function ) brute_sig.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)brute_sig.Text; is_first_capture = false; } - + bool is_param_pack = false; if ( check(TokType::Varadic_Argument) ) { @@ -4534,7 +4534,7 @@ CodeType parse_type( bool* is_function ) if ( attributes ) result->Attributes = attributes; - + if ( is_param_pack ) result->IsParamPack = true; From 7249a7317deebd194071b350af108883af89bf09 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 26 Aug 2023 11:55:05 -0400 Subject: [PATCH 11/26] Added space stripping during for content of various ASTs * Typedef/Typename * Function Names * Pragmas * Attributes --- project/components/ast.cpp | 14 +++-- project/components/ast.hpp | 8 +++ project/components/ast_types.hpp | 28 ++++++++++ project/components/interface.cpp | 1 + project/components/interface.parsing.cpp | 71 ++++++++++++++++++++---- project/dependencies/strings.hpp | 21 +++++++ 6 files changed, 126 insertions(+), 17 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index d8e6ff9..fc0b8dc 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -11,9 +11,9 @@ char const* AST::debug_str() String result = String::make_reserve( GlobalAllocator, kilobytes(1) ); result.append_fmt( - "\nType : %s" - "\nParent : %s %s" - "\nName : %s" + "\n\tType : %s" + "\n\tParent : %s %s" + "\n\tName : %s" , type_str() , Parent->type_str() , Parent->Name, Name ? Name : "" @@ -25,8 +25,8 @@ char const* AST::debug_str() String result = String::make_reserve( GlobalAllocator, kilobytes(1) ); result.append_fmt( - "\nType : %s" - "\nName : %s" + "\n\tType : %s" + "\n\tName : %s" , type_str() , Name ? Name : "" ); @@ -1066,7 +1066,11 @@ bool AST::is_equal( AST* other ) case Preprocess_EndIf: return true; + + // Comments are not validated. case Comment: + // return true; + case Execution: case PlatformAttributes: case Untyped: diff --git a/project/components/ast.hpp b/project/components/ast.hpp index 96040ca..308da4d 100644 --- a/project/components/ast.hpp +++ b/project/components/ast.hpp @@ -4,6 +4,11 @@ #include "gen/eoperator.hpp" #include "gen/especifier.hpp" +namespace Parser +{ + struct Token; +} + struct AST; struct AST_Body; struct AST_Attributes; @@ -220,6 +225,7 @@ struct AST - sizeof(CodeT) - sizeof(ModuleFlag) - sizeof(u32) + - sizeof(s32) ) / sizeof(SpecifierT) - 1; // -1 for 4 extra bytes @@ -270,6 +276,7 @@ struct AST AccessSpec ParentAccess; s32 NumEntries; }; + s32 Token; // Handle to the token, stored in the CodeFile (Otherwise unretrivable) }; struct AST_POD @@ -321,6 +328,7 @@ struct AST_POD AccessSpec ParentAccess; s32 NumEntries; }; + s32 Token; // Handle to the token, stored in the CodeFile (Otherwise unretrivable) }; // Its intended for the AST to have equivalent size to its POD. diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 00ec5ad..8ff257d 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -18,6 +18,7 @@ struct AST_Body CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) ]; s32 NumEntries; + s32 Token; }; static_assert( sizeof(AST_Body) == sizeof(AST), "ERROR: AST_Filtered is not the same size as AST"); @@ -33,6 +34,7 @@ struct AST_Attributes StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Attributes) == sizeof(AST), "ERROR: AST_Attributes is not the same size as AST"); @@ -48,6 +50,7 @@ struct AST_Comment StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Comment) == sizeof(AST), "ERROR: AST_Comment is not the same size as AST"); @@ -72,6 +75,7 @@ struct AST_Class CodeT Type; ModuleFlag ModuleFlags; AccessSpec ParentAccess; + s32 Token; }; static_assert( sizeof(AST_Class) == sizeof(AST), "ERROR: AST_Class is not the same size as AST"); @@ -95,6 +99,7 @@ struct AST_Constructor char _PAD_NAME_[ sizeof(StringCached) ]; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Constructor) == sizeof(AST), "ERROR: AST_Constructor is not the same size as AST"); @@ -110,6 +115,7 @@ struct AST_Define StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Define) == sizeof(AST), "ERROR: AST_Define is not the same size as AST"); @@ -132,6 +138,7 @@ struct AST_Destructor char _PAD_NAME_[ sizeof(StringCached) ]; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Destructor) == sizeof(AST), "ERROR: AST_Destructor is not the same size as AST"); @@ -156,6 +163,7 @@ struct AST_Enum CodeT Type; ModuleFlag ModuleFlags; char _PAD_UNUSED_[ sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Enum) == sizeof(AST), "ERROR: AST_Enum is not the same size as AST"); @@ -171,6 +179,7 @@ struct AST_Exec StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Exec) == sizeof(AST), "ERROR: AST_Exec is not the same size as AST"); @@ -190,6 +199,7 @@ struct AST_Extern StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Extern) == sizeof(AST), "ERROR: AST_Extern is not the same size as AST"); @@ -205,6 +215,7 @@ struct AST_Include StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Include) == sizeof(AST), "ERROR: AST_Include is not the same size as AST"); @@ -225,6 +236,7 @@ struct AST_Friend StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Friend) == sizeof(AST), "ERROR: AST_Friend is not the same size as AST"); @@ -249,6 +261,7 @@ struct AST_Fn CodeT Type; ModuleFlag ModuleFlags; char _PAD_UNUSED_[ sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Fn) == sizeof(AST), "ERROR: AST_Fn is not the same size as AST"); @@ -262,6 +275,7 @@ struct AST_Module CodeT Type; ModuleFlag ModuleFlags; char _PAD_UNUSED_[ sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Module) == sizeof(AST), "ERROR: AST_Module is not the same size as AST"); @@ -281,6 +295,7 @@ struct AST_NS CodeT Type; ModuleFlag ModuleFlags; char _PAD_UNUSED_[ sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_NS) == sizeof(AST), "ERROR: AST_NS is not the same size as AST"); @@ -305,6 +320,7 @@ struct AST_Operator CodeT Type; ModuleFlag ModuleFlags; OperatorT Op; + s32 Token; }; static_assert( sizeof(AST_Operator) == sizeof(AST), "ERROR: AST_Operator is not the same size as AST"); @@ -328,6 +344,7 @@ struct AST_OpCast StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_OpCast) == sizeof(AST), "ERROR: AST_OpCast is not the same size as AST"); @@ -350,6 +367,7 @@ struct AST_Param CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) ]; s32 NumEntries; + s32 Token; }; static_assert( sizeof(AST_Param) == sizeof(AST), "ERROR: AST_Param is not the same size as AST"); @@ -365,6 +383,7 @@ struct AST_Pragma StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Pragma) == sizeof(AST), "ERROR: AST_Pragma is not the same size as AST"); @@ -380,6 +399,7 @@ struct AST_PreprocessCond StringCached Name; CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_PreprocessCond) == sizeof(AST), "ERROR: AST_PreprocessCond is not the same size as AST"); @@ -393,6 +413,7 @@ struct AST_Specifiers CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) ]; s32 NumEntries; + s32 Token; }; static_assert( sizeof(AST_Specifiers) == sizeof(AST), "ERROR: AST_Specifier is not the same size as AST"); @@ -417,6 +438,7 @@ struct AST_Struct CodeT Type; ModuleFlag ModuleFlags; AccessSpec ParentAccess; + s32 Token; }; static_assert( sizeof(AST_Struct) == sizeof(AST), "ERROR: AST_Struct is not the same size as AST"); @@ -438,6 +460,7 @@ struct AST_Template CodeT Type; ModuleFlag ModuleFlags; char _PAD_UNUSED_[ sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Template) == sizeof(AST), "ERROR: AST_Template is not the same size as AST"); @@ -461,6 +484,7 @@ struct AST_Type CodeT Type; char _PAD_UNUSED_[ sizeof(ModuleFlag) ]; b32 IsParamPack; + s32 Token; }; static_assert( sizeof(AST_Type) == sizeof(AST), "ERROR: AST_Type is not the same size as AST"); @@ -483,6 +507,7 @@ struct AST_Typedef CodeT Type; ModuleFlag ModuleFlags; b32 IsFunction; + s32 Token; }; static_assert( sizeof(AST_Typedef) == sizeof(AST), "ERROR: AST_Typedef is not the same size as AST"); @@ -505,6 +530,7 @@ struct AST_Union CodeT Type; ModuleFlag ModuleFlags; char _PAD_UNUSED_[ sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Union) == sizeof(AST), "ERROR: AST_Union is not the same size as AST"); @@ -528,6 +554,7 @@ struct AST_Using CodeT Type; ModuleFlag ModuleFlags; char _PAD_UNUSED_[ sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Using) == sizeof(AST), "ERROR: AST_Using is not the same size as AST"); @@ -552,6 +579,7 @@ struct AST_Var CodeT Type; ModuleFlag ModuleFlags; char _PAD_UNUSED_[ sizeof(u32) ]; + s32 Token; }; static_assert( sizeof(AST_Var) == sizeof(AST), "ERROR: AST_Var is not the same size as AST"); diff --git a/project/components/interface.cpp b/project/components/interface.cpp index c29c805..707d47b 100644 --- a/project/components/interface.cpp +++ b/project/components/interface.cpp @@ -424,6 +424,7 @@ Code make_code() result->Type = ECode::Invalid; result->ModuleFlags = ModuleFlag::Invalid; result->NumEntries = 0; + result->Token = -1; return result; } diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index c568273..7df12ca 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -521,6 +521,8 @@ namespace Parser s32 within_string = false; s32 within_char = false; + + // SkipWhitespace(); while ( left ) { if ( current == '"' && ! within_char ) @@ -563,13 +565,11 @@ namespace Parser if ( current == '\r' ) { move_forward(); - // content.Length++; } if ( current == '\n' ) { move_forward(); - // content.Length++; break; } @@ -1274,6 +1274,18 @@ CodeDefine parse_define() return CodeInvalid; } + // s32 left = currtok.Length; + // char const* scanner = currtok.Text; + // while ( left ) + // { + // if ( scanner[0] == ' ' ) + // { + // scanner++; + // left--; + // continue; + // } + // } + define->Content = get_cached_string( currtok ); eat( TokType::Preprocess_Content ); @@ -1359,7 +1371,12 @@ CodePragma parse_pragma() } Context.Scope->Name = currtok; - pragma->Content = get_cached_string( currtok ); + + String + content_stripped = String::make( GlobalAllocator, currtok ); + content_stripped.strip_space(); + + pragma->Content = get_cached_string( content_stripped ); eat( TokType::Preprocess_Content ); Context.pop(); @@ -1398,7 +1415,16 @@ Code parse_static_assert() content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text; + + String + content_stripped = String::make( GlobalAllocator, content ); + content_stripped.strip_space(); + char const* result = str_fmt_buf( "%.*s\n", content.Length, content.Text ); + if ( content_stripped ) + { + result = str_fmt_buf( "%S\n", content_stripped ); + } assert->Content = get_cached_string( to_str( result ) ); assert->Name = assert->Content; @@ -1538,7 +1564,18 @@ CodeAttributes parse_attributes() { StrC attribute_txt = { len, start.Text }; Context.pop(); - return def_attributes( attribute_txt ); + + String + name_stripped = String::make( GlobalAllocator, attribute_txt ); + name_stripped.strip_space(); + + Code + result = make_code(); + result->Type = ECode::PlatformAttributes; + result->Name = get_cached_string( name_stripped ); + result->Content = result->Name; + + return (CodeAttributes) result; } Context.pop(); @@ -1867,9 +1904,15 @@ CodeFn parse_function_after_name( using namespace ECode; + + + String + name_stripped = String::make( GlobalAllocator, name ); + name_stripped.strip_space(); + CodeFn result = (CodeFn) make_code(); - result->Name = get_cached_string( name ); + result->Name = get_cached_string( name_stripped ); result->ModuleFlags = mflags; if ( body ) @@ -3445,7 +3488,7 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers ) return CodeInvalid; } } - + CodeComment inline_cmt = NoCode; if ( check( TokType::BraceCurly_Open ) ) @@ -3454,7 +3497,7 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers ) { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); } @@ -3471,7 +3514,7 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers ) } else result->Type = ECode::Destructor_Fwd; - + if ( inline_cmt ) result->InlineCmt = inline_cmt; @@ -4151,7 +4194,7 @@ CodeOpCast parse_operator_cast( CodeSpecifiers specifiers ) { Token stmt_end = currtok; eat( TokType::Statement_End ); - + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); } @@ -4530,7 +4573,11 @@ CodeType parse_type( bool* is_function ) } } - result->Name = get_cached_string( name ); + String + name_stripped = String::make( GlobalAllocator, name ); + name_stripped.strip_space(); + + result->Name = get_cached_string( name_stripped ); if ( attributes ) result->Attributes = attributes; @@ -4941,7 +4988,7 @@ CodeUsing parse_using() Token stmt_end = currtok; eat( TokType::Statement_End ); - + CodeComment inline_cmt = NoCode; if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) { @@ -4971,7 +5018,7 @@ CodeUsing parse_using() if ( attributes ) result->Attributes = attributes; - + if ( inline_cmt ) result->InlineCmt = inline_cmt; } diff --git a/project/dependencies/strings.hpp b/project/dependencies/strings.hpp index 2e82a2c..5fc28f0 100644 --- a/project/dependencies/strings.hpp +++ b/project/dependencies/strings.hpp @@ -213,6 +213,27 @@ struct String #undef current } + void strip_space() + { + char* write_pos = Data; + char* read_pos = Data; + + while ( * read_pos) + { + if ( ! char_is_space( *read_pos )) + { + *write_pos = *read_pos; + write_pos++; + } + read_pos++; + } + + write_pos[0] = '\0'; // Null-terminate the modified string + + // Update the length if needed + get_header().Length = write_pos - Data; + } + void trim( char const* cut_set ) { sw len = 0; From 0197afd5434bf76704a9c526a29526e3ff71d424 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Mon, 28 Aug 2023 23:46:50 -0400 Subject: [PATCH 12/26] Changed how editor intellisense directives are handled for compoenents and dependencies Didn't like the way I was processing them in scan_file. --- .vscode/c_cpp_properties.json | 6 +- project/auxillary/scanner.hpp | 108 +++++++++++++++-------- project/bootstrap.cpp | 28 +++--- project/components/ast.cpp | 2 + project/components/ast.hpp | 2 + project/components/ast_types.hpp | 6 +- project/components/gen/ast_inlines.hpp | 3 + project/components/gen/ecode.hpp | 2 + project/components/gen/eoperator.hpp | 2 + project/components/gen/especifier.hpp | 2 + project/components/header_end.hpp | 2 + project/components/inlines.hpp | 2 + project/components/interface.cpp | 2 + project/components/interface.hpp | 2 + project/components/interface.parsing.cpp | 2 + project/components/interface.untyped.cpp | 2 + project/components/interface.upfront.cpp | 2 + project/components/static_data.cpp | 2 + project/components/types.hpp | 2 + project/dependencies/basic_types.hpp | 6 +- project/dependencies/containers.hpp | 6 +- project/dependencies/debug.cpp | 4 +- project/dependencies/debug.hpp | 6 +- project/dependencies/filesystem.cpp | 6 +- project/dependencies/filesystem.hpp | 6 +- project/dependencies/hashing.cpp | 6 +- project/dependencies/hashing.hpp | 2 + project/dependencies/macros.hpp | 6 +- project/dependencies/memory.cpp | 6 +- project/dependencies/memory.hpp | 6 +- project/dependencies/parsing.cpp | 4 +- project/dependencies/parsing.hpp | 4 +- project/dependencies/printing.cpp | 6 +- project/dependencies/printing.hpp | 6 +- project/dependencies/string_ops.cpp | 6 +- project/dependencies/string_ops.hpp | 6 +- project/dependencies/strings.cpp | 6 +- project/dependencies/strings.hpp | 6 +- project/dependencies/timing.cpp | 6 +- project/dependencies/timing.hpp | 6 +- 40 files changed, 202 insertions(+), 93 deletions(-) diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index fd5dbf0..f2775d2 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -10,8 +10,9 @@ "UNICODE", "_UNICODE", "GEN_TIME", - "GEN_IMPLEMENTATION" + "GEN_IMPLEMENTATION", // "GEN_DONT_USE_NAMESPACE" + "GEN_INTELLISENSE_DIRECTIVES" ], "windowsSdkVersion": "10.0.19041.0", "compilerPath": "C:/Program Files/Microsoft Visual Studio/2022/Professional/VC/Tools/MSVC/14.29.30133/bin/HostX64/x64/cl.exe", @@ -28,8 +29,9 @@ "UNICODE", "_UNICODE", "GEN_TIME", - "GEN_IMPLEMENTATION" + "GEN_IMPLEMENTATION", // "GEN_DONT_USE_NAMESPACE" + "GEN_INTELLISENSE_DIRECTIVES" ], "windowsSdkVersion": "10.0.19041.0", "compilerPath": "C:/Users/Ed/scoop/apps/llvm/current/bin/clang++.exe", diff --git a/project/auxillary/scanner.hpp b/project/auxillary/scanner.hpp index ffb75e2..6bdffa8 100644 --- a/project/auxillary/scanner.hpp +++ b/project/auxillary/scanner.hpp @@ -1,10 +1,11 @@ + #pragma once #include "gen.hpp" // This is a simple file reader that reads the entire file into memory. // It has an extra option to skip the first few lines for undesired includes. // This is done so that includes can be kept in dependency and component files so that intellisense works. -Code scan_file( char const* path, bool skip_initial_directives = true ) +Code scan_file( char const* path ) { FileInfo file; @@ -24,63 +25,92 @@ Code scan_file( char const* path, bool skip_initial_directives = true ) file_read( & file, str, fsize ); str.get_header().Length = fsize; - if ( skip_initial_directives ) + // Skip GEN_INTELLISENSE_DIRECTIVES preprocessor blocks + // Its designed so that the directive should be the first thing in the file. + // Anything that comes before it will also be omitted. { #define current (*scanner) - StrC toks[] { - txt( "pragma once" ), - txt( "include" ) - }; + #define matched 0 + #define move_fwd() do { ++ scanner; -- left; } while (0) + const StrC directive_start = txt( "ifdef" ); + const StrC directive_end = txt( "endif" ); + const StrC def_intellisense = txt("GEN_INTELLISENSE_DIRECTIVES" ); - char* scanner = str; - while ( current != '\r' && current != '\n' ) + bool found_directive = false; + char const* scanner = str.Data; + s32 left = fsize; + while ( left ) { - for ( StrC tok : toks ) + // Processing directive. + if ( current == '#' ) { - if ( current == '#' ) - { - ++ scanner; - } + move_fwd(); + while ( left && char_is_space( current ) ) + move_fwd(); - if ( strncmp( scanner, tok.Ptr, tok.Len ) == 0 ) + if ( ! found_directive ) { - scanner += tok.Len; - while ( scanner < ( str.Data + str.length() ) && current != '\r' && current != '\n' ) + if ( left && str_compare( scanner, directive_start.Ptr, directive_start.Len ) == matched ) { - ++ scanner; + scanner += directive_start.Len; + left -= directive_start.Len; + + while ( left && char_is_space( current ) ) + move_fwd(); + + if ( left && str_compare( scanner, def_intellisense.Ptr, def_intellisense.Len ) == matched ) + { + scanner += def_intellisense.Len; + left -= def_intellisense.Len; + + found_directive = true; + } } - // Skip the line - sptr skip_size = sptr( scanner - str.Data ); - if ( (scanner + 2) >= ( str.Data + str.length() ) ) + // Skip to end of line + while ( left && current != '\r' && current != '\n' ) + move_fwd(); + move_fwd(); + + if ( left && current == '\n' ) + move_fwd(); + + continue; + } + + if ( left && str_compare( scanner, directive_end.Ptr, directive_end.Len ) == matched ) + { + scanner += directive_end.Len; + left -= directive_end.Len; + + // Skip to end of line + while ( left && current != '\r' && current != '\n' ) + move_fwd(); + move_fwd(); + + if ( left && current == '\n' ) + move_fwd(); + + sptr skip_size = fsize - left; + if ( (scanner + 2) >= ( str.Data + fsize ) ) { - sptr new_length = sptr( str.get_header().Length ) - skip_size; - mem_move( str, scanner, new_length ); - str.get_header().Length = new_length; + mem_move( str, scanner, left ); + str.get_header().Length = left; break; } - if ( current == '\r' ) - { - skip_size += 2; - scanner += 2; - } - else - { - skip_size += 1; - scanner += 1; - } + mem_move( str, scanner, left ); + str.get_header().Length = left; - sptr new_length = sptr( str.get_header().Length ) - skip_size; - mem_move( str, scanner, new_length ); - str.get_header().Length = new_length; - - scanner = str; + break; } + } - ++ scanner; + move_fwd(); } + #undef move_fwd + #undef matched #undef current } diff --git a/project/bootstrap.cpp b/project/bootstrap.cpp index 2efad37..6e661ed 100644 --- a/project/bootstrap.cpp +++ b/project/bootstrap.cpp @@ -20,18 +20,16 @@ constexpr char const* generation_notice = "// This file was generated automatially by gencpp's bootstrap.cpp " "(See: https://github.com/Ed94/gencpp)\n\n"; -constexpr bool DontSkipInitialDirectives = false; - int gen_main() { gen::init(); - Code push_ignores = scan_file( "helpers/push_ignores.inline.hpp", DontSkipInitialDirectives ); - Code pop_ignores = scan_file( "helpers/pop_ignores.inline.hpp", DontSkipInitialDirectives ); + Code push_ignores = scan_file( "helpers/push_ignores.inline.hpp" ); + Code pop_ignores = scan_file( "helpers/pop_ignores.inline.hpp" ); // gen_dep.hpp { - Code header_start = scan_file( "dependencies/header_start.hpp", DontSkipInitialDirectives ); + Code header_start = scan_file( "dependencies/header_start.hpp" ); Code macros = scan_file( "dependencies/macros.hpp" ); Code basic_types = scan_file( "dependencies/basic_types.hpp" ); Code debug = scan_file( "dependencies/debug.hpp" ); @@ -147,31 +145,35 @@ int gen_main() header.print( pop_ignores ); header.write(); + CodeBody gen_component_header = def_global_body( args( + def_preprocess_cond( PreprocessCond_IfDef, txt("GEN_INTELLISENSE_DIRECTIVES") ), + pragma_once, + preprocess_endif, + fmt_newline, + untyped_str( to_str(generation_notice) ) + )); + Builder header_ecode = Builder::open( "components/gen/ecode.hpp" ); - header_ecode.print( pragma_once ); - header_ecode.print_fmt( generation_notice ); + header_ecode.print( gen_component_header ); header_ecode.print( ecode ); header_ecode.write(); Builder header_eoperator = Builder::open( "components/gen/eoperator.hpp" ); - header_eoperator.print( pragma_once ); - header_eoperator.print_fmt( generation_notice ); + header_eoperator.print( gen_component_header ); header_eoperator.print( eoperator ); header_eoperator.write(); Builder header_especifier = Builder::open( "components/gen/especifier.hpp" ); - header_especifier.print( pragma_once ); - header_especifier.print_fmt( generation_notice ); + header_especifier.print( gen_component_header ); header_especifier.print( especifier ); header_especifier.write(); Builder header_ast_inlines = Builder::open( "components/gen/ast_inlines.hpp" ); - header_ast_inlines.print( pragma_once ); - header_ast_inlines.print_fmt( generation_notice ); + header_ast_inlines.print( gen_component_header ); header_ast_inlines.print( ast_inlines ); header_ast_inlines.write(); } diff --git a/project/components/ast.cpp b/project/components/ast.cpp index fc0b8dc..804d3f3 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -1,5 +1,7 @@ +#if GEN_INTELLISENSE_DIRECTIVES #pragma once #include "static_data.cpp" +#endif Code Code::Global; Code Code::Invalid; diff --git a/project/components/ast.hpp b/project/components/ast.hpp index 308da4d..c920c76 100644 --- a/project/components/ast.hpp +++ b/project/components/ast.hpp @@ -1,8 +1,10 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "types.hpp" #include "gen/ecode.hpp" #include "gen/eoperator.hpp" #include "gen/especifier.hpp" +#endif namespace Parser { diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 8ff257d..236aab9 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "ast.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "ast.hpp" +#endif #pragma region AST Types /* diff --git a/project/components/gen/ast_inlines.hpp b/project/components/gen/ast_inlines.hpp index 7cb3fad..de72dbf 100644 --- a/project/components/gen/ast_inlines.hpp +++ b/project/components/gen/ast_inlines.hpp @@ -1,4 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once +#endif + // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) #pragma region generated code inline implementation diff --git a/project/components/gen/ecode.hpp b/project/components/gen/ecode.hpp index a2e61dc..ee46c90 100644 --- a/project/components/gen/ecode.hpp +++ b/project/components/gen/ecode.hpp @@ -1,4 +1,6 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once +#endif // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) diff --git a/project/components/gen/eoperator.hpp b/project/components/gen/eoperator.hpp index cea26b1..3feb117 100644 --- a/project/components/gen/eoperator.hpp +++ b/project/components/gen/eoperator.hpp @@ -1,4 +1,6 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once +#endif // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) diff --git a/project/components/gen/especifier.hpp b/project/components/gen/especifier.hpp index 4dedcf7..387d614 100644 --- a/project/components/gen/especifier.hpp +++ b/project/components/gen/especifier.hpp @@ -1,4 +1,6 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once +#endif // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) diff --git a/project/components/header_end.hpp b/project/components/header_end.hpp index b733045..efc2424 100644 --- a/project/components/header_end.hpp +++ b/project/components/header_end.hpp @@ -1,6 +1,8 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "inlines.hpp" #include "gen/ast_inlines.hpp" +#endif #pragma region Constants diff --git a/project/components/inlines.hpp b/project/components/inlines.hpp index 20aac3d..f1d53fd 100644 --- a/project/components/inlines.hpp +++ b/project/components/inlines.hpp @@ -1,5 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "interface.hpp" +#endif void AST::append( AST* other ) { diff --git a/project/components/interface.cpp b/project/components/interface.cpp index 707d47b..8050ca1 100644 --- a/project/components/interface.cpp +++ b/project/components/interface.cpp @@ -1,5 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "ast.cpp" +#endif internal void init_parser(); internal void deinit_parser(); diff --git a/project/components/interface.hpp b/project/components/interface.hpp index 91ecab9..d39eae5 100644 --- a/project/components/interface.hpp +++ b/project/components/interface.hpp @@ -1,5 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "ast_types.hpp" +#endif #pragma region Gen Interface diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 7df12ca..87eba4b 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -1,6 +1,8 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "gen/etoktype.cpp" #include "interface.upfront.cpp" +#endif namespace Parser { diff --git a/project/components/interface.untyped.cpp b/project/components/interface.untyped.cpp index ceb65b2..01ce56b 100644 --- a/project/components/interface.untyped.cpp +++ b/project/components/interface.untyped.cpp @@ -1,5 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "interface.parsing.cpp" +#endif sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va ) { diff --git a/project/components/interface.upfront.cpp b/project/components/interface.upfront.cpp index af42b74..0fca812 100644 --- a/project/components/interface.upfront.cpp +++ b/project/components/interface.upfront.cpp @@ -1,5 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "interface.cpp" +#endif #pragma region Upfront diff --git a/project/components/static_data.cpp b/project/components/static_data.cpp index 9fd2495..c5e9eab 100644 --- a/project/components/static_data.cpp +++ b/project/components/static_data.cpp @@ -1,5 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "gen.hpp" +#endif #pragma region StaticData diff --git a/project/components/types.hpp b/project/components/types.hpp index 7f16a2f..bd8e760 100644 --- a/project/components/types.hpp +++ b/project/components/types.hpp @@ -1,5 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "header_start.hpp" +#endif using LogFailType = sw(*)(char const*, ...); diff --git a/project/dependencies/basic_types.hpp b/project/dependencies/basic_types.hpp index 6dc3693..415c607 100644 --- a/project/dependencies/basic_types.hpp +++ b/project/dependencies/basic_types.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "macros.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "macros.hpp" +#endif #pragma region Basic Types diff --git a/project/dependencies/containers.hpp b/project/dependencies/containers.hpp index 1c77aea..391df5e 100644 --- a/project/dependencies/containers.hpp +++ b/project/dependencies/containers.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "printing.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "printing.hpp" +#endif #pragma region Containers diff --git a/project/dependencies/debug.cpp b/project/dependencies/debug.cpp index 1b6705a..3b5adef 100644 --- a/project/dependencies/debug.cpp +++ b/project/dependencies/debug.cpp @@ -1,4 +1,6 @@ -#pragma once +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +#endif #pragma region Debug diff --git a/project/dependencies/debug.hpp b/project/dependencies/debug.hpp index 8e8fba1..43d50cb 100644 --- a/project/dependencies/debug.hpp +++ b/project/dependencies/debug.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "basic_types.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVESj +# pragma once +# include "basic_types.hpp" +#endif #pragma region Debug diff --git a/project/dependencies/filesystem.cpp b/project/dependencies/filesystem.cpp index 1861e9a..b78089a 100644 --- a/project/dependencies/filesystem.cpp +++ b/project/dependencies/filesystem.cpp @@ -1,5 +1,7 @@ -#pragma once -#include "strings.cpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "strings.cpp" +#endif #pragma region File Handling diff --git a/project/dependencies/filesystem.hpp b/project/dependencies/filesystem.hpp index 113f839..cec987e 100644 --- a/project/dependencies/filesystem.hpp +++ b/project/dependencies/filesystem.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "strings.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "strings.hpp" +#endif #pragma region File Handling diff --git a/project/dependencies/hashing.cpp b/project/dependencies/hashing.cpp index 2294810..caa46e9 100644 --- a/project/dependencies/hashing.cpp +++ b/project/dependencies/hashing.cpp @@ -1,5 +1,7 @@ -#pragma once -#include "memory.cpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "memory.cpp" +#endif #pragma region Hashing diff --git a/project/dependencies/hashing.hpp b/project/dependencies/hashing.hpp index 4b70f4a..a9d19e1 100644 --- a/project/dependencies/hashing.hpp +++ b/project/dependencies/hashing.hpp @@ -1,5 +1,7 @@ +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "containers.hpp" +#endif #pragma region Hashing diff --git a/project/dependencies/macros.hpp b/project/dependencies/macros.hpp index b28ec65..8125840 100644 --- a/project/dependencies/macros.hpp +++ b/project/dependencies/macros.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "header_start.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "header_start.hpp" +#endif #pragma region Macros diff --git a/project/dependencies/memory.cpp b/project/dependencies/memory.cpp index e9bdc53..60088e2 100644 --- a/project/dependencies/memory.cpp +++ b/project/dependencies/memory.cpp @@ -1,5 +1,7 @@ -#pragma once -#include "printing.cpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "printing.cpp" +#endif #pragma region Memory diff --git a/project/dependencies/memory.hpp b/project/dependencies/memory.hpp index f87b68a..d436885 100644 --- a/project/dependencies/memory.hpp +++ b/project/dependencies/memory.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "debug.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "debug.hpp" +#endif #pragma region Memory diff --git a/project/dependencies/parsing.cpp b/project/dependencies/parsing.cpp index d9d3d8e..f975f17 100644 --- a/project/dependencies/parsing.cpp +++ b/project/dependencies/parsing.cpp @@ -1,4 +1,6 @@ -#pragma once +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +#endif #pragma region ADT diff --git a/project/dependencies/parsing.hpp b/project/dependencies/parsing.hpp index d89bdac..05e4b73 100644 --- a/project/dependencies/parsing.hpp +++ b/project/dependencies/parsing.hpp @@ -1,4 +1,6 @@ -#pragma once +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +#endif #pragma region ADT diff --git a/project/dependencies/printing.cpp b/project/dependencies/printing.cpp index d8342ff..f3fef3d 100644 --- a/project/dependencies/printing.cpp +++ b/project/dependencies/printing.cpp @@ -1,5 +1,7 @@ -#pragma once -#include "string_ops.cpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "string_ops.cpp" +#endif #pragma region Printing diff --git a/project/dependencies/printing.hpp b/project/dependencies/printing.hpp index 8c7b895..a4e3d57 100644 --- a/project/dependencies/printing.hpp +++ b/project/dependencies/printing.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "string_ops.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "string_ops.hpp" +#endif #pragma region Printing diff --git a/project/dependencies/string_ops.cpp b/project/dependencies/string_ops.cpp index 4fb578d..28729da 100644 --- a/project/dependencies/string_ops.cpp +++ b/project/dependencies/string_ops.cpp @@ -1,5 +1,7 @@ -#pragma once -#include "debug.cpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "debug.cpp" +#endif #pragma region String Ops diff --git a/project/dependencies/string_ops.hpp b/project/dependencies/string_ops.hpp index 7bb61d3..8963fcd 100644 --- a/project/dependencies/string_ops.hpp +++ b/project/dependencies/string_ops.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "memory.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "memory.hpp" +#endif #pragma region String Ops diff --git a/project/dependencies/strings.cpp b/project/dependencies/strings.cpp index 7b323f0..86b6247 100644 --- a/project/dependencies/strings.cpp +++ b/project/dependencies/strings.cpp @@ -1,5 +1,7 @@ -#pragma once -#include "hashing.cpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "hashing.cpp" +#endif #pragma region String diff --git a/project/dependencies/strings.hpp b/project/dependencies/strings.hpp index 5fc28f0..b834575 100644 --- a/project/dependencies/strings.hpp +++ b/project/dependencies/strings.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "hashing.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "hashing.hpp" +#endif #pragma region Strings diff --git a/project/dependencies/timing.cpp b/project/dependencies/timing.cpp index 1062b0b..763e60f 100644 --- a/project/dependencies/timing.cpp +++ b/project/dependencies/timing.cpp @@ -1,5 +1,7 @@ -#pragma once -#include "filesystem.cpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "filesystem.cpp" +#endif #pragma region Timing diff --git a/project/dependencies/timing.hpp b/project/dependencies/timing.hpp index 30390c7..e4b9575 100644 --- a/project/dependencies/timing.hpp +++ b/project/dependencies/timing.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "filesystem.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "filesystem.hpp" +#endif #pragma region Timing From a4d9a63d710eff5251bd2a455ccdba30183ed97d Mon Sep 17 00:00:00 2001 From: Ed_ Date: Mon, 28 Aug 2023 23:52:44 -0400 Subject: [PATCH 13/26] Updated single-header based on last cs and also docs --- singleheader/Readme.md | 6 ++---- singleheader/singleheader.cpp | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/singleheader/Readme.md b/singleheader/Readme.md index 7cfb66c..aa7f724 100644 --- a/singleheader/Readme.md +++ b/singleheader/Readme.md @@ -1,6 +1,4 @@ # Singleheader -`gen.singleheader.cpp` with its own `meson.build` generates the library as a single header `gen.hpp`. -Following the same convention seen in the gb, stb, and zpl libraries. - -( Currently WIP ) +Creates a single header file version of the library using `gen.singleheader.cpp`. +Follows the same convention seen in the gb, stb, and zpl libraries. diff --git a/singleheader/singleheader.cpp b/singleheader/singleheader.cpp index f79c017..ee7ed73 100644 --- a/singleheader/singleheader.cpp +++ b/singleheader/singleheader.cpp @@ -48,8 +48,6 @@ global bool generate_builder = true; global bool generate_editor = true; global bool generate_scanner = true; -constexpr bool DontSkipInitialDirectives = false; - int gen_main() { #define project_dir "../project/" @@ -57,7 +55,7 @@ int gen_main() Code push_ignores = scan_file( project_dir "helpers/push_ignores.inline.hpp" ); Code pop_ignores = scan_file( project_dir "helpers/pop_ignores.inline.hpp" ); - Code single_header_start = scan_file( "components/header_start.hpp", DontSkipInitialDirectives ); + Code single_header_start = scan_file( "components/header_start.hpp" ); Builder header = Builder::open( "gen/gen.hpp" ); @@ -71,7 +69,7 @@ int gen_main() if ( generate_gen_dep ) { - Code header_start = scan_file( project_dir "dependencies/header_start.hpp", DontSkipInitialDirectives ); + Code header_start = scan_file( project_dir "dependencies/header_start.hpp" ); Code macros = scan_file( project_dir "dependencies/macros.hpp" ); Code basic_types = scan_file( project_dir "dependencies/basic_types.hpp" ); Code debug = scan_file( project_dir "dependencies/debug.hpp" ); From c4c308c8bafcec4b6c56ce74f143d6a60c14eace Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 29 Aug 2023 00:03:08 -0400 Subject: [PATCH 14/26] Fixes for auxilary code + typo fix in ast.cpp Needed the intellisense directive ifdef wrap. --- project/auxillary/builder.cpp | 4 ++- project/auxillary/builder.hpp | 6 ++-- project/auxillary/editor.hpp | 67 ----------------------------------- project/auxillary/scanner.hpp | 7 ++-- project/components/ast.cpp | 2 +- 5 files changed, 12 insertions(+), 74 deletions(-) delete mode 100644 project/auxillary/editor.hpp diff --git a/project/auxillary/builder.cpp b/project/auxillary/builder.cpp index 0ba2a7b..d0ca76d 100644 --- a/project/auxillary/builder.cpp +++ b/project/auxillary/builder.cpp @@ -1,4 +1,6 @@ -#include "builder.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# include "builder.hpp" +#endif Builder Builder::open( char const* path ) { diff --git a/project/auxillary/builder.hpp b/project/auxillary/builder.hpp index ecb459a..4c4eb1c 100644 --- a/project/auxillary/builder.hpp +++ b/project/auxillary/builder.hpp @@ -1,5 +1,7 @@ -#pragma once -#include "gen.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "gen.hpp" +#endif struct Builder { diff --git a/project/auxillary/editor.hpp b/project/auxillary/editor.hpp deleted file mode 100644 index c8eae23..0000000 --- a/project/auxillary/editor.hpp +++ /dev/null @@ -1,67 +0,0 @@ -#pragma once -#include "gen.scanner.hpp" - -struct Policy -{ - // Nothing for now. -}; - -enum class SymbolType : u32 -{ - Code, - Line, - Marker -}; - -struct Editor -{ - enum RequestType : u32 - { - Add, - Replace, - Remove - }; - - struct SymbolData - { - Policy Policy; - SymbolInfo Info; - }; - - struct RequestEntry - { - union { - SymbolData Symbol; - String Specification; - }; - RequestType Type; - }; - - struct Receipt - { - StringCached File; - Code Found; - Code Written; - bool Result; - }; - - static AllocatorInfo Allocator; - - static void set_allocator( AllocatorInfo allocator ); - - Array Files; - String Buffer; - Array Requests; - - void add_files( s32 num, char const** files ); - - void add ( SymbolInfo definition, Policy policy, Code to_inject ); - void remove ( SymbolInfo definition, Policy policy ); - void replace( SymbolInfo definition, Policy policy, Code to_replace); - -# ifdef GEN_FEATURE_EDITOR_REFACTOR - void refactor( char const* file_path, char const* specification_path ); -# endif - - bool process_requests( Array out_receipts ); -}; diff --git a/project/auxillary/scanner.hpp b/project/auxillary/scanner.hpp index 6bdffa8..5bc60d5 100644 --- a/project/auxillary/scanner.hpp +++ b/project/auxillary/scanner.hpp @@ -1,6 +1,7 @@ - -#pragma once -#include "gen.hpp" +#ifdef GEN_INTELLISENSE_DIRECTIVES +# pragma once +# include "gen.hpp" +#endif // This is a simple file reader that reads the entire file into memory. // It has an extra option to skip the first few lines for undesired includes. diff --git a/project/components/ast.cpp b/project/components/ast.cpp index 804d3f3..b5e1866 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -1,4 +1,4 @@ -#if GEN_INTELLISENSE_DIRECTIVES +#ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "static_data.cpp" #endif From 107681825073265679a9e190a1fc526052baf7e4 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 3 Sep 2023 20:29:45 -0400 Subject: [PATCH 15/26] Got whitepace stripping properly working (AFAICT) for parse_define() Made debug for viewing whitespace in AST::is_equal with String::visualize_whitespace() Format stripping code is currently confined within parse_define() I plan to move it to its own function soon, I just want to make sure its finalized first. Other unvalidated content will need to have an extra check for preprocessed lines. Example: Function bodies can have a #define . I cannot strip the last as it will break the semantic importance to distinguish that line. So it needs to be: In the content string that is minimally preserved --- project/components/ast.cpp | 160 ++++++++++++++--------- project/components/interface.parsing.cpp | 153 ++++++++++++++++++++-- project/dependencies/strings.hpp | 47 ++++++- 3 files changed, 281 insertions(+), 79 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index b5e1866..c405510 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -981,36 +981,36 @@ bool AST::is_equal( AST* other ) { using namespace ECode; - #define check_member_val( val ) \ - if ( val != other->val ) \ - { \ - log_fmt("AST::is_equal: Member - " #val " failed\n" \ - "AST : %S\n" \ - "Other: %S\n" \ - , debug_str() \ - , other->debug_str() \ - ); \ - \ - return false; \ + #define check_member_val( val ) \ + if ( val != other->val ) \ + { \ + log_fmt("\nAST::is_equal: Member - " #val " failed\n" \ + "AST : %S\n" \ + "Other: %S\n" \ + , debug_str() \ + , other->debug_str() \ + ); \ + \ + return false; \ } - #define check_member_str( str ) \ - if ( str != other->str ) \ - { \ - log_fmt("AST::is_equal: Member string - "#str " failed\n" \ - "AST : %S\n" \ - "Other: %S\n" \ - , debug_str() \ - , other->debug_str() \ - ); \ - \ - return false; \ + #define check_member_str( str ) \ + if ( str != other->str ) \ + { \ + log_fmt("\nAST::is_equal: Member string - "#str " failed\n" \ + "AST : %S\n" \ + "Other: %S\n" \ + , debug_str() \ + , other->debug_str() \ + ); \ + \ + return false; \ } - #define check_member_content( content ) \ - if ( content != other->content ) \ - { \ - log_fmt("AST::is_equal: Member content - "#content " failed\n" \ + #define check_member_content( content ) \ + if ( content != other->content ) \ + { \ + log_fmt("\nAST::is_equal: Member content - "#content " failed\n" \ "AST : %S\n" \ "Other: %S\n" \ , debug_str() \ @@ -1021,43 +1021,43 @@ bool AST::is_equal( AST* other ) "so it must be verified by eye for now\n" \ "AST Content:\n%S\n" \ "Other Content:\n%S\n" \ - , content \ - , other->content \ + , content.visualize_whitespace() \ + , other->content.visualize_whitespace() \ ); \ } - #define check_member_ast( ast ) \ - if ( ast ) \ - { \ - if ( other->ast == nullptr ) \ - { \ - log_fmt("AST::is_equal: Failed for member " #ast " other equivalent param is null\n" \ - "AST : %s\n" \ - "Other: %s\n" \ - "For ast member: %s\n" \ - , debug_str() \ - , other->debug_str() \ - , ast->debug_str() \ - ); \ - \ - return false; \ - } \ - \ - if ( ! ast->is_equal( other->ast ) ) \ - { \ - log_fmt( "AST::is_equal: Failed for " #ast"\n" \ - "AST : %S\n" \ - "Other: %S\n" \ - "For ast member: %S\n" \ - "other's ast member: %S\n" \ - , debug_str() \ - , other->debug_str() \ - , ast->debug_str() \ - , other->ast->debug_str() \ - ); \ - \ - return false; \ - } \ + #define check_member_ast( ast ) \ + if ( ast ) \ + { \ + if ( other->ast == nullptr ) \ + { \ + log_fmt("\nAST::is_equal: Failed for member " #ast " other equivalent param is null\n" \ + "AST : %s\n" \ + "Other: %s\n" \ + "For ast member: %s\n" \ + , debug_str() \ + , other->debug_str() \ + , ast->debug_str() \ + ); \ + \ + return false; \ + } \ + \ + if ( ! ast->is_equal( other->ast ) ) \ + { \ + log_fmt( "\nAST::is_equal: Failed for " #ast"\n" \ + "AST : %S\n" \ + "Other: %S\n" \ + "For ast member: %S\n" \ + "other's ast member: %S\n" \ + , debug_str() \ + , other->debug_str() \ + , ast->debug_str() \ + , other->ast->debug_str() \ + ); \ + \ + return false; \ + } \ } case NewLine: @@ -1313,7 +1313,7 @@ bool AST::is_equal( AST* other ) { if ( curr_other == nullptr ) { - log_fmt("AST::is_equal: Failed for parameter, other equivalent param is null\n" + log_fmt("\nAST::is_equal: Failed for parameter, other equivalent param is null\n" "AST : %S\n" "Other: %S\n" "For ast member: %S\n" @@ -1323,9 +1323,39 @@ bool AST::is_equal( AST* other ) return false; } - if ( ! curr->is_equal( curr_other ) ) + if ( ! curr->Name != curr_other->Name ) { - log_fmt( "AST::is_equal: Failed for parameter\n" + log_fmt( "\nAST::is_equal: Failed for parameter name check\n" + "AST : %S\n" + "Other: %S\n" + "For ast member: %S\n" + "other's ast member: %S\n" + , debug_str() + , other->debug_str() + , curr->debug_str() + , curr_other->debug_str() + ); + return false; + } + + if ( curr->ValueType && ! curr->ValueType->is_equal(curr_other->ValueType) ) + { + log_fmt( "\nAST::is_equal: Failed for parameter value type check\n" + "AST : %S\n" + "Other: %S\n" + "For ast member: %S\n" + "other's ast member: %S\n" + , debug_str() + , other->debug_str() + , curr->debug_str() + , curr_other->debug_str() + ); + return false; + } + + if ( curr->Value && ! curr->Value->is_equal(curr_other->Value) ) + { + log_fmt( "\nAST::is_equal: Failed for parameter value check\n" "AST : %S\n" "Other: %S\n" "For ast member: %S\n" @@ -1494,7 +1524,7 @@ bool AST::is_equal( AST* other ) { if ( curr_other == nullptr ) { - log_fmt("AST::is_equal: Failed for body, other equivalent param is null\n" + log_fmt("\nAST::is_equal: Failed for body, other equivalent param is null\n" "AST : %S\n" "Other: %S\n" "For ast member: %S\n" @@ -1506,7 +1536,7 @@ bool AST::is_equal( AST* other ) if ( ! curr->is_equal( curr_other ) ) { - log_fmt( "AST::is_equal: Failed for body\n" + log_fmt( "\nAST::is_equal: Failed for body\n" "AST : %S\n" "Other: %S\n" "For ast member: %S\n" diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 87eba4b..f13c962 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -1276,19 +1276,148 @@ CodeDefine parse_define() return CodeInvalid; } - // s32 left = currtok.Length; - // char const* scanner = currtok.Text; - // while ( left ) - // { - // if ( scanner[0] == ' ' ) - // { - // scanner++; - // left--; - // continue; - // } - // } + if ( currtok.Length == 0 ) + { + define->Content = get_cached_string( currtok ); + eat( TokType::Preprocess_Content ); - define->Content = get_cached_string( currtok ); + Context.pop(); + return define; + } + + String content = String::make_reserve( GlobalAllocator, currtok.Length ); + +#define cut_length ( scanner - currtok.Text - last_cut ) +#define cut_ptr ( currtok.Text + last_cut ) +#define pos ( sptr( scanner ) - sptr( currtok.Text ) ) + s32 tokleft = currtok.Length; + sptr last_cut = 0; + char const* scanner = currtok.Text; + + if ( scanner[0] == ' ' ) + { + ++ scanner; + -- tokleft; + last_cut = 1; + } + + while ( tokleft ) + { + if ( tokleft > 1 && char_is_space( scanner[0] ) && char_is_space( scanner[ 1 ] ) ) + { + content.append( cut_ptr, cut_length ); + do + { + ++ scanner; + -- tokleft; + } + while ( tokleft && char_is_space( scanner[0] ) ); + + last_cut = sptr( scanner ) - sptr( currtok.Text ); + + // Preserve only 1 space of formattting + if ( content.back() != ' ' ) + content.append( ' ' ); + continue; + } + + if ( scanner[0] == '\t' ) + { + if ( pos > last_cut ) + content.append( cut_ptr, cut_length ); + + // Replace with a space + if ( content.back() != ' ' ) + content.append( ' ' ); + + ++ scanner; + -- tokleft; + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + if ( tokleft > 1 && scanner[0] == '\r' && scanner[1] == '\n' ) + { + if ( pos > last_cut ) + content.append( cut_ptr, cut_length ); + + // Replace with a space + if ( content.back() != ' ' ) + content.append( ' ' ); + + scanner += 2; + tokleft -= 2; + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + if ( scanner[0] == '\n' ) + { + if ( pos > last_cut ) + content.append( cut_ptr, cut_length ); + + // Replace with a space + if ( content.back() != ' ' ) + content.append( ' ' ); + + ++ scanner; + -- tokleft; + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + if ( scanner[0] == '\\' ) + { + s32 amount_to_skip = 1; + if ( tokleft > 1 && scanner[1] == '\n' ) + { + amount_to_skip = 2; + } + else if ( tokleft > 2 && scanner[1] == '\r' && scanner[2] == '\n' ) + { + amount_to_skip = 3; + } + + if ( amount_to_skip > 1 ) + { + if ( pos == last_cut ) + { + // If the backslash is the first character on the line, then skip it + scanner += amount_to_skip; + tokleft -= amount_to_skip; + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + // We have content to add. + content.append( cut_ptr, pos - last_cut ); + + scanner += amount_to_skip; + tokleft -= amount_to_skip; + } + else + { + ++ scanner; + -- tokleft; + } + + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + ++ scanner; + -- tokleft; + } + + if ( last_cut < currtok.Length ) + { + content.append( cut_ptr, currtok.Length - last_cut ); + } +#undef cut_ptr +#undef cut_length +#undef pos + + define->Content = get_cached_string( content ); eat( TokType::Preprocess_Content ); Context.pop(); diff --git a/project/dependencies/strings.hpp b/project/dependencies/strings.hpp index b834575..cc57a96 100644 --- a/project/dependencies/strings.hpp +++ b/project/dependencies/strings.hpp @@ -101,6 +101,11 @@ struct String bool make_space_for( char const* str, sw add_len ); + bool append( char c ) + { + return append( & c, 1 ); + } + bool append( char const* str ) { return append( str, str_len( str ) ); @@ -264,14 +269,52 @@ struct String return trim( " \t\r\n\v\f" ); } + // Debug function that provides a copy of the string with whitespace characters visualized. + String visualize_whitespace() const + { + Header* header = (Header*)(Data - sizeof(Header)); + + String result = make_reserve(header->Allocator, length() * 2); // Assume worst case for space requirements. + + for ( char c : *this ) + { + switch ( c ) + { + case ' ': + result.append('·'); + break; + case '\t': + result.append('→'); + break; + case '\n': + result.append('↵'); + break; + case '\r': + result.append('⏎'); + break; + case '\v': + result.append('⇕'); + break; + case '\f': + result.append('⌂'); + break; + default: + result.append(c); + break; + } + } + + return result; + } + // For-range support - char* begin() + char* begin() const { return Data; } - char* end() + char* end() const { Header const& header = * rcast( Header const*, Data - sizeof( Header )); From f2d4ec96f0969a9e8a70e602adbf8bd120f9bf63 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 3 Sep 2023 20:34:27 -0400 Subject: [PATCH 16/26] Dumb fix for name check on parameters Its now spitting out actual validation failures. --- project/components/ast.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index c405510..f8c6884 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -1323,7 +1323,7 @@ bool AST::is_equal( AST* other ) return false; } - if ( ! curr->Name != curr_other->Name ) + if ( curr->Name != curr_other->Name ) { log_fmt( "\nAST::is_equal: Failed for parameter name check\n" "AST : %S\n" From 543427dfe54ed891e371f1906b5d81048075fb00 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 3 Sep 2023 23:36:51 -0400 Subject: [PATCH 17/26] Fixes to parsing for validation Removed whitespace stripping from parse_type, prepped for doing some major changes for function signature typenames Moved template argument parsing to its own helper function since its used in more the one spot. Latest failure is due to stack overflow when validating parameters. (Shouldn't be hard to debug) --- project/components/ast.hpp | 6 +- project/components/ast_types.hpp | 3 +- project/components/interface.parsing.cpp | 148 ++++++++++------------- 3 files changed, 68 insertions(+), 89 deletions(-) diff --git a/project/components/ast.hpp b/project/components/ast.hpp index c920c76..c7460f9 100644 --- a/project/components/ast.hpp +++ b/project/components/ast.hpp @@ -236,7 +236,7 @@ struct AST { AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable - AST* Specs; // Destructor, Function, Operator, Type symbol, Variable + AST* Specs; // Destructor, Function, Operator, Typename, Variable union { AST* InitializerList; // Constructor AST* ParentType; // Class, Struct @@ -249,7 +249,7 @@ struct AST AST* Params; // Constructor, Function, Operator, Template }; union { - AST* ArrExpr; // Type Symbol + AST* ArrExpr; // Typename AST* Body; // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union AST* Declaration; // Friend, Template AST* Value; // Parameter, Variable @@ -288,7 +288,7 @@ struct AST_POD { AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable AST* Attributes; // Class, Enum, Function, Struct, Typename, Union, Using, Variable - AST* Specs; // Function, Operator, Type symbol, Variable + AST* Specs; // Function, Operator, Typename, Variable union { AST* InitializerList; // Constructor AST* ParentType; // Class, Struct diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 236aab9..54e61cc 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -475,7 +475,8 @@ struct AST_Type char _PAD_CMT_[ sizeof(AST*) ]; CodeAttributes Attributes; CodeSpecifiers Specs; - char _PAD_PROPERTIES_[ sizeof(AST*) * 2 ]; + CodeType ReturnType; // Only used for function signatures + CodeParam Params; // Only used for function signatures Code ArrExpr; }; }; diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index f13c962..fd8ff84 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -1713,6 +1713,40 @@ CodeAttributes parse_attributes() return { nullptr }; } +/* + This a brute-froce make all the arguments part of the token provided. + Can have in-place function signatures, regular identifiers, in-place typenames, compile-time expressions, parameter-pack expansion, etc. + This means that validation can only go so far, and so if there is any different in formatting + passed the basic stripping supported it report a soft failure. +*/ +internal inline +void parse_template_args( Parser::Token& token ) +{ + using namespace Parser; + + if ( currtok.Type == TokType::Operator && currtok.Text[0] == '<' && currtok.Length == 1 ) + { + eat( TokType::Operator ); + + s32 level = 0; + while ( left && ( currtok.Text[0] != '>' || level > 0 )) + { + if ( currtok.Text[0] == '<' ) + level++; + + if ( currtok.Text[0] == '>' ) + level--; + + eat( currtok.Type ); + } + + eat( TokType::Operator ); + + // Extend length of name to last token + token.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)token.Text; + } +} + internal Parser::Token parse_identifier() { @@ -1721,9 +1755,10 @@ Parser::Token parse_identifier() Token name = currtok; Context.Scope->Name = name; - eat( TokType::Identifier ); + parse_template_args( name ); + while ( check( TokType::Access_StaticSymbol ) ) { eat( TokType::Access_StaticSymbol ); @@ -1745,62 +1780,7 @@ Parser::Token parse_identifier() name.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)name.Text; eat( TokType::Identifier ); - if ( check( TokType::Operator ) && currtok.Text[0] == '<' ) - { - eat( TokType::Operator ); - - // Template arguments can be complex so were not validating if they are correct. - s32 level = 0; - while ( left && (currtok.Text[0] != '>' || level > 0 ) ) - { - if ( currtok.Text[0] == '<' ) - level++; - - else if ( currtok.Text[0] == '>' && level > 0 ) - level--; - - eat( currtok.Type ); - } - - if ( left == 0 ) - { - log_failure( "Error, unexpected end of template arguments\n%s", Context.to_string() ); - Context.pop(); - return { nullptr, 0, TokType::Invalid }; - } - - eat( TokType::Operator ); - name.Length = ( (sptr)prevtok.Text + (sptr)prevtok.Length ) - (sptr)name.Text; - } - } - - if ( check( TokType::Operator ) && currtok.Text[0] == '<' ) - { - eat( TokType::Operator ); - - // Template arguments can be complex so were not validating if they are correct. - s32 level = 0; - while ( left && (currtok.Text[0] != '>' || level > 0 ) ) - { - if ( currtok.Text[0] == '<' ) - level++; - - else if ( currtok.Text[0] == '>' && level > 0 ) - level--; - - eat( currtok.Type ); - } - - if ( left == 0 ) - { - log_failure( "Error, unexpected end of template arguments\n%s", Context.to_string() ); - Context.pop(); - return { nullptr, 0, TokType::Invalid }; - } - - eat( TokType::Operator ); - - name.Length = ( (sptr)prevtok.Text + (sptr)prevtok.Length ) - (sptr)name.Text; + parse_template_args( name ); } Context.pop(); @@ -4525,6 +4505,9 @@ CodeTemplate parse_template( StrC def ) return parse_template(); } +// This is a bit of a mess, but it works +// Parsing typename is arguably one of the worst aspects of C/C++. +// This is an effort to parse it without a full blown or half-blown compliant parser. internal CodeType parse_type( bool* is_function ) { @@ -4539,8 +4522,10 @@ CodeType parse_type( bool* is_function ) Token name = { nullptr, 0, TokType::Invalid }; Token brute_sig = { currtok.Text, 0, TokType::Invalid }; + // Attributes are assumed to be before the type signature CodeAttributes attributes = parse_attributes(); + // Deal with specifiers before the type signature while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -4564,9 +4549,11 @@ CodeType parse_type( bool* is_function ) return CodeInvalid; } + // All kinds of nonsense can makeup a type signature, first we check for a in-place definition of a class, enum, or struct if ( currtok.Type == TokType::Decl_Class || currtok.Type == TokType::Decl_Enum - || currtok.Type == TokType::Decl_Struct ) + || currtok.Type == TokType::Decl_Struct + || currtok.Type == TokType::Decl_Union ) { name = currtok; eat( currtok.Type ); @@ -4575,6 +4562,8 @@ CodeType parse_type( bool* is_function ) eat( TokType::Identifier ); Context.Scope->Name = name; } + + // Check if native type keywords are used, eat them for the signature. else if ( currtok.Type >= TokType::Type_Unsigned && currtok.Type <= TokType::Type_MS_W64 ) { name = currtok; @@ -4588,6 +4577,8 @@ CodeType parse_type( bool* is_function ) name.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)name.Text; Context.Scope->Name = name; } + + // The usual Identifier type signature that may have namespace qualifiers else { name = parse_identifier(); @@ -4598,29 +4589,6 @@ CodeType parse_type( bool* is_function ) Context.pop(); return CodeInvalid; } - - // Problably dealing with a templated symbol - if ( currtok.Type == TokType::Operator && currtok.Text[0] == '<' && currtok.Length == 1 ) - { - eat( TokType::Operator ); - - s32 level = 0; - while ( left && ( currtok.Text[0] != '>' || level > 0 )) - { - if ( currtok.Text[0] == '<' ) - level++; - - if ( currtok.Text[0] == '>' ) - level--; - - eat( currtok.Type ); - } - - eat( TokType::Operator ); - - // Extend length of name to last token - name.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)name.Text; - } } while ( left && currtok.is_specifier() ) @@ -4642,10 +4610,14 @@ CodeType parse_type( bool* is_function ) eat( currtok.Type ); } + // For function type signatures + CodeType return_type = NoCode; + CodeParam params = NoCode; + bool is_first_capture = true; while ( check( TokType::Capture_Start ) && context_tok.Type != TokType::Decl_Operator ) { - // Brute force capture the entire thing. + // Brute force capture the entire thing // Function typedefs are complicated and there are not worth dealing with for validation at this point... eat( TokType::Capture_Start ); @@ -4704,9 +4676,9 @@ CodeType parse_type( bool* is_function ) } } - String - name_stripped = String::make( GlobalAllocator, name ); - name_stripped.strip_space(); + // This is bad we cannot strip the name if it contains the full function signature's parameters, parameters at minimum must be separate. + String name_stripped = String::make( GlobalAllocator, name ); + // name_stripped.strip_space(); result->Name = get_cached_string( name_stripped ); @@ -4716,6 +4688,12 @@ CodeType parse_type( bool* is_function ) if ( is_param_pack ) result->IsParamPack = true; + if ( return_type ) + result->ReturnType = return_type; + + if ( params ) + result->Params = params; + Context.pop(); return result; } From 3868e1e8111c1c39c05fb9f764e540b29b830fc1 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Mon, 4 Sep 2023 12:32:31 -0400 Subject: [PATCH 18/26] Added cursed typedef --- test/CURSED_TYPEDEF.h | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 test/CURSED_TYPEDEF.h diff --git a/test/CURSED_TYPEDEF.h b/test/CURSED_TYPEDEF.h new file mode 100644 index 0000000..c2bad60 --- /dev/null +++ b/test/CURSED_TYPEDEF.h @@ -0,0 +1,35 @@ +#include + +class MyClass; + +enum class MyEnum : short { VAL1, VAL2 }; + +struct OuterStruct { + union NamedUnion { + struct InnerStruct { + double d; + char c; + } inner; + int i; + } unionInstance; +}; + +template +struct TemplateStruct { + T member[N]; +}; + +template<> +struct TemplateStruct { + int specialMember[10]; +}; + +typedef decltype(nullptr) (MyClass::*InsaneComplexTypeDef)( + decltype((MyEnum::VAL1 == MyEnum::VAL2) ? 1 : 2.0) + (TemplateStruct::*ptr)[5][alignof(double)], + std::function&&, + void (MyClass::*memFnPtr)(TemplateStruct))>, + int (MyClass::*&refToMemFnPtr)(TemplateStruct), + int (TemplateStruct::*memberPointer)[10], + char&&... +) volatile const && noexcept; From 3e249d9bc566d49ee10b4139651b17443ff8c11d Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 5 Sep 2023 01:44:04 -0400 Subject: [PATCH 19/26] Reorganization of parser, refactor of parse_type( bool* ) and progression of parser docs Wanted to make parser implementation easier to sift through, so I emphasized alphabetical order more. Since I couldn't just strip whitespace from typenames I decided to make the parse_type more aware of the typename's components if it was a function signature. This ofc lead to the dark & damp hell that is parsing typenames. Also made initial implementation to support parsing decltype within a typename signature.. The test failure for the singleheader is still a thing, these changes have not addressed that. --- .vscode/settings.json | 4 +- docs/ASTs.md | 2 + docs/Parser_Algo.md | 91 + docs/Parsing.md | 11 +- project/components/ast.cpp | 43 +- project/components/ast.hpp | 47 +- project/components/ast_types.hpp | 6 +- project/components/gen/especifier.hpp | 2 + project/components/interface.parsing.cpp | 2925 ++++++++++++---------- project/enums/ESpecifier.csv | 1 + 10 files changed, 1752 insertions(+), 1380 deletions(-) create mode 100644 docs/Parser_Algo.md diff --git a/.vscode/settings.json b/.vscode/settings.json index cd8b34c..0db6b4d 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,7 +24,9 @@ "filesystem": "cpp", "format": "cpp", "ratio": "cpp", - "xstring": "cpp" + "xstring": "cpp", + "functional": "cpp", + "vector": "cpp" }, "C_Cpp.intelliSenseEngineFallback": "disabled", "mesonbuild.configureOnOpen": true, diff --git a/docs/ASTs.md b/docs/ASTs.md index 81e19ba..ea065df 100644 --- a/docs/ASTs.md +++ b/docs/ASTs.md @@ -561,6 +561,8 @@ Fields: ```cpp CodeAttributes Attributes; CodeSpecifiers Specs; +CodeReturnType ReturnType; +CodeParam Params; Code ArrExpr; Code Prev; Code Next; diff --git a/docs/Parser_Algo.md b/docs/Parser_Algo.md new file mode 100644 index 0000000..9c4958e --- /dev/null +++ b/docs/Parser_Algo.md @@ -0,0 +1,91 @@ +# Parser's Algorithim + +gencpp uses a hand-written recursive descent parser. Both the lexer and parser handle a full C/C++ file in a single pass. + +## Notable implementation background + +### Lexer + +The lex procedure does the lexical pass of content provided as a `StrC` type. +The tokens are stored (for now) in `gen::Parser::Tokens`. + +Fields: +```cpp +Array Arr; +s32 Idx; +``` + + +What token types are supported can be found in [ETokType.csv](../project/enums/ETokType.csv) you can also find the token types in [ETokType.h](../project/components/gen/etoktype.cpp) , which is the generated enum from the csv file. + +Tokens are defined with the struct `gen::Parser::Token`: + +Fields: +```cpp +char const* Text; +sptr Length; +TokType Type; +s32 Line; +s32 Column; +bool IsAssign; +``` + +`IsAssign` is a flag that is set when the token is an assignment operator. Which is used for various purposes: + +* Using statment assignment +* Parameter argument default value assignment +* Variable declaration initialization assignment + +I plan to replace IsAssign with a general flags field and properly keep track of all operator types instead of abstracting it away to `ETokType::Operator`. + +Traversing the tokens is done with the following interface macros: + +| Macro | Description | +| --- | --- | +| `currtok_noskip` | Get the current token without skipping whitespace | +| `currtok` | Get the current token, skip any whitespace tokens | +| `prevtok` | Get the previous token (does not skip whitespace) | +| `nexttok` | Get the next token (does not skip whitespace) | +| `eat( Token Type )` | Check to see if the current token is of the given type, if so, advance Token's index to the next token | +| `left` | Get the number of tokens left in the token array | +| `check_noskip` | Check to see if the current token is of the given type, without skipping whitespace | +| `check` | Check to see if the current token is of the given type, skip any whitespace tokens | + +### Parser + +The parser has a limited user interface, only specific types of definitions or statements are expected to be provided by the user directly when using to construct an AST dynamically (See SOA for example). It however does attempt to provide capability to parse a full C/C++ from production codebases. + +Each public user interface procedure has the following format: + +```cpp +CodeStruct parse_( StrC def ) +{ + check_parse_args( def ); + using namespace Parser; + + TokArray toks = lex( def ); + if ( toks.Arr == nullptr ) + return CodeInvalid; + + // Parse the tokens and return a constructed AST using internal procedures + ... +} +``` + +The most top-level parsing procedure used for C/C++ file parsing is `parse_global_body`: + +It uses a helper procedure called `parse_global_nspace`. + +Each internal procedure will be + +## parse_global_nspace + +1. Make sure the type provided to the helper function is a `Namespace_Body`, `Global_Body`, `Export_Body`, `Extern_Linkage_body`. +2. If its not a `Global_Body` eat the opening brace for the scope. +3. + + +## parse_type + +This is the worst part of the parser. Because other than actual expression values in C++, typenames are the second worst thing to parse in the langauge. + diff --git a/docs/Parsing.md b/docs/Parsing.md index 12f189b..198df87 100644 --- a/docs/Parsing.md +++ b/docs/Parsing.md @@ -1,11 +1,12 @@ # Parsing -The library features a naive parser tailored for only what the library needs to construct the supported syntax of C++ into its AST. +The library features a naive parser tailored for only what the library needs to construct the supported syntax of C++ into its AST. + This parser does not, and should not do the compiler's job. By only supporting this minimal set of features, the parser is kept (so far) around 5000 loc. You can think of this parser of a frontend parser vs a semantic parser. Its intuitively similar to WYSIWYG. What you precerive as the syntax from the user-side before the compiler gets a hold of it, is what you get. -The parsing implementation supports the following for the user: +User exposed interface: ```cpp CodeClass parse_class ( StrC class_def ); @@ -55,9 +56,9 @@ Any preprocessor definition abuse that changes the syntax of the core language i Exceptions: * function signatures are allowed for a preprocessed macro: `neverinline MACRO() { ... }` - * Disable with: `#define GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES` + * Disable with: `#define GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES` * typedefs allow for a preprocessed macro: `typedef MACRO();` - * Disable with: `#define GEN_PARSER_DISABLE_MACRO_TYPEDEF` + * Disable with: `#define GEN_PARSER_DISABLE_MACRO_TYPEDEF` *(See functions `parse_operator_function_or_variable` and `parse_typedef` )* @@ -75,8 +76,6 @@ The lexing and parsing takes shortcuts from whats expected in the standard. * Assumed to *come before specifiers* (`const`, `constexpr`, `extern`, `static`, etc) for a function * Or in the usual spot for class, structs, (*right after the declaration keyword*) * typedefs have attributes with the type (`parse_type`) -* As a general rule; if its not available from the upfront constructors, its not available in the parsing constructors. - * *Upfront constructors are not necessarily used in the parsing constructors, this is just a good metric to know what can be parsed.* * Parsing attributes can be extended to support user defined macros by defining `GEN_DEFINE_ATTRIBUTE_TOKENS` (see `gen.hpp` for the formatting) Empty lines used throughout the file are preserved for formatting purposes during ast serialization. diff --git a/project/components/ast.cpp b/project/components/ast.cpp index f8c6884..b83a6ed 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -773,7 +773,8 @@ String AST::to_string() result.append( "typedef "); - if ( IsFunction ) + // Determines if the typedef is a function typename + if ( UnderlyingType->ReturnType ) result.append( UnderlyingType->to_string() ); else result.append_fmt( "%S %S", UnderlyingType->to_string(), Name ); @@ -796,21 +797,45 @@ String AST::to_string() case Typename: { - if ( Attributes || Specs ) + #if GEN_USE_NEW_TYPENAME_PARSING + if ( ReturnType && Params ) { if ( Attributes ) result.append_fmt( "%S ", Attributes->to_string() ); - - if ( Specs ) - result.append_fmt( "%S %S", Name, Specs->to_string() ); - else - result.append_fmt( "%S", Name ); + { + if ( Specs ) + result.append_fmt( "%S ( %S ) ( %S ) %S", ReturnType->to_string(), Name, Params->to_string(), Specs->to_string() ); + else + result.append_fmt( "%S ( %S ) ( %S )", ReturnType->to_string(), Name, Params->to_string() ); + } + + break; } - else + #else + if ( ReturnType && Params ) { - result.append_fmt( "%S", Name ); + if ( Attributes ) + result.append_fmt( "%S ", Attributes->to_string() ); + else + { + if ( Specs ) + result.append_fmt( "%S %S ( %S ) %S", ReturnType->to_string(), Name, Params->to_string(), Specs->to_string() ); + else + result.append_fmt( "%S %S ( %S )", ReturnType->to_string(), Name, Params->to_string() ); + } + + break; } + #endif + + if ( Attributes ) + result.append_fmt( "%S ", Attributes->to_string() ); + + if ( Specs ) + result.append_fmt( "%S %S", Name, Specs->to_string() ); + else + result.append_fmt( "%S", Name ); if ( IsParamPack ) result.append("..."); diff --git a/project/components/ast.hpp b/project/components/ast.hpp index c7460f9..13f0604 100644 --- a/project/components/ast.hpp +++ b/project/components/ast.hpp @@ -6,11 +6,6 @@ #include "gen/especifier.hpp" #endif -namespace Parser -{ - struct Token; -} - struct AST; struct AST_Body; struct AST_Attributes; @@ -234,12 +229,15 @@ struct AST union { struct { - AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable - AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable - AST* Specs; // Destructor, Function, Operator, Typename, Variable + union { + AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable + AST* SpecsFuncSuffix; // Only used with typenames, to store the function suffix if typename is function signature. + }; + AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable + AST* Specs; // Destructor, Function, Operator, Typename, Variable union { AST* InitializerList; // Constructor - AST* ParentType; // Class, Struct + AST* ParentType; // Class, Struct, ParentType->Next has a possible list of interfaces. AST* ReturnType; // Function, Operator AST* UnderlyingType; // Enum, Typedef AST* ValueType; // Parameter, Variable @@ -249,13 +247,13 @@ struct AST AST* Params; // Constructor, Function, Operator, Template }; union { - AST* ArrExpr; // Typename - AST* Body; // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union - AST* Declaration; // Friend, Template - AST* Value; // Parameter, Variable + AST* ArrExpr; // Typename + AST* Body; // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union + AST* Declaration; // Friend, Template + AST* Value; // Parameter, Variable }; }; - StringCached Content; // Attributes, Comment, Execution, Include + StringCached Content; // Attributes, Comment, Execution, Include SpecifierT ArrSpecs[AST::ArrSpecs_Cap]; // Specifiers }; union { @@ -286,12 +284,15 @@ struct AST_POD union { struct { - AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable - AST* Attributes; // Class, Enum, Function, Struct, Typename, Union, Using, Variable - AST* Specs; // Function, Operator, Typename, Variable + union { + AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable + AST* SpecsFuncSuffix; // Only used with typenames, to store the function suffix if typename is function signature. + }; + AST* Attributes; // Class, Enum, Function, Struct, Typename, Union, Using, Variable + AST* Specs; // Function, Operator, Typename, Variable union { AST* InitializerList; // Constructor - AST* ParentType; // Class, Struct + AST* ParentType; // Class, Struct, ParentType->Next has a possible list of interfaces. AST* ReturnType; // Function, Operator AST* UnderlyingType; // Enum, Typedef AST* ValueType; // Parameter, Variable @@ -301,13 +302,13 @@ struct AST_POD AST* Params; // Function, Operator, Template }; union { - AST* ArrExpr; // Type Symbol - AST* Body; // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union - AST* Declaration; // Friend, Template - AST* Value; // Parameter, Variable + AST* ArrExpr; // Type Symbol + AST* Body; // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union + AST* Declaration; // Friend, Template + AST* Value; // Parameter, Variable }; }; - StringCached Content; // Attributes, Comment, Execution, Include + StringCached Content; // Attributes, Comment, Execution, Include SpecifierT ArrSpecs[AST::ArrSpecs_Cap]; // Specifiers }; union { diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 54e61cc..754f63c 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -472,11 +472,11 @@ struct AST_Type char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - char _PAD_CMT_[ sizeof(AST*) ]; + CodeSpecifiers SpecsFuncSuffix; // Only used for function signatures CodeAttributes Attributes; CodeSpecifiers Specs; - CodeType ReturnType; // Only used for function signatures - CodeParam Params; // Only used for function signatures + CodeType ReturnType; // Only used for function signatures + CodeParam Params; // Only used for function signatures Code ArrExpr; }; }; diff --git a/project/components/gen/especifier.hpp b/project/components/gen/especifier.hpp index 387d614..e94dfd1 100644 --- a/project/components/gen/especifier.hpp +++ b/project/components/gen/especifier.hpp @@ -31,6 +31,7 @@ namespace ESpecifier Virtual, Const, Final, + NoExceptions, Override, Pure, NumSpecifiers @@ -67,6 +68,7 @@ namespace ESpecifier { sizeof( "virtual" ), "virtual" }, { sizeof( "const" ), "const" }, { sizeof( "final" ), "final" }, + { sizeof( "noexcept" ), "noexcept" }, { sizeof( "override" ), "override" }, { sizeof( "= 0" ), "= 0" }, }; diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index fd8ff84..1ba4f21 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -1204,8 +1204,30 @@ if ( def.Ptr == nullptr ) \ #pragma endregion Helper Macros -internal Code parse_function_body(); -internal Code parse_global_nspace(); +// Procedure Forwards ( Entire parser internal parser interface ) + +internal Code parse_array_decl (); +internal CodeAttributes parse_attributes (); +internal CodeComment parse_comment (); +internal Code parse_compilcated_definition (); +internal CodeBody parse_class_struct_body ( Parser::TokType which, Parser::Token name = Parser::NullToken ); +internal Code parse_class_struct ( Parser::TokType which, bool inplace_def ); +internal CodeDefine parse_define (); +internal Code parse_foward_or_definition ( Parser::TokType which, bool is_inplace ); +internal CodeFn parse_function_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeType ret_type, Parser::Token name ); +internal Code parse_function_body (); +internal Code parse_global_nspace (); +internal Parser::Token parse_identifier ( bool* possible_member_function = nullptr ); +internal CodeInclude parse_include (); +internal CodeOperator parse_operator_after_ret_type ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeType ret_type ); +internal Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ); +internal CodePragma parse_pragma (); +internal CodeParam parse_params ( bool use_template_capture = false ); +internal CodePreprocessCond parse_preprocess_cond (); +internal Code parse_simple_preprocess ( Parser::TokType which ); +internal Code parse_static_assert (); +internal void parse_template_args ( Parser::Token& token ); +internal CodeVar parse_variable_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeType type, StrC name ); internal CodeClass parse_class ( bool inplace_def = false ); internal CodeConstructor parse_constructor (); @@ -1228,341 +1250,7 @@ internal CodeUsing parse_using (); constexpr bool inplace_def = true; -internal -CodeComment parse_comment() -{ - using namespace Parser; - StackNode scope { nullptr, currtok_noskip, NullToken, txt( __func__ ) }; - Context.push( & scope ); - - CodeComment - result = (CodeComment) make_code(); - result->Type = ECode::Comment; - result->Content = get_cached_string( currtok_noskip ); - result->Name = result->Content; - eat( TokType::Comment ); - - Context.pop(); - return result; -} - -internal inline -CodeDefine parse_define() -{ - using namespace Parser; - push_scope(); - - eat( TokType::Preprocess_Define ); - - CodeDefine - define = (CodeDefine) make_code(); - define->Type = ECode::Preprocess_Define; - - if ( ! check( TokType::Identifier ) ) - { - log_failure( "Error, expected identifier after #define\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - Context.Scope->Name = currtok; - define->Name = get_cached_string( currtok ); - eat( TokType::Identifier ); - - if ( ! check( TokType::Preprocess_Content )) - { - log_failure( "Error, expected content after #define %s\n%s", define->Name, Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - if ( currtok.Length == 0 ) - { - define->Content = get_cached_string( currtok ); - eat( TokType::Preprocess_Content ); - - Context.pop(); - return define; - } - - String content = String::make_reserve( GlobalAllocator, currtok.Length ); - -#define cut_length ( scanner - currtok.Text - last_cut ) -#define cut_ptr ( currtok.Text + last_cut ) -#define pos ( sptr( scanner ) - sptr( currtok.Text ) ) - s32 tokleft = currtok.Length; - sptr last_cut = 0; - char const* scanner = currtok.Text; - - if ( scanner[0] == ' ' ) - { - ++ scanner; - -- tokleft; - last_cut = 1; - } - - while ( tokleft ) - { - if ( tokleft > 1 && char_is_space( scanner[0] ) && char_is_space( scanner[ 1 ] ) ) - { - content.append( cut_ptr, cut_length ); - do - { - ++ scanner; - -- tokleft; - } - while ( tokleft && char_is_space( scanner[0] ) ); - - last_cut = sptr( scanner ) - sptr( currtok.Text ); - - // Preserve only 1 space of formattting - if ( content.back() != ' ' ) - content.append( ' ' ); - continue; - } - - if ( scanner[0] == '\t' ) - { - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - // Replace with a space - if ( content.back() != ' ' ) - content.append( ' ' ); - - ++ scanner; - -- tokleft; - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - if ( tokleft > 1 && scanner[0] == '\r' && scanner[1] == '\n' ) - { - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - // Replace with a space - if ( content.back() != ' ' ) - content.append( ' ' ); - - scanner += 2; - tokleft -= 2; - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - if ( scanner[0] == '\n' ) - { - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - // Replace with a space - if ( content.back() != ' ' ) - content.append( ' ' ); - - ++ scanner; - -- tokleft; - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - if ( scanner[0] == '\\' ) - { - s32 amount_to_skip = 1; - if ( tokleft > 1 && scanner[1] == '\n' ) - { - amount_to_skip = 2; - } - else if ( tokleft > 2 && scanner[1] == '\r' && scanner[2] == '\n' ) - { - amount_to_skip = 3; - } - - if ( amount_to_skip > 1 ) - { - if ( pos == last_cut ) - { - // If the backslash is the first character on the line, then skip it - scanner += amount_to_skip; - tokleft -= amount_to_skip; - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - // We have content to add. - content.append( cut_ptr, pos - last_cut ); - - scanner += amount_to_skip; - tokleft -= amount_to_skip; - } - else - { - ++ scanner; - -- tokleft; - } - - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - ++ scanner; - -- tokleft; - } - - if ( last_cut < currtok.Length ) - { - content.append( cut_ptr, currtok.Length - last_cut ); - } -#undef cut_ptr -#undef cut_length -#undef pos - - define->Content = get_cached_string( content ); - eat( TokType::Preprocess_Content ); - - Context.pop(); - return define; -} - -internal -CodePreprocessCond parse_preprocess_cond() -{ - using namespace Parser; - push_scope(); - - if ( ! currtok.is_preprocess_cond() ) - { - log_failure( "Error, expected preprocess conditional\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - CodePreprocessCond - cond = (CodePreprocessCond) make_code(); - cond->Type = scast(CodeT, currtok.Type - (ETokType::Preprocess_If - ECode::Preprocess_If) ); - eat( currtok.Type ); - - if ( ! check( TokType::Preprocess_Content )) - { - log_failure( "Error, expected content after #define\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - Context.Scope->Name = currtok; - cond->Content = get_cached_string( currtok ); - eat( TokType::Preprocess_Content ); - - Context.pop(); - return cond; -} - -internal -CodeInclude parse_include() -{ - using namespace Parser; - push_scope(); - - CodeInclude - include = (CodeInclude) make_code(); - include->Type = ECode::Preprocess_Include; - eat( TokType::Preprocess_Include ); - - if ( ! check( TokType::String )) - { - log_failure( "Error, expected include string after #include\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - Context.Scope->Name = currtok; - include->Content = get_cached_string( currtok ); - eat( TokType::String ); - - Context.pop(); - return include; -} - -internal -CodePragma parse_pragma() -{ - using namespace Parser; - push_scope(); - - CodePragma - pragma = (CodePragma) make_code(); - pragma->Type = ECode::Preprocess_Pragma; - eat( TokType::Preprocess_Pragma ); - - if ( ! check( TokType::Preprocess_Content )) - { - log_failure( "Error, expected content after #pragma\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - Context.Scope->Name = currtok; - - String - content_stripped = String::make( GlobalAllocator, currtok ); - content_stripped.strip_space(); - - pragma->Content = get_cached_string( content_stripped ); - eat( TokType::Preprocess_Content ); - - Context.pop(); - return pragma; -} - -internal -Code parse_static_assert() -{ - using namespace Parser; - push_scope(); - - Code - assert = make_code(); - assert->Type = ECode::Untyped; - - Token content = currtok; - - Context.Scope->Name = content; - - eat( TokType::StaticAssert ); - eat( TokType::Capture_Start ); - - s32 level = 0; - while ( left && ( currtok.Type != TokType::Capture_End || level > 0 ) ) - { - if ( currtok.Type == TokType::Capture_Start ) - level++; - else if ( currtok.Type == TokType::Capture_End ) - level--; - - eat( currtok.Type ); - } - eat( TokType::Capture_End ); - eat( TokType::Statement_End ); - - content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text; - - - String - content_stripped = String::make( GlobalAllocator, content ); - content_stripped.strip_space(); - - char const* result = str_fmt_buf( "%.*s\n", content.Length, content.Text ); - if ( content_stripped ) - { - result = str_fmt_buf( "%S\n", content_stripped ); - } - - assert->Content = get_cached_string( to_str( result ) ); - assert->Name = assert->Content; - - Context.pop(); - return assert; -} +// Internal parsing functions internal Code parse_array_decl() @@ -1713,969 +1401,22 @@ CodeAttributes parse_attributes() return { nullptr }; } -/* - This a brute-froce make all the arguments part of the token provided. - Can have in-place function signatures, regular identifiers, in-place typenames, compile-time expressions, parameter-pack expansion, etc. - This means that validation can only go so far, and so if there is any different in formatting - passed the basic stripping supported it report a soft failure. -*/ -internal inline -void parse_template_args( Parser::Token& token ) -{ - using namespace Parser; - - if ( currtok.Type == TokType::Operator && currtok.Text[0] == '<' && currtok.Length == 1 ) - { - eat( TokType::Operator ); - - s32 level = 0; - while ( left && ( currtok.Text[0] != '>' || level > 0 )) - { - if ( currtok.Text[0] == '<' ) - level++; - - if ( currtok.Text[0] == '>' ) - level--; - - eat( currtok.Type ); - } - - eat( TokType::Operator ); - - // Extend length of name to last token - token.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)token.Text; - } -} - internal -Parser::Token parse_identifier() +CodeComment parse_comment() { using namespace Parser; - push_scope(); + StackNode scope { nullptr, currtok_noskip, NullToken, txt( __func__ ) }; + Context.push( & scope ); - Token name = currtok; - Context.Scope->Name = name; - eat( TokType::Identifier ); - - parse_template_args( name ); - - while ( check( TokType::Access_StaticSymbol ) ) - { - eat( TokType::Access_StaticSymbol ); - - if ( left == 0 ) - { - log_failure( "Error, unexpected end of static symbol identifier\n%s", Context.to_string() ); - Context.pop(); - return { nullptr, 0, TokType::Invalid }; - } - - if ( currtok.Type != TokType::Identifier ) - { - log_failure( "Error, expected static symbol identifier, not %s\n%s", ETokType::to_str( currtok.Type ), Context.to_string() ); - Context.pop(); - return { nullptr, 0, TokType::Invalid }; - } - - name.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)name.Text; - eat( TokType::Identifier ); - - parse_template_args( name ); - } - - Context.pop(); - return name; -} - -internal inline -CodeParam parse_params( bool use_template_capture = false ) -{ - using namespace Parser; - using namespace ECode; - push_scope(); - - if ( ! use_template_capture ) - eat( TokType::Capture_Start ); - - else - { - if ( check ( TokType::Operator ) && currtok.Text[0] == '<' ) - eat( TokType::Operator ); - } - - if ( ! use_template_capture && check(TokType::Capture_End) ) - { - eat( TokType::Capture_End ); - Context.pop(); - return { nullptr }; - } - - CodeType type = { nullptr }; - Code value = { nullptr }; - - if ( check( TokType::Varadic_Argument) ) - { - eat( TokType::Varadic_Argument ); - - Context.pop(); - return param_varadic; - } - - type = parse_type(); - if ( type == Code::Invalid ) - { - Context.pop(); - return CodeInvalid; - } - - Token name = NullToken; - - if ( check( TokType::Identifier ) ) - { - name = currtok; - eat( TokType::Identifier ); - - if ( currtok.IsAssign ) - { - eat( TokType::Operator ); - - Token value_tok = currtok; - - if ( currtok.Type == TokType::Comma ) - { - log_failure( "Expected value after assignment operator\n%s.", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - while ( left - && currtok.Type != TokType::Comma - && currtok.Type != TokType::Capture_End - ) - { - value_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)value_tok.Text; - eat( currtok.Type ); - } - - value = untyped_str( value_tok ); - } - } - - CodeParam - result = (CodeParam) make_code(); - result->Type = Parameters; - - if ( name.Length > 0 ) - result->Name = get_cached_string( name ); - - result->ValueType = type; - - if ( value ) - result->Value = value; - - result->NumEntries++; - - while ( left - && use_template_capture ? - currtok.Type != TokType::Operator && currtok.Text[0] != '>' - : currtok.Type != TokType::Capture_End ) - { - eat( TokType::Comma ); - - Code type = { nullptr }; - Code value = { nullptr }; - - if ( check( TokType::Varadic_Argument) ) - { - eat( TokType::Varadic_Argument ); - result.append( param_varadic ); - continue; - } - - type = parse_type(); - if ( type == Code::Invalid ) - { - Context.pop(); - return CodeInvalid; - } - - name = { nullptr, 0, TokType::Invalid, false }; - - if ( check( TokType::Identifier ) ) - { - name = currtok; - eat( TokType::Identifier ); - - if ( currtok.IsAssign ) - { - eat( TokType::Operator ); - - Token value_tok = currtok; - - if ( currtok.Type == TokType::Comma ) - { - log_failure( "Expected value after assignment operator\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - while ( left - && currtok.Type != TokType::Comma && currtok.Type != TokType::Capture_End - ) - { - value_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)value_tok.Text; - eat( currtok.Type ); - } - - value = untyped_str( value_tok ); - } - } - - CodeParam - param = (CodeParam) make_code(); - param->Type = Parameters; - - if ( name.Length > 0 ) - param->Name = get_cached_string( name ); - - param->ValueType = type; - - if ( value ) - param->Value = value; - - result.append( param ); - } - - if ( ! use_template_capture ) - eat( TokType::Capture_End ); - - else - { - if ( ! check( TokType::Operator) || currtok.Text[0] != '>' ) - { - log_failure("Expected '<' after 'template' keyword\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - eat( TokType::Operator ); - } + CodeComment + result = (CodeComment) make_code(); + result->Type = ECode::Comment; + result->Content = get_cached_string( currtok_noskip ); + result->Name = result->Content; + eat( TokType::Comment ); Context.pop(); return result; -# undef context -} - -// Function parsing is handled in multiple places because its initial signature is shared with variable parsing -internal inline -CodeFn parse_function_after_name( - ModuleFlag mflags - , CodeAttributes attributes - , CodeSpecifiers specifiers - , CodeType ret_type - , Parser::Token name -) -{ - using namespace Parser; - push_scope(); - - CodeParam params = parse_params(); - - while ( left && currtok.is_specifier() ) - { - if ( specifiers.ast == nullptr ) - { - specifiers = def_specifier( ESpecifier::to_type(currtok) ); - eat( currtok.Type ); - continue; - } - - specifiers.append( ESpecifier::to_type(currtok) ); - eat( currtok.Type ); - } - - CodeBody body = NoCode; - CodeComment inline_cmt = NoCode; - if ( check( TokType::BraceCurly_Open ) ) - { - body = parse_function_body(); - if ( body == Code::Invalid ) - { - Context.pop(); - return CodeInvalid; - } - } - else - { - Token stmt_end = currtok; - eat( TokType::Statement_End ); - - if ( currtok_noskip.Type && TokType::Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); - } - - using namespace ECode; - - - - String - name_stripped = String::make( GlobalAllocator, name ); - name_stripped.strip_space(); - - CodeFn - result = (CodeFn) make_code(); - result->Name = get_cached_string( name_stripped ); - result->ModuleFlags = mflags; - - if ( body ) - { - switch ( body->Type ) - { - case Function_Body: - case Untyped: - break; - - default: - { - log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", body.debug_str(), Context.to_string()); - Context.pop(); - return CodeInvalid; - } - } - - result->Type = Function; - result->Body = body; - } - else - { - result->Type = Function_Fwd; - } - - if ( specifiers ) - result->Specs = specifiers; - - result->ReturnType = ret_type; - - if ( params ) - result->Params = params; - - if ( inline_cmt ) - result->InlineCmt = inline_cmt; - - Context.pop(); - return result; -} - -internal -CodeOperator parse_operator_after_ret_type( - ModuleFlag mflags - , CodeAttributes attributes - , CodeSpecifiers specifiers - , CodeType ret_type -) -{ - using namespace Parser; - using namespace EOperator; - push_scope(); - - Token nspace = NullToken; - if ( check( TokType::Identifier ) ) - { - nspace = currtok; - while ( left && currtok.Type == TokType::Identifier ) - { - eat( TokType::Identifier ); - - if ( currtok.Type == TokType::Access_StaticSymbol ) - eat( TokType::Access_StaticSymbol ); - } - - nspace.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)nspace.Text; - } - - eat( TokType::Decl_Operator ); - - if ( ! left && currtok.Type != TokType::Operator - && currtok.Type != TokType::Star - && currtok.Type != TokType::Ampersand - && currtok.Type != TokType::Ampersand_DBL ) - { - log_failure( "Expected operator after 'operator' keyword\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - Context.Scope->Name = currtok; - - OperatorT op = Invalid; - switch ( currtok.Text[0] ) - { - case '+': - { - if ( currtok.Text[1] == '=' ) - op = Assign_Add; - - if ( currtok.Text[1] == '+' ) - op = Increment; - - else - op = Add; - } - break; - case '-': - { - if ( currtok.Text[1] == '>' ) - { - if ( currtok.Text[2] == '*' ) - op = MemberOfPointer; - - else - op = MemberOfPointer; - - break; - } - - if ( currtok.Text[1] == '=' ) - op = Assign_Subtract; - - else - op = Subtract; - } - break; - case '*': - { - if ( currtok.Text[1] == '=' ) - op = Assign_Multiply; - - else - { - Token& finder = prevtok; - while ( finder.Type != TokType::Decl_Operator ) - { - if ( finder.Type == TokType::Identifier) - { - op = Indirection; - break; - } - } - - if ( op == Invalid) - op = Multiply; - } - } - break; - case '/': - { - if ( currtok.Text[1] == '=' ) - op = Assign_Divide; - - else - op = Divide; - } - break; - case '%': - { - if ( currtok.Text[1] == '=' ) - op = Assign_Modulo; - - else - op = Modulo; - } - break; - case '&': - { - if ( currtok.Text[1] == '=' ) - op = Assign_BAnd; - - else if ( currtok.Text[1] == '&' ) - op = LAnd; - - else - { - - - if ( op == Invalid ) - op = BAnd; - } - } - break; - case '|': - { - if ( currtok.Text[1] == '=' ) - op = Assign_BOr; - - else if ( currtok.Text[1] == '|' ) - op = LOr; - - else - op = BOr; - } - break; - case '^': - { - if ( currtok.Text[1] == '=' ) - op = Assign_BXOr; - - else - op = BXOr; - } - break; - case '~': - { - op = BNot; - } - break; - case '!': - { - if ( currtok.Text[1] == '=' ) - op = LNot; - - else - op = UnaryNot; - } - break; - case '=': - { - if ( currtok.Text[1] == '=' ) - op = LEqual; - - else - op = Assign; - } - break; - case '<': - { - if ( currtok.Text[1] == '=' ) - op = LEqual; - - else if ( currtok.Text[1] == '<' ) - { - if ( currtok.Text[2] == '=' ) - op = Assign_LShift; - - else - op = LShift; - } - else - op = Lesser; - } - break; - case '>': - { - if ( currtok.Text[1] == '=' ) - op = GreaterEqual; - - else if ( currtok.Text[1] == '>' ) - { - if ( currtok.Text[2] == '=' ) - op = Assign_RShift; - - else - op = RShift; - } - else - op = Greater; - } - break; - case '(': - { - if ( currtok.Text[1] == ')' ) - op = FunctionCall; - - else - op = Invalid; - } - break; - case '[': - { - if ( currtok.Text[1] == ']' ) - op = Subscript; - - else - op = Invalid; - } - break; - default: - { - break; - } - } - - if ( op == Invalid ) - { - log_failure( "Invalid operator '%s'\n%s", currtok.Text, Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - eat( currtok.Type ); - - // Parse Params - CodeParam params = parse_params(); - - if ( params.ast == nullptr && op == EOperator::Multiply ) - op = MemberOfPointer; - - while ( left && currtok.is_specifier() ) - { - if ( specifiers.ast == nullptr ) - { - specifiers = def_specifier( ESpecifier::to_type(currtok) ); - eat( currtok.Type ); - continue; - } - - specifiers.append( ESpecifier::to_type(currtok) ); - eat( currtok.Type ); - } - - // Parse Body - CodeBody body = { nullptr }; - CodeComment inline_cmt = NoCode; - if ( check( TokType::BraceCurly_Open ) ) - { - body = parse_function_body(); - if ( body == Code::Invalid ) - { - Context.pop(); - return CodeInvalid; - } - } - else - { - Token stmt_end = currtok; - eat( TokType::Statement_End ); - - if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); - } - - // OpValidateResult check_result = operator__validate( op, params, ret_type, specifiers ); - CodeOperator result = def_operator( op, nspace, params, ret_type, body, specifiers, attributes, mflags ); - - if ( inline_cmt ) - result->InlineCmt = inline_cmt; - - Context.pop(); - return result; -} - -// Variable parsing is handled in multiple places because its initial signature is shared with function parsing -internal -CodeVar parse_variable_after_name( - ModuleFlag mflags - , CodeAttributes attributes - ,CodeSpecifiers specifiers - , CodeType type - , StrC name -) -{ - using namespace Parser; - push_scope(); - - Code array_expr = parse_array_decl(); - Code expr = { nullptr }; - Code bitfield_expr = { nullptr }; - - if ( currtok.IsAssign ) - { - eat( TokType::Operator ); - - Token expr_tok = currtok; - - if ( currtok.Type == TokType::Statement_End ) - { - log_failure( "Expected expression after assignment operator\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - while ( left && currtok.Type != TokType::Statement_End ) - { - eat( currtok.Type ); - } - - expr_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)expr_tok.Text - 1; - expr = untyped_str( expr_tok ); - } - - if ( currtok.Type == TokType::BraceCurly_Open ) - { - Token expr_tok = currtok; - - eat( TokType::BraceCurly_Open ); - - s32 level = 0; - while ( left && ( currtok.Type != TokType::BraceCurly_Close || level > 0 ) ) - { - if ( currtok.Type == TokType::BraceCurly_Open ) - level++; - - else if ( currtok.Type == TokType::BraceCurly_Close && level > 0 ) - level--; - - eat( currtok.Type ); - } - eat( TokType::BraceCurly_Close ); - - expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text; - expr = untyped_str( expr_tok ); - } - - if ( currtok.Type == TokType::Assign_Classifer ) - { - eat( TokType::Assign_Classifer ); - - Token expr_tok = currtok; - - if ( currtok.Type == TokType::Statement_End ) - { - log_failure( "Expected expression after bitfield \n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - while ( left && currtok.Type != TokType::Statement_End ) - { - eat( currtok.Type ); - } - - expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text; - bitfield_expr = untyped_str( expr_tok ); - } - - Token stmt_end = currtok; - eat( TokType::Statement_End ); - - // Check for inline comment : = ; // - CodeComment inline_cmt = NoCode; - if ( left - && ( currtok_noskip.Type == TokType::Comment ) - && currtok_noskip.Line == stmt_end.Line ) - { - inline_cmt = parse_comment(); - } - - using namespace ECode; - - CodeVar - result = (CodeVar) make_code(); - result->Type = Variable; - result->Name = get_cached_string( name ); - result->ModuleFlags = mflags; - - result->ValueType = type; - - if (array_expr ) - type->ArrExpr = array_expr; - - if ( bitfield_expr ) - result->BitfieldSize = bitfield_expr; - - if ( attributes ) - result->Attributes = attributes; - - if ( specifiers ) - result->Specs = specifiers; - - if ( expr ) - result->Value = expr; - - if ( inline_cmt ) - result->InlineCmt = inline_cmt; - - Context.pop(); - return result; -} - -internal inline -Code parse_simple_preprocess( Parser::TokType which ) -{ - using namespace Parser; - push_scope(); - - Token tok = currtok; - eat( which ); - - if ( currtok.Type == TokType::BraceCurly_Open ) - { - // Eat the block scope right after the macro. Were assuming the macro defines a function definition's signature - eat( TokType::BraceCurly_Open ); - - s32 level = 0; - while ( left && ( currtok.Type != TokType::BraceCurly_Close || level > 0 ) ) - { - if ( currtok.Type == TokType::BraceCurly_Open ) - level++; - - else if ( currtok.Type == TokType::BraceCurly_Close && level > 0 ) - level--; - - eat( currtok.Type ); - } - eat( TokType::BraceCurly_Close ); - - StrC prev_proc = Context.Scope->Prev->ProcName; - if ( str_compare( prev_proc.Ptr, "parse_typedef", prev_proc.Len ) != 0 ) - { - if ( check( TokType::Statement_End )) - { - Token stmt_end = currtok; - eat( TokType::Statement_End ); - - if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) - eat( TokType::Comment ); - } - } - - tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)tok.Text; - } - else - { - if ( str_compare( Context.Scope->Prev->ProcName.Ptr, "parse_typedef", Context.Scope->Prev->ProcName.Len ) != 0 ) - { - if ( check( TokType::Statement_End )) - { - Token stmt_end = currtok; - eat( TokType::Statement_End ); - - if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) - eat( TokType::Comment ); - } - } - - tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)tok.Text; - } - - char const* content = str_fmt_buf( "%.*s ", tok.Length, tok.Text ); - - Code result = untyped_str( to_str( content ) ); - Context.Scope->Name = tok; - - Context.pop(); - return result; -} - -internal -Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ) -{ - using namespace Parser; - push_scope(); - - Code result = CodeInvalid; - -#ifndef GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES - if ( currtok.Type == TokType::Preprocess_Macro ) - { - // Were dealing with a macro after attributes/specifiers. - result = parse_simple_preprocess( TokType::Preprocess_Macro ); - Context.pop(); - return result; - } -#endif - - CodeType type = parse_type(); - - if ( type == CodeInvalid ) - { - Context.pop(); - return CodeInvalid; - } - - bool found_operator = false; - s32 idx = Context.Tokens.Idx; - - for ( ; idx < Context.Tokens.Arr.num(); idx++ ) - { - Token tok = Context.Tokens[ idx ]; - - if ( tok.Type == TokType::Identifier ) - { - idx++; - tok = Context.Tokens[ idx ]; - if ( tok.Type == TokType::Access_StaticSymbol ) - continue; - - break; - } - - if ( tok.Type == TokType::Decl_Operator ) - found_operator = true; - - break; - } - - if ( found_operator ) - { - // Dealing with an operator overload - result = parse_operator_after_ret_type( ModuleFlag::None, attributes, specifiers, type ); - } - else - { - Token name = parse_identifier(); - Context.Scope->Name = name; - - if ( check( TokType::Capture_Start) ) - { - // Dealing with a function - result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, type, name ); - } - else - { - if ( expects_function ) - { - log_failure( "Expected function declaration (consteval was used)\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - // Dealing with a variable - result = parse_variable_after_name( ModuleFlag::None, attributes, specifiers, type, name ); - } - } - - Context.pop(); - return result; -} - -internal inline -Code parse_foward_or_definition( Parser::TokType which, bool is_inplace ) -{ - using namespace Parser; - - Code result = CodeInvalid; - - switch ( which ) - { - case TokType::Decl_Class: - result = parse_class( is_inplace ); - Context.pop(); - return result; - - case TokType::Decl_Enum: - result = parse_enum( is_inplace ); - Context.pop(); - return result; - - case TokType::Decl_Struct: - result = parse_struct( is_inplace ); - Context.pop(); - return result; - - case TokType::Decl_Union: - result = parse_union( is_inplace ); - Context.pop(); - return result; - - default: - log_failure( "Error, wrong token type given to parse_complicated_definition " - "(only supports class, enum, struct, union) \n%s" - , Context.to_string() ); - - Context.pop(); - return CodeInvalid; - } - - return CodeInvalid; } internal @@ -2772,7 +1513,7 @@ Code parse_complicated_definition( Parser::TokType which ) } internal neverinline -CodeBody parse_class_struct_body( Parser::TokType which, Parser::Token name = Parser::NullToken ) +CodeBody parse_class_struct_body( Parser::TokType which, Parser::Token name ) { using namespace Parser; using namespace ECode; @@ -3137,6 +1878,325 @@ Code parse_class_struct( Parser::TokType which, bool inplace_def = false ) return result; } +internal inline +CodeDefine parse_define() +{ + using namespace Parser; + push_scope(); + + eat( TokType::Preprocess_Define ); + + CodeDefine + define = (CodeDefine) make_code(); + define->Type = ECode::Preprocess_Define; + + if ( ! check( TokType::Identifier ) ) + { + log_failure( "Error, expected identifier after #define\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + Context.Scope->Name = currtok; + define->Name = get_cached_string( currtok ); + eat( TokType::Identifier ); + + if ( ! check( TokType::Preprocess_Content )) + { + log_failure( "Error, expected content after #define %s\n%s", define->Name, Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + if ( currtok.Length == 0 ) + { + define->Content = get_cached_string( currtok ); + eat( TokType::Preprocess_Content ); + + Context.pop(); + return define; + } + + String content = String::make_reserve( GlobalAllocator, currtok.Length ); + +#define cut_length ( scanner - currtok.Text - last_cut ) +#define cut_ptr ( currtok.Text + last_cut ) +#define pos ( sptr( scanner ) - sptr( currtok.Text ) ) + s32 tokleft = currtok.Length; + sptr last_cut = 0; + char const* scanner = currtok.Text; + + if ( scanner[0] == ' ' ) + { + ++ scanner; + -- tokleft; + last_cut = 1; + } + + while ( tokleft ) + { + if ( tokleft > 1 && char_is_space( scanner[0] ) && char_is_space( scanner[ 1 ] ) ) + { + content.append( cut_ptr, cut_length ); + do + { + ++ scanner; + -- tokleft; + } + while ( tokleft && char_is_space( scanner[0] ) ); + + last_cut = sptr( scanner ) - sptr( currtok.Text ); + + // Preserve only 1 space of formattting + if ( content.back() != ' ' ) + content.append( ' ' ); + continue; + } + + if ( scanner[0] == '\t' ) + { + if ( pos > last_cut ) + content.append( cut_ptr, cut_length ); + + // Replace with a space + if ( content.back() != ' ' ) + content.append( ' ' ); + + ++ scanner; + -- tokleft; + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + if ( tokleft > 1 && scanner[0] == '\r' && scanner[1] == '\n' ) + { + if ( pos > last_cut ) + content.append( cut_ptr, cut_length ); + + // Replace with a space + if ( content.back() != ' ' ) + content.append( ' ' ); + + scanner += 2; + tokleft -= 2; + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + if ( scanner[0] == '\n' ) + { + if ( pos > last_cut ) + content.append( cut_ptr, cut_length ); + + // Replace with a space + if ( content.back() != ' ' ) + content.append( ' ' ); + + ++ scanner; + -- tokleft; + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + if ( scanner[0] == '\\' ) + { + s32 amount_to_skip = 1; + if ( tokleft > 1 && scanner[1] == '\n' ) + { + amount_to_skip = 2; + } + else if ( tokleft > 2 && scanner[1] == '\r' && scanner[2] == '\n' ) + { + amount_to_skip = 3; + } + + if ( amount_to_skip > 1 ) + { + if ( pos == last_cut ) + { + // If the backslash is the first character on the line, then skip it + scanner += amount_to_skip; + tokleft -= amount_to_skip; + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + // We have content to add. + content.append( cut_ptr, pos - last_cut ); + + scanner += amount_to_skip; + tokleft -= amount_to_skip; + } + else + { + ++ scanner; + -- tokleft; + } + + last_cut = sptr( scanner ) - sptr( currtok.Text ); + continue; + } + + ++ scanner; + -- tokleft; + } + + if ( last_cut < currtok.Length ) + { + content.append( cut_ptr, currtok.Length - last_cut ); + } +#undef cut_ptr +#undef cut_length +#undef pos + + define->Content = get_cached_string( content ); + eat( TokType::Preprocess_Content ); + + Context.pop(); + return define; +} + +internal inline +Code parse_foward_or_definition( Parser::TokType which, bool is_inplace ) +{ + using namespace Parser; + + Code result = CodeInvalid; + + switch ( which ) + { + case TokType::Decl_Class: + result = parse_class( is_inplace ); + Context.pop(); + return result; + + case TokType::Decl_Enum: + result = parse_enum( is_inplace ); + Context.pop(); + return result; + + case TokType::Decl_Struct: + result = parse_struct( is_inplace ); + Context.pop(); + return result; + + case TokType::Decl_Union: + result = parse_union( is_inplace ); + Context.pop(); + return result; + + default: + log_failure( "Error, wrong token type given to parse_complicated_definition " + "(only supports class, enum, struct, union) \n%s" + , Context.to_string() ); + + Context.pop(); + return CodeInvalid; + } + + return CodeInvalid; +} + +// Function parsing is handled in multiple places because its initial signature is shared with variable parsing +internal inline +CodeFn parse_function_after_name( + ModuleFlag mflags + , CodeAttributes attributes + , CodeSpecifiers specifiers + , CodeType ret_type + , Parser::Token name +) +{ + using namespace Parser; + push_scope(); + + CodeParam params = parse_params(); + + while ( left && currtok.is_specifier() ) + { + if ( specifiers.ast == nullptr ) + { + specifiers = def_specifier( ESpecifier::to_type(currtok) ); + eat( currtok.Type ); + continue; + } + + specifiers.append( ESpecifier::to_type(currtok) ); + eat( currtok.Type ); + } + + CodeBody body = NoCode; + CodeComment inline_cmt = NoCode; + if ( check( TokType::BraceCurly_Open ) ) + { + body = parse_function_body(); + if ( body == Code::Invalid ) + { + Context.pop(); + return CodeInvalid; + } + } + else + { + Token stmt_end = currtok; + eat( TokType::Statement_End ); + + if ( currtok_noskip.Type && TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); + } + + using namespace ECode; + + + + String + name_stripped = String::make( GlobalAllocator, name ); + name_stripped.strip_space(); + + CodeFn + result = (CodeFn) make_code(); + result->Name = get_cached_string( name_stripped ); + result->ModuleFlags = mflags; + + if ( body ) + { + switch ( body->Type ) + { + case Function_Body: + case Untyped: + break; + + default: + { + log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", body.debug_str(), Context.to_string()); + Context.pop(); + return CodeInvalid; + } + } + + result->Type = Function; + result->Body = body; + } + else + { + result->Type = Function_Fwd; + } + + if ( specifiers ) + result->Specs = specifiers; + + result->ReturnType = ret_type; + + if ( params ) + result->Params = params; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; + + Context.pop(); + return result; +} + internal Code parse_function_body() { @@ -3444,6 +2504,985 @@ CodeBody parse_global_nspace( CodeT which ) return result; } +internal +Parser::Token parse_identifier( bool* possible_member_function ) +{ + using namespace Parser; + push_scope(); + + Token name = currtok; + Context.Scope->Name = name; + eat( TokType::Identifier ); + + parse_template_args( name ); + + while ( check( TokType::Access_StaticSymbol ) ) + { + eat( TokType::Access_StaticSymbol ); + + if ( left == 0 ) + { + log_failure( "Error, unexpected end of static symbol identifier\n%s", Context.to_string() ); + Context.pop(); + return { nullptr, 0, TokType::Invalid }; + } + + if ( currtok.Type == TokType::Operator && currtok.Text[0] == '*' && currtok.Length == 1 ) + { + if ( possible_member_function ) + *possible_member_function = true; + + else + { + log_failure( "Found a member function pointer identifier but the parsing context did not expect it\n%s", Context.to_string() ); + Context.pop(); + return { nullptr, 0, TokType::Invalid }; + } + } + + if ( currtok.Type != TokType::Identifier ) + { + log_failure( "Error, expected static symbol identifier, not %s\n%s", ETokType::to_str( currtok.Type ), Context.to_string() ); + Context.pop(); + return { nullptr, 0, TokType::Invalid }; + } + + name.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)name.Text; + eat( TokType::Identifier ); + + parse_template_args( name ); + } + + Context.pop(); + return name; +} + +internal +CodeInclude parse_include() +{ + using namespace Parser; + push_scope(); + + CodeInclude + include = (CodeInclude) make_code(); + include->Type = ECode::Preprocess_Include; + eat( TokType::Preprocess_Include ); + + if ( ! check( TokType::String )) + { + log_failure( "Error, expected include string after #include\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + Context.Scope->Name = currtok; + include->Content = get_cached_string( currtok ); + eat( TokType::String ); + + Context.pop(); + return include; +} + +internal +CodeOperator parse_operator_after_ret_type( + ModuleFlag mflags + , CodeAttributes attributes + , CodeSpecifiers specifiers + , CodeType ret_type +) +{ + using namespace Parser; + using namespace EOperator; + push_scope(); + + Token nspace = NullToken; + if ( check( TokType::Identifier ) ) + { + nspace = currtok; + while ( left && currtok.Type == TokType::Identifier ) + { + eat( TokType::Identifier ); + + if ( currtok.Type == TokType::Access_StaticSymbol ) + eat( TokType::Access_StaticSymbol ); + } + + nspace.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)nspace.Text; + } + + eat( TokType::Decl_Operator ); + + if ( ! left && currtok.Type != TokType::Operator + && currtok.Type != TokType::Star + && currtok.Type != TokType::Ampersand + && currtok.Type != TokType::Ampersand_DBL ) + { + log_failure( "Expected operator after 'operator' keyword\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + Context.Scope->Name = currtok; + + OperatorT op = Invalid; + switch ( currtok.Text[0] ) + { + case '+': + { + if ( currtok.Text[1] == '=' ) + op = Assign_Add; + + if ( currtok.Text[1] == '+' ) + op = Increment; + + else + op = Add; + } + break; + case '-': + { + if ( currtok.Text[1] == '>' ) + { + if ( currtok.Text[2] == '*' ) + op = MemberOfPointer; + + else + op = MemberOfPointer; + + break; + } + + if ( currtok.Text[1] == '=' ) + op = Assign_Subtract; + + else + op = Subtract; + } + break; + case '*': + { + if ( currtok.Text[1] == '=' ) + op = Assign_Multiply; + + else + { + Token& finder = prevtok; + while ( finder.Type != TokType::Decl_Operator ) + { + if ( finder.Type == TokType::Identifier) + { + op = Indirection; + break; + } + } + + if ( op == Invalid) + op = Multiply; + } + } + break; + case '/': + { + if ( currtok.Text[1] == '=' ) + op = Assign_Divide; + + else + op = Divide; + } + break; + case '%': + { + if ( currtok.Text[1] == '=' ) + op = Assign_Modulo; + + else + op = Modulo; + } + break; + case '&': + { + if ( currtok.Text[1] == '=' ) + op = Assign_BAnd; + + else if ( currtok.Text[1] == '&' ) + op = LAnd; + + else + { + + + if ( op == Invalid ) + op = BAnd; + } + } + break; + case '|': + { + if ( currtok.Text[1] == '=' ) + op = Assign_BOr; + + else if ( currtok.Text[1] == '|' ) + op = LOr; + + else + op = BOr; + } + break; + case '^': + { + if ( currtok.Text[1] == '=' ) + op = Assign_BXOr; + + else + op = BXOr; + } + break; + case '~': + { + op = BNot; + } + break; + case '!': + { + if ( currtok.Text[1] == '=' ) + op = LNot; + + else + op = UnaryNot; + } + break; + case '=': + { + if ( currtok.Text[1] == '=' ) + op = LEqual; + + else + op = Assign; + } + break; + case '<': + { + if ( currtok.Text[1] == '=' ) + op = LEqual; + + else if ( currtok.Text[1] == '<' ) + { + if ( currtok.Text[2] == '=' ) + op = Assign_LShift; + + else + op = LShift; + } + else + op = Lesser; + } + break; + case '>': + { + if ( currtok.Text[1] == '=' ) + op = GreaterEqual; + + else if ( currtok.Text[1] == '>' ) + { + if ( currtok.Text[2] == '=' ) + op = Assign_RShift; + + else + op = RShift; + } + else + op = Greater; + } + break; + case '(': + { + if ( currtok.Text[1] == ')' ) + op = FunctionCall; + + else + op = Invalid; + } + break; + case '[': + { + if ( currtok.Text[1] == ']' ) + op = Subscript; + + else + op = Invalid; + } + break; + default: + { + break; + } + } + + if ( op == Invalid ) + { + log_failure( "Invalid operator '%s'\n%s", currtok.Text, Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + eat( currtok.Type ); + + // Parse Params + CodeParam params = parse_params(); + + if ( params.ast == nullptr && op == EOperator::Multiply ) + op = MemberOfPointer; + + while ( left && currtok.is_specifier() ) + { + if ( specifiers.ast == nullptr ) + { + specifiers = def_specifier( ESpecifier::to_type(currtok) ); + eat( currtok.Type ); + continue; + } + + specifiers.append( ESpecifier::to_type(currtok) ); + eat( currtok.Type ); + } + + // Parse Body + CodeBody body = { nullptr }; + CodeComment inline_cmt = NoCode; + if ( check( TokType::BraceCurly_Open ) ) + { + body = parse_function_body(); + if ( body == Code::Invalid ) + { + Context.pop(); + return CodeInvalid; + } + } + else + { + Token stmt_end = currtok; + eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + inline_cmt = parse_comment(); + } + + // OpValidateResult check_result = operator__validate( op, params, ret_type, specifiers ); + CodeOperator result = def_operator( op, nspace, params, ret_type, body, specifiers, attributes, mflags ); + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; + + Context.pop(); + return result; +} + +internal +Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ) +{ + using namespace Parser; + push_scope(); + + Code result = CodeInvalid; + +#ifndef GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES + if ( currtok.Type == TokType::Preprocess_Macro ) + { + // Were dealing with a macro after attributes/specifiers. + result = parse_simple_preprocess( TokType::Preprocess_Macro ); + Context.pop(); + return result; + } +#endif + + CodeType type = parse_type(); + + if ( type == CodeInvalid ) + { + Context.pop(); + return CodeInvalid; + } + + bool found_operator = false; + s32 idx = Context.Tokens.Idx; + + for ( ; idx < Context.Tokens.Arr.num(); idx++ ) + { + Token tok = Context.Tokens[ idx ]; + + if ( tok.Type == TokType::Identifier ) + { + idx++; + tok = Context.Tokens[ idx ]; + if ( tok.Type == TokType::Access_StaticSymbol ) + continue; + + break; + } + + if ( tok.Type == TokType::Decl_Operator ) + found_operator = true; + + break; + } + + if ( found_operator ) + { + // Dealing with an operator overload + result = parse_operator_after_ret_type( ModuleFlag::None, attributes, specifiers, type ); + } + else + { + Token name = parse_identifier(); + Context.Scope->Name = name; + + if ( check( TokType::Capture_Start) ) + { + // Dealing with a function + result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, type, name ); + } + else + { + if ( expects_function ) + { + log_failure( "Expected function declaration (consteval was used)\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + // Dealing with a variable + result = parse_variable_after_name( ModuleFlag::None, attributes, specifiers, type, name ); + } + } + + Context.pop(); + return result; +} + +internal +CodePragma parse_pragma() +{ + using namespace Parser; + push_scope(); + + CodePragma + pragma = (CodePragma) make_code(); + pragma->Type = ECode::Preprocess_Pragma; + eat( TokType::Preprocess_Pragma ); + + if ( ! check( TokType::Preprocess_Content )) + { + log_failure( "Error, expected content after #pragma\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + Context.Scope->Name = currtok; + + String + content_stripped = String::make( GlobalAllocator, currtok ); + content_stripped.strip_space(); + + pragma->Content = get_cached_string( content_stripped ); + eat( TokType::Preprocess_Content ); + + Context.pop(); + return pragma; +} + +internal inline +CodeParam parse_params( bool use_template_capture ) +{ + using namespace Parser; + using namespace ECode; + push_scope(); + + if ( ! use_template_capture ) + eat( TokType::Capture_Start ); + + else + { + if ( check ( TokType::Operator ) && currtok.Text[0] == '<' ) + eat( TokType::Operator ); + } + + if ( ! use_template_capture && check(TokType::Capture_End) ) + { + eat( TokType::Capture_End ); + Context.pop(); + return { nullptr }; + } + + CodeType type = { nullptr }; + Code value = { nullptr }; + + if ( check( TokType::Varadic_Argument) ) + { + eat( TokType::Varadic_Argument ); + + Context.pop(); + return param_varadic; + } + + type = parse_type(); + if ( type == Code::Invalid ) + { + Context.pop(); + return CodeInvalid; + } + + Token name = NullToken; + + if ( check( TokType::Identifier ) ) + { + name = currtok; + eat( TokType::Identifier ); + + if ( currtok.IsAssign ) + { + eat( TokType::Operator ); + + Token value_tok = currtok; + + if ( currtok.Type == TokType::Comma ) + { + log_failure( "Expected value after assignment operator\n%s.", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + while ( left + && currtok.Type != TokType::Comma + && currtok.Type != TokType::Capture_End + ) + { + value_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)value_tok.Text; + eat( currtok.Type ); + } + + value = untyped_str( value_tok ); + } + } + + CodeParam + result = (CodeParam) make_code(); + result->Type = Parameters; + + if ( name.Length > 0 ) + result->Name = get_cached_string( name ); + + result->ValueType = type; + + if ( value ) + result->Value = value; + + result->NumEntries++; + + while ( left + && use_template_capture ? + currtok.Type != TokType::Operator && currtok.Text[0] != '>' + : currtok.Type != TokType::Capture_End ) + { + eat( TokType::Comma ); + + Code type = { nullptr }; + Code value = { nullptr }; + + if ( check( TokType::Varadic_Argument) ) + { + eat( TokType::Varadic_Argument ); + result.append( param_varadic ); + continue; + } + + type = parse_type(); + if ( type == Code::Invalid ) + { + Context.pop(); + return CodeInvalid; + } + + name = { nullptr, 0, TokType::Invalid, false }; + + if ( check( TokType::Identifier ) ) + { + name = currtok; + eat( TokType::Identifier ); + + if ( currtok.IsAssign ) + { + eat( TokType::Operator ); + + Token value_tok = currtok; + + if ( currtok.Type == TokType::Comma ) + { + log_failure( "Expected value after assignment operator\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + while ( left + && currtok.Type != TokType::Comma && currtok.Type != TokType::Capture_End + ) + { + value_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)value_tok.Text; + eat( currtok.Type ); + } + + value = untyped_str( value_tok ); + } + } + + CodeParam + param = (CodeParam) make_code(); + param->Type = Parameters; + + if ( name.Length > 0 ) + param->Name = get_cached_string( name ); + + param->ValueType = type; + + if ( value ) + param->Value = value; + + result.append( param ); + } + + if ( ! use_template_capture ) + eat( TokType::Capture_End ); + + else + { + if ( ! check( TokType::Operator) || currtok.Text[0] != '>' ) + { + log_failure("Expected '<' after 'template' keyword\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + eat( TokType::Operator ); + } + + Context.pop(); + return result; +# undef context +} + +internal +CodePreprocessCond parse_preprocess_cond() +{ + using namespace Parser; + push_scope(); + + if ( ! currtok.is_preprocess_cond() ) + { + log_failure( "Error, expected preprocess conditional\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + CodePreprocessCond + cond = (CodePreprocessCond) make_code(); + cond->Type = scast(CodeT, currtok.Type - (ETokType::Preprocess_If - ECode::Preprocess_If) ); + eat( currtok.Type ); + + if ( ! check( TokType::Preprocess_Content )) + { + log_failure( "Error, expected content after #define\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + Context.Scope->Name = currtok; + cond->Content = get_cached_string( currtok ); + eat( TokType::Preprocess_Content ); + + Context.pop(); + return cond; +} + +internal inline +Code parse_simple_preprocess( Parser::TokType which ) +{ + using namespace Parser; + push_scope(); + + Token tok = currtok; + eat( which ); + + if ( currtok.Type == TokType::BraceCurly_Open ) + { + // Eat the block scope right after the macro. Were assuming the macro defines a function definition's signature + eat( TokType::BraceCurly_Open ); + + s32 level = 0; + while ( left && ( currtok.Type != TokType::BraceCurly_Close || level > 0 ) ) + { + if ( currtok.Type == TokType::BraceCurly_Open ) + level++; + + else if ( currtok.Type == TokType::BraceCurly_Close && level > 0 ) + level--; + + eat( currtok.Type ); + } + eat( TokType::BraceCurly_Close ); + + StrC prev_proc = Context.Scope->Prev->ProcName; + if ( str_compare( prev_proc.Ptr, "parse_typedef", prev_proc.Len ) != 0 ) + { + if ( check( TokType::Statement_End )) + { + Token stmt_end = currtok; + eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + eat( TokType::Comment ); + } + } + + tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)tok.Text; + } + else + { + if ( str_compare( Context.Scope->Prev->ProcName.Ptr, "parse_typedef", Context.Scope->Prev->ProcName.Len ) != 0 ) + { + if ( check( TokType::Statement_End )) + { + Token stmt_end = currtok; + eat( TokType::Statement_End ); + + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + eat( TokType::Comment ); + } + } + + tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)tok.Text; + } + + char const* content = str_fmt_buf( "%.*s ", tok.Length, tok.Text ); + + Code result = untyped_str( to_str( content ) ); + Context.Scope->Name = tok; + + Context.pop(); + return result; +} + +internal +Code parse_static_assert() +{ + using namespace Parser; + push_scope(); + + Code + assert = make_code(); + assert->Type = ECode::Untyped; + + Token content = currtok; + + Context.Scope->Name = content; + + eat( TokType::StaticAssert ); + eat( TokType::Capture_Start ); + + s32 level = 0; + while ( left && ( currtok.Type != TokType::Capture_End || level > 0 ) ) + { + if ( currtok.Type == TokType::Capture_Start ) + level++; + else if ( currtok.Type == TokType::Capture_End ) + level--; + + eat( currtok.Type ); + } + eat( TokType::Capture_End ); + eat( TokType::Statement_End ); + + content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text; + + + String + content_stripped = String::make( GlobalAllocator, content ); + content_stripped.strip_space(); + + char const* result = str_fmt_buf( "%.*s\n", content.Length, content.Text ); + if ( content_stripped ) + { + result = str_fmt_buf( "%S\n", content_stripped ); + } + + assert->Content = get_cached_string( to_str( result ) ); + assert->Name = assert->Content; + + Context.pop(); + return assert; +} + +/* + This a brute-froce make all the arguments part of the token provided. + Can have in-place function signatures, regular identifiers, in-place typenames, compile-time expressions, parameter-pack expansion, etc. + This means that validation can only go so far, and so if there is any different in formatting + passed the basic stripping supported it report a soft failure. +*/ +internal inline +void parse_template_args( Parser::Token& token ) +{ + using namespace Parser; + + if ( currtok.Type == TokType::Operator && currtok.Text[0] == '<' && currtok.Length == 1 ) + { + eat( TokType::Operator ); + + s32 level = 0; + while ( left && ( currtok.Text[0] != '>' || level > 0 )) + { + if ( currtok.Text[0] == '<' ) + level++; + + if ( currtok.Text[0] == '>' ) + level--; + + eat( currtok.Type ); + } + + eat( TokType::Operator ); + + // Extend length of name to last token + token.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)token.Text; + } +} + +// Variable parsing is handled in multiple places because its initial signature is shared with function parsing +internal +CodeVar parse_variable_after_name( + ModuleFlag mflags + , CodeAttributes attributes + ,CodeSpecifiers specifiers + , CodeType type + , StrC name +) +{ + using namespace Parser; + push_scope(); + + Code array_expr = parse_array_decl(); + Code expr = { nullptr }; + Code bitfield_expr = { nullptr }; + + if ( currtok.IsAssign ) + { + eat( TokType::Operator ); + + Token expr_tok = currtok; + + if ( currtok.Type == TokType::Statement_End ) + { + log_failure( "Expected expression after assignment operator\n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + while ( left && currtok.Type != TokType::Statement_End ) + { + eat( currtok.Type ); + } + + expr_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)expr_tok.Text - 1; + expr = untyped_str( expr_tok ); + } + + if ( currtok.Type == TokType::BraceCurly_Open ) + { + Token expr_tok = currtok; + + eat( TokType::BraceCurly_Open ); + + s32 level = 0; + while ( left && ( currtok.Type != TokType::BraceCurly_Close || level > 0 ) ) + { + if ( currtok.Type == TokType::BraceCurly_Open ) + level++; + + else if ( currtok.Type == TokType::BraceCurly_Close && level > 0 ) + level--; + + eat( currtok.Type ); + } + eat( TokType::BraceCurly_Close ); + + expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text; + expr = untyped_str( expr_tok ); + } + + if ( currtok.Type == TokType::Assign_Classifer ) + { + eat( TokType::Assign_Classifer ); + + Token expr_tok = currtok; + + if ( currtok.Type == TokType::Statement_End ) + { + log_failure( "Expected expression after bitfield \n%s", Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + while ( left && currtok.Type != TokType::Statement_End ) + { + eat( currtok.Type ); + } + + expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text; + bitfield_expr = untyped_str( expr_tok ); + } + + Token stmt_end = currtok; + eat( TokType::Statement_End ); + + // Check for inline comment : = ; // + CodeComment inline_cmt = NoCode; + if ( left + && ( currtok_noskip.Type == TokType::Comment ) + && currtok_noskip.Line == stmt_end.Line ) + { + inline_cmt = parse_comment(); + } + + using namespace ECode; + + CodeVar + result = (CodeVar) make_code(); + result->Type = Variable; + result->Name = get_cached_string( name ); + result->ModuleFlags = mflags; + + result->ValueType = type; + + if (array_expr ) + type->ArrExpr = array_expr; + + if ( bitfield_expr ) + result->BitfieldSize = bitfield_expr; + + if ( attributes ) + result->Attributes = attributes; + + if ( specifiers ) + result->Specs = specifiers; + + if ( expr ) + result->Value = expr; + + if ( inline_cmt ) + result->InlineCmt = inline_cmt; + + Context.pop(); + return result; +} + +// Publically Exposed Interface + internal CodeClass parse_class( bool inplace_def ) { @@ -4505,11 +4544,21 @@ CodeTemplate parse_template( StrC def ) return parse_template(); } -// This is a bit of a mess, but it works -// Parsing typename is arguably one of the worst aspects of C/C++. -// This is an effort to parse it without a full blown or half-blown compliant parser. +/* + This is a mess, but it works + Parsing typename is arguably one of the worst aspects of C/C++. + This is an effort to parse it without a full blown or half-blown compliant parser. + + Recursive function typenames are not supported, if they are used expect it to serailize just fine, but validation with AST::is_equal + will not be possible if two ASTs share the same definiton but the formatting is slightly different: + AST_1->Name: (* A ( int (*) (short a,unsigned b,long c) ) ) + AST_2->Name: (* A ( int(*)(short a, unsigned b, long c) ) ) + + The excess whitespace can be stripped however, because there is no semantic awareness within the first capture group, + it cannot entirely remove the whitespaceto remove insignificant whitespace. +*/ internal -CodeType parse_type( bool* is_function ) +CodeType parse_type( bool* typedef_is_function ) { using namespace Parser; push_scope(); @@ -4519,13 +4568,12 @@ CodeType parse_type( bool* is_function ) SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; - Token name = { nullptr, 0, TokType::Invalid }; - Token brute_sig = { currtok.Text, 0, TokType::Invalid }; + Token name = { nullptr, 0, TokType::Invalid }; // Attributes are assumed to be before the type signature CodeAttributes attributes = parse_attributes(); - // Deal with specifiers before the type signature + // Prefix specifiers while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -4563,6 +4611,31 @@ CodeType parse_type( bool* is_function ) Context.Scope->Name = name; } +#if 0 + else if ( currtok.Type == TokType::DeclType ) + { + // Will have a capture and its own parsing rules, were going to just shove everything in a string. + name = currtok; + eat( TokType::DeclType ); + + eat( TokType::Capture_Start ); + while ( left && currtok.Type != TokType::Capture_End ) + { + if ( currtok.Type == TokType::Capture_Start ) + level++; + + if ( currtok.Type == TokType::Capture_End ) + level--; + + eat( currtok.Type ); + } + eat( TokType::Capture_End ); + + name.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)name.Text; + Context.Scope->Name = name; + } +#endif + // Check if native type keywords are used, eat them for the signature. else if ( currtok.Type >= TokType::Type_Unsigned && currtok.Type <= TokType::Type_MS_W64 ) { @@ -4591,6 +4664,7 @@ CodeType parse_type( bool* is_function ) } } + // Suffix specifiers for typename. while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -4610,43 +4684,201 @@ CodeType parse_type( bool* is_function ) eat( currtok.Type ); } +#ifdef GEN_USE_NEW_TYPENAME_PARSING + if ( NumSpecifiers ) + { + specifiers = def_specifiers( NumSpecifiers, specs_found ); + NumSpecifiers = 0; + } +#endif + // For function type signatures CodeType return_type = NoCode; CodeParam params = NoCode; - bool is_first_capture = true; - while ( check( TokType::Capture_Start ) && context_tok.Type != TokType::Decl_Operator ) +#ifdef GEN_USE_NEW_TYPENAME_PARSING + CodeParam params_nested = NoCode; +#endif + + bool is_function_typename = false; + Token* last_capture = nullptr; { - // Brute force capture the entire thing - // Function typedefs are complicated and there are not worth dealing with for validation at this point... - eat( TokType::Capture_Start ); + Token* scanner = Context.Tokens.Arr + Context.Tokens.Idx; - if ( is_function && is_first_capture ) + // An identifier being within a typename's signature only occurs if were parsing a typename for a typedef. + if ( typedef_is_function && scanner->Type == TokType::Identifier ) { - while ( check( TokType::Star )) - { - eat( TokType::Star ); - } + is_function_typename = true; + ++ scanner; + } + is_function_typename = scanner->Type == TokType::Capture_Start; - * is_function = true; - eat( TokType::Identifier ); + Token* first_capture = scanner; + if ( is_function_typename ) + { + // Go to the end of the signature + while ( scanner->Type != TokType::Statement_End && scanner->TokType::BraceCurly_Open ) + ++ scanner; + + // Go back to the first capture start found + while ( scanner->Type != TokType::Capture_Start ) + -- scanner; + + last_capture = scanner; } - s32 level = 0; - while ( left && ( currtok.Type != TokType::Capture_End || level > 0 )) + bool is_for_opcast = str_compare( Context.Scope->Prev->ProcName, "parse_operator_cast" ) == 0; + if ( is_for_opcast && is_function_typename && last_capture ) { - if ( currtok.Type == TokType::Capture_Start ) - level++; + // If we're parsing for an operator cast, having one capture start is not enough + // we need to make sure that the capture is not for the cast definition. + is_function_typename = false; - if ( currtok.Type == TokType::Capture_End ) - level--; + if ( last_capture == first_capture ) + { + // The capture start in question is the first capture start, this is not a function typename. + is_function_typename = false; + } + } + } + if ( is_function_typename ) + { + // We're dealing with a function typename. + // By this point, decltype should have been taken care of for return type, along with any all its specifiers + + // The previous information with exception to attributes will be considered the return type. + return_type = (CodeType) make_code(); + return_type->Type = ECode::Typename; + + // String + // name_stripped = String::make( GlobalAllocator, name ); + // name_stripped.strip_space(); + return_type->Name = get_cached_string( name ); + + #ifdef GEN_USE_NEW_TYPENAME_PARSING + if ( specifiers ) + { + return_type->Specs = specifiers; + specifiers = nullptr; + } + + #else + if ( NumSpecifiers ) + return_type->Specs = def_specifiers( NumSpecifiers, (SpecifierT*)specs_found ); + + // Reset specifiers, the function itself will have its own suffix specifiers possibly. + NumSpecifiers = 0; + #endif + + name = { nullptr, 0, TokType::Invalid }; + + // The next token can either be a capture for the identifier or it could be the identifier exposed. + if ( ! check( TokType::Capture_Start ) ) + { + // Started with an identifier immeidately, which means its of the format: ; + name = parse_identifier(); + } + + // If the next token is a capture start and is not the last capture, then we're dealing with function typename whoose identifier is within the capture. + else if ( ( Context.Tokens.Arr + Context.Tokens.Idx ) != last_capture ) + { + // WIP : Possible alternative without much pain... + // If this were to be parsed properly... + // Eat Capture Start + // Deal with possible binding specifiers (*, &, &&) and modifiers on those bindings (const, volatile) + // Parse specifiers for the typename with an optional identifier, + // we can shove these specific specifiers into a specs, and then leave the suffix ones for a separate member of the AST. + // Parse immeidate capture which would be with parse_params() + // Eat Capture End + #ifdef GEN_USE_NEW_TYPENAME_PARSING + eat( TokType::Capture_Start ); + + // Binding specifiers + while ( left && currtok.is_specifier() ) + { + SpecifierT spec = ESpecifier::to_type( currtok ); + + if ( spec != ESpecifier::Ptr + && spec != ESpecifier::Ref + && spec != ESpecifier::RValue ) + { + log_failure( "Error, invalid specifier used in type definition: %s\n%s", currtok.Text, Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + specs_found[NumSpecifiers] = spec; + NumSpecifiers++; + eat( currtok.Type ); + } + + if ( NumSpecifiers ) + { + specifiers = def_specifiers( NumSpecifiers, specs_found ); + } + NumSpecifiers = 0; + + if ( check( TokType::Identifier )) + name = parse_identifier(); + + // Immeidate parameters + + if ( check( TokType::Capture_Start )) + params_nested = parse_params(); + + #else + // Starting immediatley with a capture, most likely declaring a typename for a member function pointer. + // Everything within this capture will just be shoved into the name field including the capture tokens themselves. + name = currtok; + + eat( TokType::Capture_Start ); + s32 level = 0; + while ( left && ( currtok.Type != TokType::Capture_End || level > 0 )) + { + if ( currtok.Type == TokType::Capture_Start ) + level++; + + if ( currtok.Type == TokType::Capture_End ) + level--; + + eat( currtok.Type ); + } + eat( TokType::Capture_End ); + + name.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)name.Text; + #endif + } + + // Were now dealing with the parameters of the function + params = parse_params(); + + // Look for suffix specifiers for the function + while ( left && currtok.is_specifier() ) + { + SpecifierT spec = ESpecifier::to_type( currtok ); + + if ( spec != ESpecifier::Const + // && spec != ESpecifier::NoExcept + && spec != ESpecifier::RValue ) + { + log_failure( "Error, invalid specifier used in type definition: %s\n%s", currtok.Text, Context.to_string() ); + Context.pop(); + return CodeInvalid; + } + + specs_found[NumSpecifiers] = spec; + NumSpecifiers++; eat( currtok.Type ); } - eat( TokType::Capture_End ); - brute_sig.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)brute_sig.Text; - is_first_capture = false; + #ifdef GEN_USE_NEW_TYPENAME_PARSING + if ( NumSpecifiers ) + { + func_suffix_specs = def_specifiers( NumSpecifiers, specs_found ); + NumSpecifiers = 0; + } + #endif } bool is_param_pack = false; @@ -4662,35 +4894,52 @@ CodeType parse_type( bool* is_function ) result = (CodeType) make_code(); result->Type = Typename; - if ( brute_sig.Length > 0 ) - { - // Bruteforce all tokens together. - name = brute_sig; - } - else - { - if (NumSpecifiers) - { - Code specifiers = def_specifiers( NumSpecifiers, (SpecifierT*)specs_found ); - result->Specs = specifiers; - } - } + String + name_stripped = String::make( GlobalAllocator, name ); + name_stripped.strip_space(); - // This is bad we cannot strip the name if it contains the full function signature's parameters, parameters at minimum must be separate. - String name_stripped = String::make( GlobalAllocator, name ); - // name_stripped.strip_space(); +#ifdef GEN_USE_NEW_TYPENAME_PARSING + if ( params_nested ) + { + name_stripped.append( params_nested->to_string() ); + } +#endif result->Name = get_cached_string( name_stripped ); if ( attributes ) result->Attributes = attributes; +#ifdef GEN_USE_NEW_TYPENAME_PARSING + if ( specifiers ) + { + result->Specs = specifiers; + } + + if ( func_suffix_specs ) + { + result->FuncSuffixSpecs = func_suffix_specs; + } +#else + if (NumSpecifiers) + { + Code specifiers = def_specifiers( NumSpecifiers, (SpecifierT*)specs_found ); + result->Specs = specifiers; + } +#endif + if ( is_param_pack ) result->IsParamPack = true; + // These following are only populated if its a function typename if ( return_type ) + { result->ReturnType = return_type; + if ( typedef_is_function ) + *typedef_is_function = true; + } + if ( params ) result->Params = params; diff --git a/project/enums/ESpecifier.csv b/project/enums/ESpecifier.csv index d7c1181..11bc2d4 100644 --- a/project/enums/ESpecifier.csv +++ b/project/enums/ESpecifier.csv @@ -21,5 +21,6 @@ Volatile, volatile Virtual, virtual Const, const Final, final +NoExceptions, noexcept Override, override Pure, = 0 From 2200bcde9aa2e725012a37d7d8445da0274ff35c Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 5 Sep 2023 13:36:59 -0400 Subject: [PATCH 20/26] Improved singleheader test Need to make the debug_str provided by the AST type aware to provide as much contextual information as possible (finally got to this point with validation). Singleheader test now directly calls clang-format to cleanup the reconstructed copy of the singleheader. Its needed to remove any sort of formatting discrepancies found by the parser since its sensistive to that for new-lines, etc. --- project/components/interface.parsing.cpp | 6 ++--- scripts/build.ci.ps1 | 3 ++- test/validate.singleheader.cpp | 33 +++++++++++++++++------- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 1ba4f21..9c89c97 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -4554,8 +4554,7 @@ CodeTemplate parse_template( StrC def ) AST_1->Name: (* A ( int (*) (short a,unsigned b,long c) ) ) AST_2->Name: (* A ( int(*)(short a, unsigned b, long c) ) ) - The excess whitespace can be stripped however, because there is no semantic awareness within the first capture group, - it cannot entirely remove the whitespaceto remove insignificant whitespace. + The excess whitespace cannot be stripped however, because there is no semantic awareness within the first capture group. */ internal CodeType parse_type( bool* typedef_is_function ) @@ -4894,9 +4893,10 @@ CodeType parse_type( bool* typedef_is_function ) result = (CodeType) make_code(); result->Type = Typename; + // Need to wait until were using the new parsing method to do this. String name_stripped = String::make( GlobalAllocator, name ); - name_stripped.strip_space(); + // name_stripped.strip_space(); #ifdef GEN_USE_NEW_TYPENAME_PARSING if ( params_nested ) diff --git a/scripts/build.ci.ps1 b/scripts/build.ci.ps1 index 6acb0fc..e1a4db6 100644 --- a/scripts/build.ci.ps1 +++ b/scripts/build.ci.ps1 @@ -396,6 +396,7 @@ if ( $test ) build-simple $includes $unit $executable Push-Location $path_test + Write-Host $path_test if ( Test-Path( $executable ) ) { write-host "`nRunning test generator" $time_taken = Measure-Command { & $executable @@ -458,7 +459,7 @@ if ( $singleheader -and (Test-Path (Join-Path $path_singleheader "gen/gen.hpp")) format-cpp $path_gen $include $exclude } -if ( $test ) +if ( $test -and $false ) { $path_gen = join-path $path_test gen $include = @( diff --git a/test/validate.singleheader.cpp b/test/validate.singleheader.cpp index fbfef34..b9d252b 100644 --- a/test/validate.singleheader.cpp +++ b/test/validate.singleheader.cpp @@ -7,34 +7,49 @@ #include "gen.scanner.hpp" using namespace gen; +#ifdef GEN_SYSTEM_WINDOWS + #include +#endif + void check_singleheader_ast() { - #define project_dir "../" + #define root_dir "../" gen::init(); log_fmt("\ncheck_singleheader_ast:\n"); - FileContents file = file_read_contents( GlobalAllocator, true, project_dir "singleheader/gen/gen.hpp" ); + FileContents file = file_read_contents( GlobalAllocator, true, root_dir "singleheader/gen/gen.hpp" ); u64 time_start = time_rel_ms(); CodeBody ast = parse_global_body( { file.size, (char const*)file.data } ); - log_fmt("\nAst generated. Time taken: %llu ms\n", time_rel_ms() - time_start); log_fmt("\nSerializng ast:\n"); time_start = time_rel_ms(); - Builder builder = Builder::open( "gen/singleheader_copy.gen.hpp" ); builder.print( ast ); builder.write(); - log_fmt("Serialized. Time taken: %llu ms\n", time_rel_ms() - time_start); + // Need to execute clang format on the generated file to get it to match the original. + #define script_path root_dir "scripts/" + #define clang_format "clang-format " + #define cf_format_inplace "-i " + #define cf_style "-style=file:" "C:/projects/gencpp/scripts/.clang-format " + #define cf_verbose "-verbose " + + log_fmt("\nRunning clang-format on generated file:\n"); + system( clang_format cf_format_inplace cf_style cf_verbose "gen/singleheader_copy.gen.hpp" ); + log_fmt("clang-format finished reformatting.\n"); + #undef script_path + #undef cf_cmd + #undef cf_format_inplace + #undef cf_style + #undef cf_verbse + FileContents file_gen = file_read_contents( GlobalAllocator, true, "gen/singleheader_copy.gen.hpp" ); - log_fmt("\nReconstructing from generated file:\n"); - time_start = time_rel_ms(); - CodeBody ast_gen = parse_global_body( { file_gen.size, (char const*)file_gen.data } ); - + time_start = time_rel_ms(); + CodeBody ast_gen = parse_global_body( { file_gen.size, (char const*)file_gen.data } ); log_fmt("\nAst generated. Time taken: %llu ms\n\n", time_rel_ms() - time_start); time_start = time_rel_ms(); From f1fb75cc1ce927710a7dd4d40772143f4ab3c89a Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 6 Sep 2023 00:30:34 -0400 Subject: [PATCH 21/26] Progress on strip_formatting function, support for multi-dimentional array variables and typenames. strip_formatting suffers from some edge failure with what looks to be escaped character literals (not entirely sure). I've decided to not remove formatting from unvalidated function bodies since I plan to support parsing its content properly. However expression values for a statement will fail to have their formatting removed with this. Since I don't plan to parse those anytime soon, I'll have to fix any edge cases for those at least.. --- .vscode/settings.json | 4 +- project/components/ast.cpp | 110 +++--- project/components/interface.parsing.cpp | 459 +++++++++++++++-------- 3 files changed, 345 insertions(+), 228 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 0db6b4d..4d693b5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -26,7 +26,9 @@ "ratio": "cpp", "xstring": "cpp", "functional": "cpp", - "vector": "cpp" + "vector": "cpp", + "list": "cpp", + "xhash": "cpp" }, "C_Cpp.intelliSenseEngineFallback": "disabled", "mesonbuild.configureOnOpen": true, diff --git a/project/components/ast.cpp b/project/components/ast.cpp index b83a6ed..b95e64a 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -781,7 +781,14 @@ String AST::to_string() if ( UnderlyingType->Type == Typename && UnderlyingType->ArrExpr ) { - result.append_fmt( "[%S];", UnderlyingType->ArrExpr->to_string() ); + result.append_fmt( "[ %S ];", UnderlyingType->ArrExpr->to_string() ); + + AST* next_arr_expr = UnderlyingType->ArrExpr->Next; + while ( next_arr_expr ) + { + result.append_fmt( "[ %S ];", next_arr_expr->to_string() ); + next_arr_expr = next_arr_expr->Next; + } } else { @@ -885,7 +892,16 @@ String AST::to_string() result.append_fmt( "using %S = %S", Name, UnderlyingType->to_string() ); if ( UnderlyingType->ArrExpr ) - result.append_fmt( "[%S]", UnderlyingType->ArrExpr->to_string() ); + { + result.append_fmt( "[ %S ]", UnderlyingType->ArrExpr->to_string() ); + + AST* next_arr_expr = UnderlyingType->ArrExpr->Next; + while ( next_arr_expr ) + { + result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); + next_arr_expr = next_arr_expr->Next; + } + } result.append( ";" ); } @@ -922,7 +938,16 @@ String AST::to_string() result.append_fmt( "%S %S", ValueType->to_string(), Name ); if ( ValueType->ArrExpr ) - result.append_fmt( "[%S]", ValueType->ArrExpr->to_string() ); + { + result.append_fmt( "[ %S ]", ValueType->ArrExpr->to_string() ); + + AST* next_arr_expr = ValueType->ArrExpr->Next; + while ( next_arr_expr ) + { + result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); + next_arr_expr = next_arr_expr->Next; + } + } if ( BitfieldSize ) result.append_fmt( " : %S", BitfieldSize->to_string() ); @@ -941,11 +966,22 @@ String AST::to_string() if ( BitfieldSize ) result.append_fmt( "%S %S : %S;", ValueType->to_string(), Name, BitfieldSize->to_string() ); - else if ( UnderlyingType->ArrExpr ) - result.append_fmt( "%S %S[%S];", UnderlyingType->to_string(), Name, UnderlyingType->ArrExpr->to_string() ); + else if ( ValueType->ArrExpr ) + { + result.append_fmt( "%S %S[ %S ]", ValueType->to_string(), Name, ValueType->ArrExpr->to_string() ); + + AST* next_arr_expr = ValueType->ArrExpr->Next; + while ( next_arr_expr ) + { + result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); + next_arr_expr = next_arr_expr->Next; + } + + result.append( ";" ); + } else - result.append_fmt( "%S %S;", UnderlyingType->to_string(), Name ); + result.append_fmt( "%S %S;", ValueType->to_string(), Name ); if ( InlineCmt ) result.append_fmt(" %S", InlineCmt->Content); @@ -1096,15 +1132,15 @@ bool AST::is_equal( AST* other ) // Comments are not validated. case Comment: - // return true; + return true; case Execution: case PlatformAttributes: case Untyped: { check_member_content( Content ); - // check_member_ast( Prev ); - // check_member_ast( Next ); + + return true; } case Class_Fwd: @@ -1114,8 +1150,6 @@ bool AST::is_equal( AST* other ) check_member_ast( ParentType ); check_member_val( ParentAccess ); check_member_ast( Attributes ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1129,8 +1163,6 @@ bool AST::is_equal( AST* other ) check_member_val( ParentAccess ); check_member_ast( Attributes ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1140,8 +1172,6 @@ bool AST::is_equal( AST* other ) check_member_ast( InitializerList ); check_member_ast( Params ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1150,8 +1180,6 @@ bool AST::is_equal( AST* other ) { check_member_ast( InitializerList ); check_member_ast( Params ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1160,8 +1188,6 @@ bool AST::is_equal( AST* other ) { check_member_ast( Specs ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1169,8 +1195,6 @@ bool AST::is_equal( AST* other ) case Destructor_Fwd: { check_member_ast( Specs ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1183,8 +1207,6 @@ bool AST::is_equal( AST* other ) check_member_ast( Attributes ); check_member_ast( UnderlyingType ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1196,8 +1218,6 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Attributes ); check_member_ast( UnderlyingType ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1206,8 +1226,6 @@ bool AST::is_equal( AST* other ) { check_member_str( Name ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1216,8 +1234,6 @@ bool AST::is_equal( AST* other ) { check_member_str( Name ); check_member_ast( Declaration ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1231,8 +1247,6 @@ bool AST::is_equal( AST* other ) check_member_ast( Specs ); check_member_ast( Params ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1245,8 +1259,6 @@ bool AST::is_equal( AST* other ) check_member_ast( Attributes ); check_member_ast( Specs ); check_member_ast( Params ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1255,8 +1267,6 @@ bool AST::is_equal( AST* other ) { check_member_val( ModuleFlags ); check_member_str( Name ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1266,8 +1276,6 @@ bool AST::is_equal( AST* other ) check_member_val( ModuleFlags ); check_member_str( Name ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1282,8 +1290,6 @@ bool AST::is_equal( AST* other ) check_member_ast( Specs ); check_member_ast( Params ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1297,8 +1303,6 @@ bool AST::is_equal( AST* other ) check_member_ast( Attributes ); check_member_ast( Specs ); check_member_ast( Params ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1309,8 +1313,6 @@ bool AST::is_equal( AST* other ) check_member_ast( Specs ); check_member_ast( ValueType ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1320,8 +1322,6 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Specs ); check_member_ast( ValueType ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1415,8 +1415,6 @@ bool AST::is_equal( AST* other ) { check_member_str( Name ); check_member_content( Content ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1427,8 +1425,6 @@ bool AST::is_equal( AST* other ) case Preprocess_ElIf: { check_member_content( Content ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1437,8 +1433,6 @@ bool AST::is_equal( AST* other ) case Preprocess_Pragma: { check_member_content( Content ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1451,8 +1445,6 @@ bool AST::is_equal( AST* other ) { check_member_val( ArrSpecs[ idx ] ); } - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1462,8 +1454,6 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Params ); check_member_ast( Declaration ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1475,8 +1465,6 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Specs ); check_member_ast( UnderlyingType ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1486,8 +1474,6 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Specs ); check_member_ast( ArrExpr ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1498,8 +1484,6 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( Attributes ); check_member_ast( Body ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1511,8 +1495,6 @@ bool AST::is_equal( AST* other ) check_member_str( Name ); check_member_ast( UnderlyingType ); check_member_ast( Attributes ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } @@ -1526,8 +1508,6 @@ bool AST::is_equal( AST* other ) check_member_ast( Value ); check_member_ast( Attributes ); check_member_ast( Specs ); - // check_member_ast( Prev ); - // check_member_ast( Next ); return true; } diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 9c89c97..bdafc22 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -184,7 +184,7 @@ namespace Parser result.append_fmt("\tScope : %s\n", line ); line.free(); - sptr dist = (sptr)last_valid.Text - (sptr)scope_start.Text + 2; + sptr dist = (sptr)last_valid.Text - (sptr)scope_start.Text + 2; sptr length_from_err = dist; String line_from_err = String::make( GlobalAllocator, { length_from_err, last_valid.Text } ); @@ -396,10 +396,10 @@ namespace Parser s32 within_char = false; while ( left ) { - if ( current == '"' ) + if ( current == '"' && ! within_char ) within_string ^= true; - if ( current == '\'' ) + if ( current == '\'' && ! within_string ) within_char ^= true; if ( current == '\\' && ! within_string && ! within_char ) @@ -1252,6 +1252,276 @@ constexpr bool inplace_def = true; // Internal parsing functions +constexpr bool strip_formatting_dont_preserve_newlines = false; +// constexpr bool strip_formatting_for_preprocess_define = true; +/* + This function was an attempt at stripping formatting from any c++ code. + It has edge case failures that prevent it from being used in function bodies. +*/ +String strip_formatting( StrC raw_text, bool preserve_newlines = true ) +//, bool for_preprocess_define = false ) +{ + String content = String::make_reserve( GlobalAllocator, raw_text.Len ); + + if ( raw_text.Len == 0 ) + return content; + +#define cut_length ( scanner - raw_text.Ptr - last_cut ) +#define cut_ptr ( raw_text.Ptr + last_cut ) +#define pos ( sptr( scanner ) - sptr( raw_text.Ptr ) ) +#define move_fwd() do { scanner++; tokleft--; } while(0) + + s32 tokleft = raw_text.Len; + sptr last_cut = 0; + char const* scanner = raw_text.Ptr; + + if ( scanner[0] == ' ' ) + { + move_fwd(); + last_cut = 1; + } + + bool within_string = false; + bool within_char = false; + bool must_keep_newline = false; + while ( tokleft ) + { + // Skip over the content of string literals + if ( scanner[0] == '"' ) + { + // content.append( cut_ptr, cut_length ); + // last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + + move_fwd(); + + while ( tokleft && ( scanner[0] != '"' || *( scanner - 1 ) == '\\' ) ) + { + if ( scanner[0] == '\\' && tokleft > 1 ) + { + scanner += 2; + tokleft -= 2; + } + else + { + move_fwd(); + } + } + + // Skip the closing " + if ( tokleft ) + move_fwd(); + + content.append( cut_ptr, cut_length ); + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + // Skip over the content of character literals + if ( scanner[0] == '\'' ) + { + // content.append( cut_ptr, cut_length ); + // last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + + move_fwd(); + + while ( tokleft + && ( scanner[0] != '\'' + || ( *(scanner -1 ) == '\\' ) + ) ) + { + move_fwd(); + } + + // Skip the closing ' + if ( tokleft ) + move_fwd(); + + content.append( cut_ptr, cut_length ); + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + // Most likely removing as its only useful for funciton bodies. + #if 0 + // Preprocessed lines + if ( ! for_preprocess_define && scanner[0] == '#') + { + must_keep_newline = true; + + if ( content.back() != '\n' ) + content.append( '\n' ); + + move_fwd(); + continue; + } + #endif + + // Block comments + if ( tokleft > 1 && scanner[0] == '/' && scanner[1] == '*' ) + { + while ( tokleft > 1 && !(scanner[0] == '*' && scanner[1] == '/') ) + move_fwd(); + + scanner += 2; + tokleft -= 2; + + content.append( cut_ptr, cut_length ); + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + // Line comments + if ( tokleft > 1 && scanner[0] == '/' && scanner[1] == '/' ) + { + must_keep_newline = true; + + // if ( content.back() != '\n' ) + // content.append( '\n' ); + + scanner += 2; + tokleft -= 2; + + while ( tokleft && scanner[ 0 ] != '\n' ) + move_fwd(); + + if (tokleft) + move_fwd(); + + content.append( cut_ptr, cut_length ); + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + // Tabs + if (scanner[0] == '\t') + { + if (pos > last_cut) + content.append(cut_ptr, cut_length); + + if ( content.back() != ' ' ) + content.append(' '); + + move_fwd(); + last_cut = sptr(scanner) - sptr(raw_text.Ptr); + continue; + } + + if ( tokleft > 1 && scanner[0] == '\r' && scanner[1] == '\n' ) + { + if ( must_keep_newline || preserve_newlines ) + { + must_keep_newline = false; + + scanner += 2; + tokleft -= 2; + + content.append( cut_ptr, cut_length ); + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + if ( pos > last_cut ) + content.append( cut_ptr, cut_length ); + + // Replace with a space + if ( content.back() != ' ' ) + content.append( ' ' ); + + scanner += 2; + tokleft -= 2; + + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + if ( scanner[0] == '\n' ) + { + if ( must_keep_newline || preserve_newlines ) + { + must_keep_newline = false; + + move_fwd(); + + content.append( cut_ptr, cut_length ); + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + if ( pos > last_cut ) + content.append( cut_ptr, cut_length ); + + // Replace with a space + if ( content.back() != ' ' ) + content.append( ' ' ); + + move_fwd(); + + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + // Escaped newlines + if ( scanner[0] == '\\' ) + { + content.append( cut_ptr, cut_length ); + + s32 amount_to_skip = 1; + if ( tokleft > 1 && scanner[1] == '\n' ) + { + amount_to_skip = 2; + } + else if ( tokleft > 2 && scanner[1] == '\r' && scanner[2] == '\n' ) + { + amount_to_skip = 3; + } + + if ( amount_to_skip > 1 && pos == last_cut ) + { + scanner += amount_to_skip; + tokleft -= amount_to_skip; + } + else + move_fwd(); + + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + continue; + } + + // Consectuive spaces + if ( tokleft > 1 && char_is_space( scanner[0] ) && char_is_space( scanner[ 1 ] ) ) + { + content.append( cut_ptr, cut_length ); + do + { + move_fwd(); + } + while ( tokleft && char_is_space( scanner[0] ) ); + + last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); + + // Preserve only 1 space of formattting + if ( content.back() != ' ' ) + content.append( ' ' ); + + continue; + } + + move_fwd(); + } + + if ( last_cut < raw_text.Len ) + { + content.append( cut_ptr, raw_text.Len - last_cut ); + } + +#undef cut_ptr +#undef cut_length +#undef pos +#undef move_fwd + + return content; +} + internal Code parse_array_decl() { @@ -1280,7 +1550,7 @@ Code parse_array_decl() if ( currtok.Type == TokType::BraceSquare_Close ) { - log_failure( "Error, empty array expression in typedef definition\n%s", Context.to_string() ); + log_failure( "Error, empty array expression in definition\n%s", Context.to_string() ); Context.pop(); return CodeInvalid; } @@ -1311,6 +1581,15 @@ Code parse_array_decl() } eat( TokType::BraceSquare_Close ); + + // Its a multi-dimensional array + if ( check( TokType::BraceSquare_Open )) + { + Code adjacent_arr_expr = parse_array_decl(); + + array_expr->Next = adjacent_arr_expr.ast; + } + Context.pop(); return array_expr; } @@ -1384,9 +1663,7 @@ CodeAttributes parse_attributes() StrC attribute_txt = { len, start.Text }; Context.pop(); - String - name_stripped = String::make( GlobalAllocator, attribute_txt ); - name_stripped.strip_space(); + String name_stripped = strip_formatting( attribute_txt, strip_formatting_dont_preserve_newlines ); Code result = make_code(); @@ -1917,139 +2194,7 @@ CodeDefine parse_define() return define; } - String content = String::make_reserve( GlobalAllocator, currtok.Length ); - -#define cut_length ( scanner - currtok.Text - last_cut ) -#define cut_ptr ( currtok.Text + last_cut ) -#define pos ( sptr( scanner ) - sptr( currtok.Text ) ) - s32 tokleft = currtok.Length; - sptr last_cut = 0; - char const* scanner = currtok.Text; - - if ( scanner[0] == ' ' ) - { - ++ scanner; - -- tokleft; - last_cut = 1; - } - - while ( tokleft ) - { - if ( tokleft > 1 && char_is_space( scanner[0] ) && char_is_space( scanner[ 1 ] ) ) - { - content.append( cut_ptr, cut_length ); - do - { - ++ scanner; - -- tokleft; - } - while ( tokleft && char_is_space( scanner[0] ) ); - - last_cut = sptr( scanner ) - sptr( currtok.Text ); - - // Preserve only 1 space of formattting - if ( content.back() != ' ' ) - content.append( ' ' ); - continue; - } - - if ( scanner[0] == '\t' ) - { - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - // Replace with a space - if ( content.back() != ' ' ) - content.append( ' ' ); - - ++ scanner; - -- tokleft; - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - if ( tokleft > 1 && scanner[0] == '\r' && scanner[1] == '\n' ) - { - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - // Replace with a space - if ( content.back() != ' ' ) - content.append( ' ' ); - - scanner += 2; - tokleft -= 2; - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - if ( scanner[0] == '\n' ) - { - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - // Replace with a space - if ( content.back() != ' ' ) - content.append( ' ' ); - - ++ scanner; - -- tokleft; - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - if ( scanner[0] == '\\' ) - { - s32 amount_to_skip = 1; - if ( tokleft > 1 && scanner[1] == '\n' ) - { - amount_to_skip = 2; - } - else if ( tokleft > 2 && scanner[1] == '\r' && scanner[2] == '\n' ) - { - amount_to_skip = 3; - } - - if ( amount_to_skip > 1 ) - { - if ( pos == last_cut ) - { - // If the backslash is the first character on the line, then skip it - scanner += amount_to_skip; - tokleft -= amount_to_skip; - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - // We have content to add. - content.append( cut_ptr, pos - last_cut ); - - scanner += amount_to_skip; - tokleft -= amount_to_skip; - } - else - { - ++ scanner; - -- tokleft; - } - - last_cut = sptr( scanner ) - sptr( currtok.Text ); - continue; - } - - ++ scanner; - -- tokleft; - } - - if ( last_cut < currtok.Length ) - { - content.append( cut_ptr, currtok.Length - last_cut ); - } -#undef cut_ptr -#undef cut_length -#undef pos - - define->Content = get_cached_string( content ); + define->Content = get_cached_string( strip_formatting( currtok, strip_formatting_dont_preserve_newlines ) ); eat( TokType::Preprocess_Content ); Context.pop(); @@ -2230,7 +2375,13 @@ Code parse_function_body() if ( len > 0 ) { + // #define GEN_STRIP_FUNCTION_BODY_FORMATTING + #ifdef GEN_STRIP_FUNCTION_BODY_FORMATTING + String content = strip_formatting( { len, start.Text }, strip_formatting_dont_preserve_newlines ); + result.append( def_execution( content ) ); + #else result.append( def_execution( { len, start.Text } ) ); + #endif } eat( TokType::BraceCurly_Close ); @@ -2979,11 +3130,7 @@ CodePragma parse_pragma() Context.Scope->Name = currtok; - String - content_stripped = String::make( GlobalAllocator, currtok ); - content_stripped.strip_space(); - - pragma->Content = get_cached_string( content_stripped ); + pragma->Content = get_cached_string( currtok ); eat( TokType::Preprocess_Content ); Context.pop(); @@ -3060,7 +3207,7 @@ CodeParam parse_params( bool use_template_capture ) eat( currtok.Type ); } - value = untyped_str( value_tok ); + value = untyped_str( strip_formatting( value_tok, strip_formatting_dont_preserve_newlines ) ); } } @@ -3130,7 +3277,7 @@ CodeParam parse_params( bool use_template_capture ) eat( currtok.Type ); } - value = untyped_str( value_tok ); + value = untyped_str( strip_formatting( value_tok, strip_formatting_dont_preserve_newlines ) ); } } @@ -3300,19 +3447,7 @@ Code parse_static_assert() eat( TokType::Statement_End ); content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text; - - - String - content_stripped = String::make( GlobalAllocator, content ); - content_stripped.strip_space(); - - char const* result = str_fmt_buf( "%.*s\n", content.Length, content.Text ); - if ( content_stripped ) - { - result = str_fmt_buf( "%S\n", content_stripped ); - } - - assert->Content = get_cached_string( to_str( result ) ); + assert->Content = get_cached_string( content ); assert->Name = assert->Content; Context.pop(); @@ -4894,8 +5029,8 @@ CodeType parse_type( bool* typedef_is_function ) result->Type = Typename; // Need to wait until were using the new parsing method to do this. - String - name_stripped = String::make( GlobalAllocator, name ); + String name_stripped = strip_formatting( name, strip_formatting_dont_preserve_newlines ); + // name_stripped.strip_space(); #ifdef GEN_USE_NEW_TYPENAME_PARSING From 2bfbef1d0cdddd2b6a9e042332468e9f09c4fa54 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 6 Sep 2023 02:09:26 -0400 Subject: [PATCH 22/26] strip_formatting : Remove code thats no longer needed. Its everyting related to stripping a function body. --- project/components/interface.parsing.cpp | 32 +----------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index bdafc22..3381076 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -1253,7 +1253,6 @@ constexpr bool inplace_def = true; // Internal parsing functions constexpr bool strip_formatting_dont_preserve_newlines = false; -// constexpr bool strip_formatting_for_preprocess_define = true; /* This function was an attempt at stripping formatting from any c++ code. It has edge case failures that prevent it from being used in function bodies. @@ -1289,9 +1288,6 @@ String strip_formatting( StrC raw_text, bool preserve_newlines = true ) // Skip over the content of string literals if ( scanner[0] == '"' ) { - // content.append( cut_ptr, cut_length ); - // last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - move_fwd(); while ( tokleft && ( scanner[0] != '"' || *( scanner - 1 ) == '\\' ) ) @@ -1319,9 +1315,6 @@ String strip_formatting( StrC raw_text, bool preserve_newlines = true ) // Skip over the content of character literals if ( scanner[0] == '\'' ) { - // content.append( cut_ptr, cut_length ); - // last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - move_fwd(); while ( tokleft @@ -1341,21 +1334,6 @@ String strip_formatting( StrC raw_text, bool preserve_newlines = true ) continue; } - // Most likely removing as its only useful for funciton bodies. - #if 0 - // Preprocessed lines - if ( ! for_preprocess_define && scanner[0] == '#') - { - must_keep_newline = true; - - if ( content.back() != '\n' ) - content.append( '\n' ); - - move_fwd(); - continue; - } - #endif - // Block comments if ( tokleft > 1 && scanner[0] == '/' && scanner[1] == '*' ) { @@ -1375,9 +1353,6 @@ String strip_formatting( StrC raw_text, bool preserve_newlines = true ) { must_keep_newline = true; - // if ( content.back() != '\n' ) - // content.append( '\n' ); - scanner += 2; tokleft -= 2; @@ -2355,6 +2330,7 @@ Code parse_function_body() result = (CodeBody) make_code(); result->Type = Function_Body; + // TODO : Support actual parsing of function body Token start = currtok; s32 level = 0; @@ -2375,13 +2351,7 @@ Code parse_function_body() if ( len > 0 ) { - // #define GEN_STRIP_FUNCTION_BODY_FORMATTING - #ifdef GEN_STRIP_FUNCTION_BODY_FORMATTING - String content = strip_formatting( { len, start.Text }, strip_formatting_dont_preserve_newlines ); - result.append( def_execution( content ) ); - #else result.append( def_execution( { len, start.Text } ) ); - #endif } eat( TokType::BraceCurly_Close ); From d606c790ca958393d27af09b350205b1c893f600 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Wed, 6 Sep 2023 03:06:30 -0400 Subject: [PATCH 23/26] Added a new AST member `NextVar` to be used for comma separated variable support. Interface adding has been adjusted to use ParentType->Next. Using the AST_Class->Next was bad since that breaks the linked list a body AST would have used for traversal. --- project/components/ast.cpp | 51 +++++++++++++++++++++--- project/components/ast.hpp | 42 +++++++++---------- project/components/ast_types.hpp | 22 ++++++++-- project/components/inlines.hpp | 35 +++++++++++----- project/components/interface.parsing.cpp | 1 + 5 files changed, 111 insertions(+), 40 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index b95e64a..837da3f 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -103,7 +103,7 @@ String AST::to_string() result.append_fmt( "%S : %s %S", Name, access_level, ParentType ); - CodeType interface = Next->cast< CodeType >(); + CodeType interface = ParentType->Next->cast< CodeType >(); if ( interface ) result.append( "\n" ); @@ -703,7 +703,7 @@ String AST::to_string() result.append_fmt( "%S : %s %S", Name, access_level, ParentType ); - CodeType interface = Next->cast< CodeType >(); + CodeType interface = ParentType->Next->cast< CodeType >(); if ( interface ) result.append( "\n" ); @@ -924,6 +924,35 @@ String AST::to_string() case Variable: { + if ( Parent && Parent->Type == Variable ) + { + // Its a comma-separated variable ( a NextVar ) + + if ( Specs ) + result.append_fmt( "%S ", Specs->to_string() ); + + result.append( Name ); + + if ( ArrExpr ) + { + result.append_fmt( "[ %S ]", ArrExpr->to_string() ); + + AST* next_arr_expr = ArrExpr->Next; + while ( next_arr_expr ) + { + result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); + next_arr_expr = next_arr_expr->Next; + } + } + + if ( Value ) + result.append_fmt( " = %S", Value->to_string() ); + + // Keep the chain going... + if ( NextVar ) + result.append_fmt( ", %S", NextVar->to_string() ); + } + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) result.append( "export " ); @@ -955,6 +984,9 @@ String AST::to_string() if ( Value ) result.append_fmt( " = %S", Value->to_string() ); + if ( NextVar ) + result.append_fmt( ", %S", NextVar->to_string() ); + if ( InlineCmt ) result.append_fmt("; %S", InlineCmt->Content); else @@ -964,7 +996,7 @@ String AST::to_string() } if ( BitfieldSize ) - result.append_fmt( "%S %S : %S;", ValueType->to_string(), Name, BitfieldSize->to_string() ); + result.append_fmt( "%S %S : %S", ValueType->to_string(), Name, BitfieldSize->to_string() ); else if ( ValueType->ArrExpr ) { @@ -976,12 +1008,18 @@ String AST::to_string() result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); next_arr_expr = next_arr_expr->Next; } - - result.append( ";" ); } else - result.append_fmt( "%S %S;", ValueType->to_string(), Name ); + result.append_fmt( "%S %S", ValueType->to_string(), Name ); + + if ( Value ) + result.append_fmt( " = %S", Value->to_string() ); + + if ( NextVar ) + result.append_fmt( ", %S", NextVar->to_string() ); + + result.append( ";" ); if ( InlineCmt ) result.append_fmt(" %S", InlineCmt->Content); @@ -1508,6 +1546,7 @@ bool AST::is_equal( AST* other ) check_member_ast( Value ); check_member_ast( Attributes ); check_member_ast( Specs ); + check_member_ast( NextVar ); return true; } diff --git a/project/components/ast.hpp b/project/components/ast.hpp index 13f0604..c30901f 100644 --- a/project/components/ast.hpp +++ b/project/components/ast.hpp @@ -229,22 +229,19 @@ struct AST union { struct { - union { - AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable - AST* SpecsFuncSuffix; // Only used with typenames, to store the function suffix if typename is function signature. - }; + AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable AST* Specs; // Destructor, Function, Operator, Typename, Variable union { AST* InitializerList; // Constructor AST* ParentType; // Class, Struct, ParentType->Next has a possible list of interfaces. - AST* ReturnType; // Function, Operator + AST* ReturnType; // Function, Operator, Typename AST* UnderlyingType; // Enum, Typedef AST* ValueType; // Parameter, Variable }; union { - AST* BitfieldSize; // Varaiable (Class/Struct Data Member) - AST* Params; // Constructor, Function, Operator, Template + AST* BitfieldSize; // Variable (Class/Struct Data Member) + AST* Params; // Constructor, Function, Operator, Template, Typename }; union { AST* ArrExpr; // Typename @@ -252,6 +249,10 @@ struct AST AST* Declaration; // Friend, Template AST* Value; // Parameter, Variable }; + union { + AST* NextVar; // Variable; Possible way to handle comma separated variables declarations. ( , NextVar->Specs NextVar->Name NextVar->ArrExpr = NextVar->Value ) + AST* SpecsFuncSuffix; // Only used with typenames, to store the function suffix if typename is function signature. + }; }; StringCached Content; // Attributes, Comment, Execution, Include SpecifierT ArrSpecs[AST::ArrSpecs_Cap]; // Specifiers @@ -284,28 +285,29 @@ struct AST_POD union { struct { - union { - AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable - AST* SpecsFuncSuffix; // Only used with typenames, to store the function suffix if typename is function signature. - }; - AST* Attributes; // Class, Enum, Function, Struct, Typename, Union, Using, Variable - AST* Specs; // Function, Operator, Typename, Variable + AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable + AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable + AST* Specs; // Destructor, Function, Operator, Typename, Variable union { AST* InitializerList; // Constructor AST* ParentType; // Class, Struct, ParentType->Next has a possible list of interfaces. - AST* ReturnType; // Function, Operator + AST* ReturnType; // Function, Operator, Typename AST* UnderlyingType; // Enum, Typedef AST* ValueType; // Parameter, Variable }; union { - AST* BitfieldSize; // Varaiable (Class/Struct Data Member) - AST* Params; // Function, Operator, Template + AST* BitfieldSize; // Variable (Class/Struct Data Member) + AST* Params; // Constructor, Function, Operator, Template, Typename }; union { - AST* ArrExpr; // Type Symbol - AST* Body; // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union - AST* Declaration; // Friend, Template - AST* Value; // Parameter, Variable + AST* ArrExpr; // Typename + AST* Body; // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union + AST* Declaration; // Friend, Template + AST* Value; // Parameter, Variable + }; + union { + AST* NextVar; // Variable; Possible way to handle comma separated variables declarations. ( , NextVar->Specs NextVar->Name NextVar->ArrExpr = NextVar->Value ) + AST* SpecsFuncSuffix; // Only used with typenames, to store the function suffix if typename is function signature. }; }; StringCached Content; // Attributes, Comment, Execution, Include diff --git a/project/components/ast_types.hpp b/project/components/ast_types.hpp index 754f63c..03ed5d0 100644 --- a/project/components/ast_types.hpp +++ b/project/components/ast_types.hpp @@ -68,6 +68,7 @@ struct AST_Class CodeType ParentType; char _PAD_PARAMS_[ sizeof(AST*) ]; CodeBody Body; + char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; }; }; CodeType Last; @@ -93,6 +94,7 @@ struct AST_Constructor Code InitializerList; CodeParam Params; Code Body; + char _PAD_PROPERTIES_2_ [ sizeof(AST*) * 2 ]; }; }; Code Prev; @@ -132,6 +134,7 @@ struct AST_Destructor CodeSpecifiers Specs; char _PAD_PROPERTIES_2_ [ sizeof(AST*) * 2 ]; Code Body; + char _PAD_PROPERTIES_3_ [ sizeof(AST*) ]; }; }; Code Prev; @@ -156,6 +159,7 @@ struct AST_Enum CodeType UnderlyingType; char _PAD_PARAMS_[ sizeof(AST*) ]; CodeBody Body; + char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; }; }; Code Prev; @@ -193,6 +197,7 @@ struct AST_Extern { char _PAD_PROPERTIES_[ sizeof(AST*) * 5 ]; CodeBody Body; + char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; }; }; Code Prev; @@ -230,6 +235,7 @@ struct AST_Friend CodeComment InlineCmt; char _PAD_PROPERTIES_[ sizeof(AST*) * 4 ]; Code Declaration; + char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; }; }; Code Prev; @@ -254,6 +260,7 @@ struct AST_Fn CodeType ReturnType; CodeParam Params; CodeBody Body; + char _PAD_PROPERTIES_ [ sizeof(AST*) ]; }; }; Code Prev; @@ -288,6 +295,7 @@ struct AST_NS struct { char _PAD_PROPERTIES_[ sizeof(AST*) * 5 ]; CodeBody Body; + char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; }; }; Code Prev; @@ -313,6 +321,7 @@ struct AST_Operator CodeType ReturnType; CodeParam Params; CodeBody Body; + char _PAD_PROPERTIES_ [ sizeof(AST*) ]; }; }; Code Prev; @@ -338,6 +347,7 @@ struct AST_OpCast CodeType ValueType; char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; CodeBody Body; + char _PAD_PROPERTIES_3_[ sizeof(AST*) ]; }; }; Code Prev; @@ -360,6 +370,7 @@ struct AST_Param CodeType ValueType; char _PAD_PROPERTIES_[ sizeof(AST*) ]; Code Value; + char _PAD_PROPERTIES_3_[ sizeof(AST*) ]; }; }; CodeParam Last; @@ -431,6 +442,7 @@ struct AST_Struct CodeType ParentType; char _PAD_PARAMS_[ sizeof(AST*) ]; CodeBody Body; + char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; }; }; CodeType Last; @@ -453,6 +465,7 @@ struct AST_Template char _PAD_PROPERTIES_[ sizeof(AST*) * 4 ]; CodeParam Params; Code Declaration; + char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; }; }; Code Prev; @@ -472,12 +485,13 @@ struct AST_Type char _PAD_[ sizeof(SpecifierT) * AST::ArrSpecs_Cap ]; struct { - CodeSpecifiers SpecsFuncSuffix; // Only used for function signatures + char _PAD_INLINE_CMT_[ sizeof(AST*) ]; CodeAttributes Attributes; CodeSpecifiers Specs; CodeType ReturnType; // Only used for function signatures CodeParam Params; // Only used for function signatures Code ArrExpr; + CodeSpecifiers SpecsFuncSuffix; // Only used for function signatures }; }; Code Prev; @@ -500,7 +514,7 @@ struct AST_Typedef CodeComment InlineCmt; char _PAD_PROPERTIES_[ sizeof(AST*) * 2 ]; Code UnderlyingType; - char _PAD_PROPERTIES_2_[ sizeof(AST*) * 2 ]; + char _PAD_PROPERTIES_2_[ sizeof(AST*) * 3 ]; }; }; Code Prev; @@ -524,6 +538,7 @@ struct AST_Union CodeAttributes Attributes; char _PAD_PROPERTIES_[ sizeof(AST*) * 3 ]; CodeBody Body; + char _PAD_PROPERTIES_2_[ sizeof(AST*) ]; }; }; Code Prev; @@ -547,7 +562,7 @@ struct AST_Using CodeAttributes Attributes; char _PAD_SPECS_ [ sizeof(AST*) ]; CodeType UnderlyingType; - char _PAD_PROPERTIES_[ sizeof(AST*) * 2 ]; + char _PAD_PROPERTIES_[ sizeof(AST*) * 3 ]; }; }; Code Prev; @@ -573,6 +588,7 @@ struct AST_Var CodeType ValueType; Code BitfieldSize; Code Value; + CodeVar NextVar; }; }; Code Prev; diff --git a/project/components/inlines.hpp b/project/components/inlines.hpp index f1d53fd..7f35ab9 100644 --- a/project/components/inlines.hpp +++ b/project/components/inlines.hpp @@ -67,15 +67,21 @@ Code& Code::operator ++() void CodeClass::add_interface( CodeType type ) { - if ( ! ast->Next ) + CodeType possible_slot = ast->ParentType; + if ( possible_slot.ast ) { - ast->Next = type; - ast->Last = ast->Next; - return; + // Were adding an interface to parent type, so we need to make sure the parent type is public. + ast->ParentAccess = AccessSpec::Public; + // If your planning on adding a proper parent, + // then you'll need to move this over to ParentType->next and update ParentAccess accordingly. } - ast->Next->Next = type; - ast->Last = ast->Next->Next; + while ( possible_slot.ast != nullptr ) + { + possible_slot.ast = (AST_Type*) possible_slot->Next.ast; + } + + possible_slot.ast = type.ast; } void CodeParam::append( CodeParam other ) @@ -129,14 +135,21 @@ CodeParam& CodeParam::operator ++() void CodeStruct::add_interface( CodeType type ) { - if ( ! ast->Next ) + CodeType possible_slot = ast->ParentType; + if ( possible_slot.ast ) { - ast->Next = type; - ast->Last = ast->Next; + // Were adding an interface to parent type, so we need to make sure the parent type is public. + ast->ParentAccess = AccessSpec::Public; + // If your planning on adding a proper parent, + // then you'll need to move this over to ParentType->next and update ParentAccess accordingly. } - ast->Next->Next = type; - ast->Last = ast->Next->Next; + while ( possible_slot.ast != nullptr ) + { + possible_slot.ast = (AST_Type*) possible_slot->Next.ast; + } + + possible_slot.ast = type.ast; } CodeBody def_body( CodeT type ) diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 3381076..329f958 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -4963,6 +4963,7 @@ CodeType parse_type( bool* typedef_is_function ) SpecifierT spec = ESpecifier::to_type( currtok ); if ( spec != ESpecifier::Const + // TODO : Add support for NoExcept // && spec != ESpecifier::NoExcept && spec != ESpecifier::RValue ) { From 212b4e6d167302cf948494390870c04502d558ba Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 7 Sep 2023 21:11:57 -0400 Subject: [PATCH 24/26] Fixes for compiling with clang Clang just had better errors and caught stuff msvc did not. --- project/components/interface.parsing.cpp | 2 +- project/dependencies/strings.hpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 329f958..c6fb2a1 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -4821,7 +4821,7 @@ CodeType parse_type( bool* typedef_is_function ) if ( is_function_typename ) { // Go to the end of the signature - while ( scanner->Type != TokType::Statement_End && scanner->TokType::BraceCurly_Open ) + while ( scanner->Type != TokType::Statement_End && scanner->Type != TokType::BraceCurly_Open ) ++ scanner; // Go back to the first capture start found diff --git a/project/dependencies/strings.hpp b/project/dependencies/strings.hpp index cc57a96..599c681 100644 --- a/project/dependencies/strings.hpp +++ b/project/dependencies/strings.hpp @@ -281,22 +281,22 @@ struct String switch ( c ) { case ' ': - result.append('·'); + result.append( txt("·") ); break; case '\t': - result.append('→'); + result.append( txt("→") ); break; case '\n': - result.append('↵'); + result.append( txt("↵") ); break; case '\r': - result.append('⏎'); + result.append( txt("⏎") ); break; case '\v': - result.append('⇕'); + result.append( txt("⇕") ); break; case '\f': - result.append('⌂'); + result.append( txt("⌂") ); break; default: result.append(c); From 0a8d3bfc6a94fdce7e27fb54aaf515b0e1dd2d17 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 7 Sep 2023 22:29:04 -0400 Subject: [PATCH 25/26] Added fixed arenas (just ergonomics) --- project/dependencies/memory.hpp | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/project/dependencies/memory.hpp b/project/dependencies/memory.hpp index d436885..f589875 100644 --- a/project/dependencies/memory.hpp +++ b/project/dependencies/memory.hpp @@ -477,6 +477,45 @@ struct Arena } }; +// Just a wrapper around using an arena with memory associated with its scope instead of from an allocator. +// Used for static segment or stack allocations. +template< s32 Size > +struct FixedArena +{ + static + FixedArena init() + { + FixedArena result = { Arena::init_from_memory( result.memory, Size ), 0 }; + return result; + } + + sw size_remaining( sw alignment ) + { + return arena.size_remaining( alignment ); + } + + operator AllocatorInfo() + { + return { Arena::allocator_proc, &arena }; + } + + Arena arena; + char memory[ Size ]; +}; + +using Arena_1KB = FixedArena< kilobytes( 1 ) >; +using Arena_4KB = FixedArena< kilobytes( 4 ) >; +using Arena_8KB = FixedArena< kilobytes( 8 ) >; +using Arena_16KB = FixedArena< kilobytes( 16 ) >; +using Arena_32KB = FixedArena< kilobytes( 32 ) >; +using Arena_64KB = FixedArena< kilobytes( 64 ) >; +using Arena_128KB = FixedArena< kilobytes( 128 ) >; +using Arena_256KB = FixedArena< kilobytes( 256 ) >; +using Arena_512KB = FixedArena< kilobytes( 512 ) >; +using Arena_1MB = FixedArena< megabytes( 1 ) >; +using Arena_2MB = FixedArena< megabytes( 2 ) >; +using Arena_4MB = FixedArena< megabytes( 4 ) >; + struct Pool { static From 6ea40094ee0e618a28e47eb61d8057bb9f37d5ed Mon Sep 17 00:00:00 2001 From: Ed_ Date: Thu, 7 Sep 2023 22:51:15 -0400 Subject: [PATCH 26/26] More fixes from clang warnings. Parser::lex prep for changes parse_static_assert now properly adds new-line to end of statement. I'm going to end up making a static_assert ast... that or when the statement ast is made it will handle adding that newline. --- docs/Parser_Algo.md | 2 - project/auxillary/builder.cpp | 3 +- project/auxillary/scanner.hpp | 2 +- project/components/ast.hpp | 2 +- project/components/interface.parsing.cpp | 124 ++++++++++++++++------- project/dependencies/memory.hpp | 2 +- 6 files changed, 93 insertions(+), 42 deletions(-) diff --git a/docs/Parser_Algo.md b/docs/Parser_Algo.md index 9c4958e..460a022 100644 --- a/docs/Parser_Algo.md +++ b/docs/Parser_Algo.md @@ -87,5 +87,3 @@ Each internal procedure will be ## parse_type -This is the worst part of the parser. Because other than actual expression values in C++, typenames are the second worst thing to parse in the langauge. - diff --git a/project/auxillary/builder.cpp b/project/auxillary/builder.cpp index d0ca76d..3e7d5e4 100644 --- a/project/auxillary/builder.cpp +++ b/project/auxillary/builder.cpp @@ -28,8 +28,7 @@ void Builder::pad_lines( s32 num ) void Builder::print( Code code ) { String str = code->to_string(); - const sw len = str.length(); - + // const sw len = str.length(); // log_fmt( "%s - print: %.*s\n", File.filename, len > 80 ? 80 : len, str.Data ); Buffer.append( str ); } diff --git a/project/auxillary/scanner.hpp b/project/auxillary/scanner.hpp index 5bc60d5..6b6b435 100644 --- a/project/auxillary/scanner.hpp +++ b/project/auxillary/scanner.hpp @@ -92,7 +92,7 @@ Code scan_file( char const* path ) if ( left && current == '\n' ) move_fwd(); - sptr skip_size = fsize - left; + // sptr skip_size = fsize - left; if ( (scanner + 2) >= ( str.Data + fsize ) ) { mem_move( str, scanner, left ); diff --git a/project/components/ast.hpp b/project/components/ast.hpp index c30901f..c1468ba 100644 --- a/project/components/ast.hpp +++ b/project/components/ast.hpp @@ -82,7 +82,7 @@ struct Code static Code Invalid; # pragma endregion Statics - #define Using_Code( Typename ) \ +# define Using_Code( Typename ) \ char const* debug_str(); \ Code duplicate(); \ bool is_equal( Code other ); \ diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index c6fb2a1..8089f90 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -224,8 +224,8 @@ namespace Parser return false; } - if ( Arr[ Idx ].Type == TokType::NewLine && type != TokType::NewLine - || Arr[ Idx ].Type == TokType::Comment && type != TokType::Comment ) + if ( ( Arr[ Idx ].Type == TokType::NewLine && type != TokType::NewLine ) + || ( Arr[ Idx ].Type == TokType::Comment && type != TokType::Comment ) ) { Idx++; } @@ -284,7 +284,7 @@ namespace Parser move_forward(); \ } - #define end_line() \ + # define end_line() \ do \ { \ while ( left && current == ' ' ) \ @@ -319,13 +319,8 @@ namespace Parser return { { nullptr }, 0 }; } - local_persist char defines_map_mem[ kilobytes(64) ]; - local_persist Arena defines_map_arena; - HashTable defines; - { - defines_map_arena = Arena::init_from_memory( defines_map_mem, sizeof(defines_map_mem) ); - defines = HashTable::init( defines_map_arena ); - } + local_persist Arena_64KB defines_map_arena = Arena_64KB::init(); + HashTable defines = HashTable::init( defines_map_arena ); Tokens.clear(); @@ -583,6 +578,7 @@ namespace Parser continue; // Skip found token, its all handled here. } case '.': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Access_MemberSymbol; @@ -609,8 +605,9 @@ namespace Parser } goto FoundToken; - + } case '&' : + { token.Text = scanner; token.Length = 1; token.Type = TokType::Ampersand; @@ -628,11 +625,14 @@ namespace Parser } goto FoundToken; - + } case ':': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Assign_Classifer; + // Can be either a classifier (ParentType, Bitfield width), or ternary else + // token.Type = TokType::Colon; if (left) move_forward(); @@ -644,8 +644,9 @@ namespace Parser token.Length++; } goto FoundToken; - + } case '{': + { token.Text = scanner; token.Length = 1; token.Type = TokType::BraceCurly_Open; @@ -653,8 +654,9 @@ namespace Parser if (left) move_forward(); goto FoundToken; - + } case '}': + { token.Text = scanner; token.Length = 1; token.Type = TokType::BraceCurly_Close; @@ -664,8 +666,9 @@ namespace Parser end_line(); goto FoundToken; - + } case '[': + { token.Text = scanner; token.Length = 1; token.Type = TokType::BraceSquare_Open; @@ -681,8 +684,9 @@ namespace Parser } } goto FoundToken; - + } case ']': + { token.Text = scanner; token.Length = 1; token.Type = TokType::BraceSquare_Close; @@ -690,8 +694,9 @@ namespace Parser if (left) move_forward(); goto FoundToken; - + } case '(': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Capture_Start; @@ -699,8 +704,9 @@ namespace Parser if (left) move_forward(); goto FoundToken; - + } case ')': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Capture_End; @@ -708,8 +714,9 @@ namespace Parser if (left) move_forward(); goto FoundToken; - + } case '\'': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Char; @@ -740,8 +747,9 @@ namespace Parser token.Length++; } goto FoundToken; - + } case ',': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Comma; @@ -749,8 +757,9 @@ namespace Parser if (left) move_forward(); goto FoundToken; - + } case '*': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Star; @@ -758,8 +767,9 @@ namespace Parser if (left) move_forward(); goto FoundToken; - + } case ';': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Statement_End; @@ -769,8 +779,9 @@ namespace Parser end_line(); goto FoundToken; - + } case '"': + { token.Text = scanner; token.Length = 1; token.Type = TokType::String; @@ -801,24 +812,28 @@ namespace Parser token.Length++; } goto FoundToken; - + } case '?': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Operator; + // token.Type = TokType::Ternary; token.IsAssign = false; if (left) move_forward(); goto FoundToken; - - // All other operators we just label as an operator and move forward. + } case '=': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Operator; + // token.Type = TokType::Assign; token.IsAssign = true; + // token.Flags |= TokFlags::Assignment; if (left) move_forward(); @@ -833,18 +848,46 @@ namespace Parser } goto FoundToken; - + } case '+': + { + // token.Type = TokType::Add + + } case '%': + { + // token.Type = TokType::Modulo; + + } case '^': + { + // token.Type = TokType::B_XOr; + } case '~': + { + // token.Type = TokType::Unary_Not; + + } case '!': + { + // token.Type = TokType::L_Not; + } case '<': + { + // token.Type = TokType::Lesser; + + } case '>': + { + // token.Type = TokType::Greater; + + } case '|': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Operator; + // token.Type = TokType::L_Or; if (left) move_forward(); @@ -853,6 +896,8 @@ namespace Parser { token.Length++; token.IsAssign = true; + // token.Flags |= TokFlags::Assignment; + // token.Type = TokType::Assign_L_Or; if (left) move_forward(); @@ -865,12 +910,15 @@ namespace Parser move_forward(); } goto FoundToken; + } // Dash is unfortunatlly a bit more complicated... case '-': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Operator; + // token.Type = TokType::Subtract; if ( left ) { move_forward(); @@ -890,6 +938,8 @@ namespace Parser { token.Length++; token.IsAssign = true; + // token.Flags |= TokFlags::Assignment; + // token.Type = TokType::Assign_Subtract; if (left) move_forward(); @@ -903,11 +953,13 @@ namespace Parser } } goto FoundToken; - + } case '/': + { token.Text = scanner; token.Length = 1; token.Type = TokType::Operator; + // token.Type = TokType::Divide; move_forward(); if ( left ) @@ -970,11 +1022,12 @@ namespace Parser token.Length++; } Tokens.append( token ); - end_line(); + // end_line(); continue; } } goto FoundToken; + } } if ( char_is_alpha( current ) || current == '_' ) @@ -1148,7 +1201,7 @@ namespace Parser } defines.clear(); - defines_map_arena.free(); + // defines_map_arena.free(); return { Tokens, 0 }; # undef current # undef move_forward @@ -1199,7 +1252,7 @@ if ( def.Ptr == nullptr ) \ # define check( Type_ ) ( left && currtok.Type == Type_ ) # define push_scope() \ - StackNode scope { nullptr, currtok, NullToken, txt( __func__ ) }; \ + StackNode scope { nullptr, currtok_noskip, NullToken, txt( __func__ ) }; \ Context.push( & scope ) #pragma endregion Helper Macros @@ -1258,7 +1311,6 @@ constexpr bool strip_formatting_dont_preserve_newlines = false; It has edge case failures that prevent it from being used in function bodies. */ String strip_formatting( StrC raw_text, bool preserve_newlines = true ) -//, bool for_preprocess_define = false ) { String content = String::make_reserve( GlobalAllocator, raw_text.Len ); @@ -2261,7 +2313,7 @@ CodeFn parse_function_after_name( Token stmt_end = currtok; eat( TokType::Statement_End ); - if ( currtok_noskip.Type && TokType::Comment && currtok_noskip.Line == stmt_end.Line ) + if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(); } @@ -3416,8 +3468,10 @@ Code parse_static_assert() eat( TokType::Capture_End ); eat( TokType::Statement_End ); - content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text; - assert->Content = get_cached_string( content ); + content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text; + + char const* str = str_fmt_buf( "%.*s\n", content.Length, content.Text ); + assert->Content = get_cached_string( { content.Length + 1, str } ); assert->Name = assert->Content; Context.pop(); diff --git a/project/dependencies/memory.hpp b/project/dependencies/memory.hpp index f589875..8ea2ae0 100644 --- a/project/dependencies/memory.hpp +++ b/project/dependencies/memory.hpp @@ -485,7 +485,7 @@ struct FixedArena static FixedArena init() { - FixedArena result = { Arena::init_from_memory( result.memory, Size ), 0 }; + FixedArena result = { Arena::init_from_memory( result.memory, Size ), {0} }; return result; }