ed/gencpp
1
0
Fork 0
mirror of https://github.com/Ed94/gencpp.git synced 2024-09-20 11:27:32 -07:00
Code Issues Packages Projects Releases Wiki Activity

Lexer improvement prep, segmentation of lexer and parser.

Browse Source
This commit is contained in:
Edward R. Gonzalez 2023-11-20 15:09:01 -05:00
parent 5c73fbee83
commit 9d27c7d37e
15 changed files with 5975 additions and 5706 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
project/bootstrap.cpp
View File

@ -188,7 +188,9 @@ int gen_main()
Code code_serialization = scan_file( "components/code_serialization.cpp" ); Code code_serialization = scan_file( "components/code_serialization.cpp" );
Code interface = scan_file( "components/interface.cpp" ); Code interface = scan_file( "components/interface.cpp" );
Code upfront = scan_file( "components/interface.upfront.cpp" ); Code upfront = scan_file( "components/interface.upfront.cpp" );
Code parsing = scan_file( "components/interface.parsing.cpp" ); Code lexer = scan_file( "components/lexer.cpp" );
Code parser = scan_file( "components/parser.cpp" );
Code parsing_interface = scan_file( "components/interface.parsing.cpp" );
Code untyped = scan_file( "components/interface.untyped.cpp" ); Code untyped = scan_file( "components/interface.untyped.cpp" );
CodeBody etoktype = gen_etoktype( "enums/ETokType.csv", "enums/AttributeTokens.csv" ); CodeBody etoktype = gen_etoktype( "enums/ETokType.csv", "enums/AttributeTokens.csv" );
@ -206,7 +208,7 @@ int gen_main()
src.print_fmt( "\n#pragma region AST\n\n" ); src.print_fmt( "\n#pragma region AST\n\n" );
src.print( ast_case_macros ); src.print( ast_case_macros );
src.print( ast ); src.print( ast );
src.print( code ); src.print( code_serialization );
src.print_fmt( "\n#pragma endregion AST\n" ); src.print_fmt( "\n#pragma endregion AST\n" );
src.print_fmt( "\n#pragma region Interface\n" ); src.print_fmt( "\n#pragma region Interface\n" );
@ -214,7 +216,9 @@ int gen_main()
src.print( upfront ); src.print( upfront );
src.print_fmt( "\n#pragma region Parsing\n\n" ); src.print_fmt( "\n#pragma region Parsing\n\n" );
src.print( nspaced_etoktype ); src.print( nspaced_etoktype );
src.print( parsing ); src.print( lexer );
src.print( parser );
src.print( parsing_interface );
src.print( untyped ); src.print( untyped );
src.print_fmt( "\n#pragma endregion Parsing\n\n" ); src.print_fmt( "\n#pragma endregion Parsing\n\n" );
src.print_fmt( "#pragma endregion Interface\n\n" ); src.print_fmt( "#pragma endregion Interface\n\n" );

33
project/components/ast.hpp
View File

@ -116,10 +116,16 @@ struct Code
// TODO(Ed) : Remove this overload. // TODO(Ed) : Remove this overload.
// auto& operator*() auto& operator*()
// { {
// return *this; local_persist thread_local
// } Code NullRef = { nullptr };
if ( ast == nullptr )
return NullRef;
return *this;
}
AST* ast; AST* ast;
@ -182,7 +188,10 @@ struct AST
char const* type_str(); char const* type_str();
bool validate_body(); bool validate_body();
neverinline String to_string(); String to_string();
neverinline
void to_string( String& result );
template< class Type > template< class Type >
forceinline Type cast() forceinline Type cast()
@ -282,6 +291,7 @@ struct AST
AST* Parent; AST* Parent;
StringCached Name; StringCached Name;
CodeT Type; CodeT Type;
// CodeFlag CodeFlags;
ModuleFlag ModuleFlags; ModuleFlag ModuleFlags;
union { union {
b32 IsFunction; // Used by typedef to not serialize the name field. b32 IsFunction; // Used by typedef to not serialize the name field.
@ -341,6 +351,7 @@ struct AST_POD
AST* Parent; AST* Parent;
StringCached Name; StringCached Name;
CodeT Type; CodeT Type;
CodeFlag CodeFlags;
ModuleFlag ModuleFlags; ModuleFlag ModuleFlags;
union { union {
b32 IsFunction; // Used by typedef to not serialize the name field. b32 IsFunction; // Used by typedef to not serialize the name field.
@ -369,6 +380,14 @@ static_assert( sizeof(AST_POD) == AST_POD_Size, "ERROR: AST POD is not size o
#pragma region Code Types #pragma region Code Types
// struct CodeIterator
// {
// Code begin()
// {
// }
// };
struct CodeBody struct CodeBody
{ {
Using_Code( CodeBody ); Using_Code( CodeBody );
@ -650,6 +669,8 @@ struct CodeEnum
void to_string_def( String& result ); void to_string_def( String& result );
void to_string_fwd( String& result ); void to_string_fwd( String& result );
void to_string_class_def( String& result );
void to_string_class_fwd( String& result );
AST* raw(); AST* raw();
operator Code(); operator Code();
@ -697,7 +718,7 @@ struct CodeFriend
struct CodeFn struct CodeFn
{ {
Using_Code( CodeFriend ); Using_Code( CodeFn );
void to_string_def( String& result ); void to_string_def( String& result );
void to_string_fwd( String& result ); void to_string_fwd( String& result );

92
project/components/code_serialization.cpp
View File

@ -1,4 +1,4 @@
#if GEN_INTELLISENSE_DIRECTIVES #ifdef GEN_INTELLISENSE_DIRECTIVES
#pragma once #pragma once
#include "ast.cpp" #include "ast.cpp"
#endif #endif
@ -24,6 +24,11 @@ String CodeBody::to_string()
switch ( ast->Type ) switch ( ast->Type )
{ {
using namespace ECode; using namespace ECode;
case Untyped:
case Execution:
result.append( raw()->Content );
break;
case Enum_Body: case Enum_Body:
case Class_Body: case Class_Body:
case Extern_Linkage_Body: case Extern_Linkage_Body:
@ -57,7 +62,7 @@ void CodeBody::to_string_export( String& result )
{ {
result.append_fmt( "export\n{\n" ); result.append_fmt( "export\n{\n" );
Code curr = { this }; Code curr = *this;
s32 left = ast->NumEntries; s32 left = ast->NumEntries;
while ( left-- ) while ( left-- )
{ {
@ -91,7 +96,7 @@ String CodeConstructor::to_string()
void CodeConstructor::to_string_def( String& result ) void CodeConstructor::to_string_def( String& result )
{ {
result.append( ast->Parent->name ); result.append( ast->Parent->Name );
if ( ast->Params ) if ( ast->Params )
result.append_fmt( "( %S )", ast->Params.to_string() ); result.append_fmt( "( %S )", ast->Params.to_string() );
@ -178,7 +183,7 @@ void CodeClass::to_string_def( String& result )
result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() ); result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() );
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
result.append(";\n"); result.append(";\n");
} }
@ -193,7 +198,7 @@ void CodeClass::to_string_fwd( String& result )
else result.append_fmt( "class %S", ast->Name ); else result.append_fmt( "class %S", ast->Name );
// Check if it can have an end-statement // Check if it can have an end-statement
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
{ {
if ( ast->InlineCmt ) if ( ast->InlineCmt )
result.append_fmt( "; // %S\n", ast->InlineCmt->Content ); result.append_fmt( "; // %S\n", ast->InlineCmt->Content );
@ -209,7 +214,7 @@ String CodeDefine::to_string()
void CodeDefine::to_string( String& result ) void CodeDefine::to_string( String& result )
{ {
return result.append_fmt( "#define %S %S\n", ast->Name, ast->Content ); result.append_fmt( "#define %S %S\n", ast->Name, ast->Content );
} }
String CodeDestructor::to_string() String CodeDestructor::to_string()
@ -298,7 +303,7 @@ void CodeEnum::to_string_def( String& result )
if ( ast->Attributes ) if ( ast->Attributes )
result.append_fmt( "%S ", ast->Attributes.to_string() ); result.append_fmt( "%S ", ast->Attributes.to_string() );
if ( UnderlyingType ) if ( ast->UnderlyingType )
result.append_fmt( "%S : %S\n{\n%S\n}" result.append_fmt( "%S : %S\n{\n%S\n}"
, ast->Name , ast->Name
, ast->UnderlyingType.to_string() , ast->UnderlyingType.to_string()
@ -309,7 +314,7 @@ void CodeEnum::to_string_def( String& result )
} }
else result.append_fmt( "enum %S\n{\n%S\n}", ast->Name, ast->Body.to_string() ); else result.append_fmt( "enum %S\n{\n%S\n}", ast->Name, ast->Body.to_string() );
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
result.append(";\n"); result.append(";\n");
} }
@ -323,7 +328,7 @@ void CodeEnum::to_string_fwd( String& result )
result.append_fmt( "enum %S : %S", ast->Name, ast->UnderlyingType.to_string() ); result.append_fmt( "enum %S : %S", ast->Name, ast->UnderlyingType.to_string() );
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
{ {
if ( ast->InlineCmt ) if ( ast->InlineCmt )
result.append_fmt("; %S", ast->InlineCmt->Content ); result.append_fmt("; %S", ast->InlineCmt->Content );
@ -360,7 +365,7 @@ void CodeEnum::to_string_class_def( String& result )
result.append_fmt( "enum class %S\n{\n%S\n}", ast->Body.to_string() ); result.append_fmt( "enum class %S\n{\n%S\n}", ast->Body.to_string() );
} }
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
result.append(";\n"); result.append(";\n");
} }
@ -376,7 +381,7 @@ void CodeEnum::to_string_class_fwd( String& result )
result.append_fmt( "%S : %S", ast->Name, ast->UnderlyingType.to_string() ); result.append_fmt( "%S : %S", ast->Name, ast->UnderlyingType.to_string() );
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
{ {
if ( ast->InlineCmt ) if ( ast->InlineCmt )
result.append_fmt("; %S", ast->InlineCmt->Content ); result.append_fmt("; %S", ast->InlineCmt->Content );
@ -390,13 +395,6 @@ String CodeExec::to_string()
return ast->Content.duplicate( GlobalAllocator ); return ast->Content.duplicate( GlobalAllocator );
} }
String CodeExtern::to_string()
{
String result = String::make( GlobalAllocator, "" );
to_string( result );
return result;
}
void CodeExtern::to_string( String& result ) void CodeExtern::to_string( String& result )
{ {
if ( ast->Body ) if ( ast->Body )
@ -407,7 +405,7 @@ void CodeExtern::to_string( String& result )
String CodeInclude::to_string() String CodeInclude::to_string()
{ {
return String::fmt( GlobalAllocator, "#include %S\n", ast->Content ); return String::fmt_buf( GlobalAllocator, "#include %S\n", ast->Content );
} }
void CodeInclude::to_string( String& result ) void CodeInclude::to_string( String& result )
@ -415,7 +413,7 @@ void CodeInclude::to_string( String& result )
result.append_fmt( "#include %S\n", ast->Content ); result.append_fmt( "#include %S\n", ast->Content );
} }
String CodeFriend::to_string( String& result ) String CodeFriend::to_string()
{ {
String result = String::make( GlobalAllocator, "" ); String result = String::make( GlobalAllocator, "" );
to_string( result ); to_string( result );
@ -437,7 +435,7 @@ void CodeFriend::to_string( String& result )
result.append("\n"); result.append("\n");
} }
String CodeFn::to_string( String& result ) String CodeFn::to_string()
{ {
String result = String::make( GlobalAllocator, "" ); String result = String::make( GlobalAllocator, "" );
switch ( ast->Type ) switch ( ast->Type )
@ -482,7 +480,7 @@ void CodeFn::to_string_def( String& result )
else else
result.append_fmt( "%S(", ast->Name ); result.append_fmt( "%S(", ast->Name );
if ( Params ) if ( ast->Params )
result.append_fmt( "%S)", ast->Params.to_string() ); result.append_fmt( "%S)", ast->Params.to_string() );
else else
@ -511,9 +509,9 @@ void CodeFn::to_string_fwd( String& result )
if ( ast->Attributes ) if ( ast->Attributes )
result.append_fmt( "%S ", ast->Attributes.to_string() ); result.append_fmt( "%S ", ast->Attributes.to_string() );
if ( Specs ) if ( ast->Specs )
{ {
for ( SpecifierT spec : Specs ) for ( SpecifierT spec : ast->Specs )
{ {
if ( ! ESpecifier::is_trailing( spec ) ) if ( ! ESpecifier::is_trailing( spec ) )
{ {
@ -542,7 +540,7 @@ void CodeFn::to_string_fwd( String& result )
if ( ast->Specs ) if ( ast->Specs )
{ {
for ( SpecifierT spec : Specs ) for ( SpecifierT spec : ast->Specs )
{ {
if ( ESpecifier::is_trailing( spec ) ) if ( ESpecifier::is_trailing( spec ) )
{ {
@ -673,7 +671,7 @@ void CodeOperator::to_string_fwd( String& result )
if ( ast->Specs ) if ( ast->Specs )
{ {
for ( SpecifierT spec : Specs ) for ( SpecifierT spec : ast->Specs )
{ {
if ( ! ESpecifier::is_trailing( spec ) ) if ( ! ESpecifier::is_trailing( spec ) )
{ {
@ -698,7 +696,7 @@ void CodeOperator::to_string_fwd( String& result )
if ( ast->Specs ) if ( ast->Specs )
{ {
for ( SpecifierT spec : Specs ) for ( SpecifierT spec : ast->Specs )
{ {
if ( ESpecifier::is_trailing( spec ) ) if ( ESpecifier::is_trailing( spec ) )
{ {
@ -741,7 +739,7 @@ void CodeOpCast::to_string_def( String& result )
else else
result.append_fmt( "operator %S()", ast->ValueType.to_string() ); result.append_fmt( "operator %S()", ast->ValueType.to_string() );
for ( SpecifierT spec : Specs ) for ( SpecifierT spec : ast->Specs )
{ {
if ( ESpecifier::is_trailing( spec ) ) if ( ESpecifier::is_trailing( spec ) )
{ {
@ -751,7 +749,7 @@ void CodeOpCast::to_string_def( String& result )
} }
result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() ); result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() );
break; return;
} }
if ( ast->Name && ast->Name.length() ) if ( ast->Name && ast->Name.length() )
@ -768,7 +766,7 @@ void CodeOpCast::to_string_fwd( String& result )
result.append_fmt( "operator %S()", ast->ValueType.to_string() ); result.append_fmt( "operator %S()", ast->ValueType.to_string() );
for ( SpecifierT spec : Specs ) for ( SpecifierT spec : ast->Specs )
{ {
if ( ESpecifier::is_trailing( spec ) ) if ( ESpecifier::is_trailing( spec ) )
{ {
@ -781,7 +779,7 @@ void CodeOpCast::to_string_fwd( String& result )
result.append_fmt( "; %S", ast->InlineCmt->Content ); result.append_fmt( "; %S", ast->InlineCmt->Content );
else else
result.append( ";\n" ); result.append( ";\n" );
break; return;
} }
if ( ast->InlineCmt ) if ( ast->InlineCmt )
@ -799,10 +797,10 @@ String CodeParam::to_string()
void CodeParam::to_string( String& result ) void CodeParam::to_string( String& result )
{ {
if ( ast->ValueType == nullptr ) if ( ast->ValueType.ast == nullptr )
{ {
result.append_fmt( "%S", ast->Name ); result.append_fmt( "%S", ast->Name );
break; return;
} }
if ( ast->Name ) if ( ast->Name )
@ -835,7 +833,7 @@ String CodePreprocessCond::to_string()
case Preprocess_IfDef: case Preprocess_IfDef:
to_string_ifdef( result ); to_string_ifdef( result );
break; break;
case Preprocess_IfNotDef case Preprocess_IfNotDef:
to_string_ifndef( result ); to_string_ifndef( result );
break; break;
case Preprocess_ElIf: case Preprocess_ElIf:
@ -968,7 +966,7 @@ void CodeStruct::to_string_def( String& result )
result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() ); result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() );
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
result.append(";\n"); result.append(";\n");
} }
@ -982,7 +980,7 @@ void CodeStruct::to_string_fwd( String& result )
else result.append_fmt( "struct %S", ast->Name ); else result.append_fmt( "struct %S", ast->Name );
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
{ {
if ( ast->InlineCmt ) if ( ast->InlineCmt )
result.append_fmt("; %S", ast->InlineCmt->Content ); result.append_fmt("; %S", ast->InlineCmt->Content );
@ -1029,14 +1027,14 @@ void CodeTypedef::to_string( String& result )
else else
result.append_fmt( "%S %S", ast->UnderlyingType.to_string(), ast->Name ); result.append_fmt( "%S %S", ast->UnderlyingType.to_string(), ast->Name );
if ( ast->UnderlyingType->Type == Typename && ast->UnderlyingType->ArrExpr ) if ( ast->UnderlyingType->Type == ECode::Typename && ast->UnderlyingType->ArrExpr )
{ {
result.append_fmt( "[ %S ];", ast->UnderlyingType->ArrExpr->to_string() ); result.append_fmt( "[ %S ];", ast->UnderlyingType->ArrExpr->to_string() );
AST* next_arr_expr = ast->UnderlyingType->ArrExpr->Next; AST* next_arr_expr = ast->UnderlyingType->ArrExpr->Next;
while ( next_arr_expr ) while ( next_arr_expr )
{ {
result.append_fmt( "[ %S ];", next_arr_expr.to_string() ); result.append_fmt( "[ %S ];", next_arr_expr->to_string() );
next_arr_expr = next_arr_expr->Next; next_arr_expr = next_arr_expr->Next;
} }
} }
@ -1082,13 +1080,13 @@ void CodeType::to_string( String& result )
result.append_fmt( "%S ", ast->Attributes.to_string() ); result.append_fmt( "%S ", ast->Attributes.to_string() );
else else
{ {
if ( Specs ) if ( ast->Specs )
result.append_fmt( "%S %S ( %S ) %S", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string(), ast->Specs.to_string() ); result.append_fmt( "%S %S ( %S ) %S", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string(), ast->Specs.to_string() );
else else
result.append_fmt( "%S %S ( %S )", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string() ); result.append_fmt( "%S %S ( %S )", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string() );
} }
break; return;
} }
#endif #endif
@ -1136,7 +1134,7 @@ void CodeUnion::to_string( String& result )
); );
} }
if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
result.append(";\n"); result.append(";\n");
} }
@ -1236,7 +1234,7 @@ void CodeVar::to_string( String& result )
if ( ast->NextVar ) if ( ast->NextVar )
result.append_fmt( ", %S", ast->NextVar.to_string() ); result.append_fmt( ", %S", ast->NextVar.to_string() );
break; return;
} }
if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export )) if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ))
@ -1278,13 +1276,13 @@ void CodeVar::to_string( String& result )
else else
result.append( ";\n" ); result.append( ";\n" );
break; return;
} }
if ( ast->BitfieldSize ) if ( ast->BitfieldSize )
result.append_fmt( "%S %S : %S", ast->ValueType.to_string(), ast->Name, ast->BitfieldSize.to_string() ); result.append_fmt( "%S %S : %S", ast->ValueType.to_string(), ast->Name, ast->BitfieldSize.to_string() );
else if ( ValueType->ArrExpr ) else if ( ast->ValueType->ArrExpr )
{ {
result.append_fmt( "%S %S[ %S ]", ast->ValueType.to_string(), ast->Name, ast->ValueType->ArrExpr.to_string() ); result.append_fmt( "%S %S[ %S ]", ast->ValueType.to_string(), ast->Name, ast->ValueType->ArrExpr.to_string() );
@ -1299,15 +1297,15 @@ void CodeVar::to_string( String& result )
else else
result.append_fmt( "%S %S", ast->ValueType.to_string(), ast->Name ); result.append_fmt( "%S %S", ast->ValueType.to_string(), ast->Name );
if ( Value ) if ( ast->Value )
result.append_fmt( " = %S", ast->Value.to_string() ); result.append_fmt( " = %S", ast->Value.to_string() );
if ( NextVar ) if ( ast->NextVar )
result.append_fmt( ", %S", ast->NextVar.to_string() ); result.append_fmt( ", %S", ast->NextVar.to_string() );
result.append( ";" ); result.append( ";" );
if ( InlineCmt ) if ( ast->InlineCmt )
result.append_fmt(" %S", ast->InlineCmt->Content); result.append_fmt(" %S", ast->InlineCmt->Content);
else else
result.append("\n"); result.append("\n");

10
project/components/gen/ast_inlines.hpp
View File

@ -869,16 +869,6 @@ void CodeExtern::set_global()
rcast( AST*, ast )->Parent = Code::Global.ast; rcast( AST*, ast )->Parent = Code::Global.ast;
} }
String CodeExtern::to_string()
{
if ( ast == nullptr )
{
log_failure( "Code::to_string: Cannot convert code to string, AST is null!" );
return { nullptr };
}
return rcast( AST*, ast )->to_string();
}
CodeExtern& CodeExtern::operator=( Code other ) CodeExtern& CodeExtern::operator=( Code other )
{ {
if ( other.ast && other->Parent ) if ( other.ast && other->Parent )

4
project/components/gen/etoktype.cpp
View File

@ -5,7 +5,7 @@
// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp)
namespace Parser namespace parser
{ {
namespace ETokType namespace ETokType
{ {
@ -234,4 +234,4 @@ namespace Parser
using TokType = ETokType::Type; using TokType = ETokType::Type;
} // namespace Parser } // namespace parser

10
project/components/interface.cpp
View File

@ -3,8 +3,10 @@
#include "code_serialization.cpp" #include "code_serialization.cpp"
#endif #endif
internal void init_parser(); namespace parser {
internal void deinit_parser(); internal void init();
internal void deinit();
}
internal internal
void* Global_Allocator_Proc( void* allocator_data, AllocType type, sw size, sw alignment, void* old_memory, sw old_size, u64 flags ) void* Global_Allocator_Proc( void* allocator_data, AllocType type, sw size, sw alignment, void* old_memory, sw old_size, u64 flags )
@ -288,7 +290,7 @@ void init()
} }
define_constants(); define_constants();
init_parser(); parser::init();
} }
void deinit() void deinit()
@ -331,7 +333,7 @@ void deinit()
while ( left--, left ); while ( left--, left );
Global_AllocatorBuckets.free(); Global_AllocatorBuckets.free();
deinit_parser(); parser::deinit();
} }
void reset() void reset()

5568
project/components/interface.parsing.cpp
View File

File diff suppressed because it is too large Load Diff

1207
project/components/lexer.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

4400
project/components/parser.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

2
project/dependencies/strings.hpp
View File

@ -169,7 +169,7 @@ struct String
get_header().Length = 0; get_header().Length = 0;
} }
String duplicate( AllocatorInfo allocator ) String duplicate( AllocatorInfo allocator ) const
{ {
return make_length( allocator, Data, length() ); return make_length( allocator, Data, length() );
} }

175
project/enums/ETokType_New.csv Normal file
View File

@ -0,0 +1,175 @@
Invalid, "__invalid__"
Access_Private, "private"
Access_Protected, "protected"
Access_Public, "public"
Access_MemberSymbol, "."
Access_StaticSymbol, "::"
Ampersand, "&"
Ampersand_DBL, "&&"
Assign_Classifer, ":"
Attribute_Open, "[["
Attribute_Close, "]]"
BraceCurly_Open, "{"
BraceCurly_Close, "}"
BraceSquare_Open, "["
BraceSquare_Close, "]"
Capture_Start, "("
Capture_End, ")"
Comment, "__comment__"
Comment_End, "__comment_end__"
Comment_Start, "__comment_start__"
Char, "__character__"
Comma, ","
Decl_Class, "class"
Decl_GNU_Attribute, "__attribute__"
Decl_MSVC_Attribute, "__declspec"
Decl_Enum, "enum"
Decl_Extern_Linkage, "extern"
Decl_Friend, "friend"
Decl_Module, "module"
Decl_Namespace, "namespace"
Decl_Operator, "operator"
Decl_Struct, "struct"
Decl_Template, "template"
Decl_Type, "decltype"
Decl_Typedef, "typedef"
Decl_Using, "using"
Decl_Union, "union"
Identifier, "__identifier__"
Module_Import, "import"
Module_Export, "export"
NewLine, "__new_line__"
Number, "__number__"
Operator, "__operator__"
Op_Assign, "="
Op_Assign_Add, "+="
Op_Assign_Subtract, "-="
Op_Assign_Multiply, "*="
Op_Assign_Divide, "/="
Op_Assign_Modulo, "%="
Op_Assign_Bitwise_And, "&="
Op_Assign_Bitwise_Or, "|="
Op_Assign_Bitwise_XOr, "^="
Op_Assign_Bitwise_LeftShift, "<<="
Op_Assign_Bitwise_RightShift, ">>="
Op_Increment, "++"
Op_Decrement, "--"
Op_Add, "+"
Op_Subtract, "-"
Op_Multiply, "*"
Op_Divide, "/"
Op_Modulo, "%"
Op_Bitwise_And, "&"
Op_Bitwise_Or, "|"
Op_Bitwise_XOr, "^"
Op_Bitwise_LeftShitf, "<<"
Op_Bitwise_RightShift, ">>"
Op_UnaryAdd, "+"
Op_UnaryMinus, "-"
Op_UnaryNot, "~"
Op_Logical_Not, "!"
Op_Logical_And, "&&"
Op_Logical_Or, "||"
Op_Equal, "=="
Op_NotEqual, "!="
Op_Lesser, "<"
Op_Greater, ">"
Op_LesserEqual, "<="
Op_GreaterEqual", ">=
Op_Subscript, "[]"
Op_Indirection, "*"
Op_Address_Of, "&"
Op_MemberOfObject, "."
Op_MemberOfPointer", "->"
Op_PointerToMemberOfObject, ".*"
Op_PointerToMemberOfPointer, "->*"
Op_Comma, ","
Op_Ternary, "?"
Preprocess_Hash, "#"
Preprocess_Define, "define"
Preprocess_If, "if"
Preprocess_IfDef, "ifdef"
Preprocess_IfNotDef, "ifndef"
Preprocess_ElIf, "elif"
Preprocess_Else, "else"
Preprocess_EndIf, "endif"
Preprocess_Include, "include"
Preprocess_Pragma, "pragma"
Preprocess_Content, "__macro_content__"
Preprocess_Macro, "__macro__"
Preprocess_Unsupported, "__unsupported__"
Spec_Alignof, "alignof"
Spec_Const, "const"
Spec_Consteval, "consteval"
Spec_Constexpr, "constexpr"
Spec_Constinit, "constinit"
Spec_Explicit, "explicit"
Spec_Extern, "extern"
Spec_Final, "final"
Spec_ForceInline, "forceinline"
Spec_Global, "global"
Spec_Inline, "inline"
Spec_Internal_Linkage, "internal"
Spec_LocalPersist, "local_persist"
Spec_Mutable, "mutable"
Spec_NeverInline, "neverinline"
Spec_Override, "override"
Spec_Static, "static"
Spec_ThreadLocal, "thread_local"
Spec_Volatile, "volatile"
Spec_Virtual, "virtual"
Star, "*"
Stmt_Break, "break"
Stmt_Case, "case"
Stmt_Continue, "continue"
Stmt_Do, "do"
Stmt_Else, "else"
Stmt_End, ";"
Stmt_If, "if"
Stmt_For, "for"
Stmt_Return, "return"
Stmt_Switch, "switch"
Stmt_While, "while"
StaticAssert, "static_assert"
String, "__string__"
Type_Unsigned, "unsigned"
Type_Signed, "signed"
Type_Short, "short"
Type_Long, "long"
Type_bool, "bool"
Type_char, "char"
Type_int, "int"
Type_float, "float"
Type_double, "double"
Type_MS_int8, "__int8"
Type_MS_int16, "__int16"
Type_MS_int32, "__int32"
Type_MS_int64, "__int64"
Type_MS_W64, "_W64"
Varadic_Argument, "..."
__Attributes_Start, "__attrib_start__"
Can't render this file because it contains an unexpected character in line 93 and column 16.

2
project/gen.cpp
View File

@ -25,6 +25,8 @@ GEN_NS_BEGIN
#include "components/interface.cpp" #include "components/interface.cpp"
#include "components/interface.upfront.cpp" #include "components/interface.upfront.cpp"
#include "components/gen/etoktype.cpp" #include "components/gen/etoktype.cpp"
#include "components/lexer.cpp"
#include "components/parser.cpp"
#include "components/interface.parsing.cpp" #include "components/interface.parsing.cpp"
#include "components/interface.untyped.cpp" #include "components/interface.untyped.cpp"

10
project/helpers/helper.hpp
View File

@ -379,16 +379,6 @@ CodeBody gen_ast_inlines()
rcast(AST*, ast)->Parent = Code::Global.ast; rcast(AST*, ast)->Parent = Code::Global.ast;
} }
// String <typename>::to_string()
// {
// if ( ast == nullptr )
// {
// log_failure("Code::to_string: Cannot convert code to string, AST is null!");
// return { nullptr };
// }
//
// return rcast(AST*, ast)->to_string();
// }
<typename>& <typename>::operator =( Code other ) <typename>& <typename>::operator =( Code other )
{ {
if ( other.ast && other->Parent ) if ( other.ast && other->Parent )

8
singleheader/singleheader.cpp
View File

@ -201,7 +201,9 @@ int gen_main()
Code code = scan_file( project_dir "components/code_serialization.cpp" ); Code code = scan_file( project_dir "components/code_serialization.cpp" );
Code interface = scan_file( project_dir "components/interface.cpp" ); Code interface = scan_file( project_dir "components/interface.cpp" );
Code upfront = scan_file( project_dir "components/interface.upfront.cpp" ); Code upfront = scan_file( project_dir "components/interface.upfront.cpp" );
Code parsing = scan_file( project_dir "components/interface.parsing.cpp" ); Code lexer = scan_file( project_dir "components/lexer.cpp" );
Code parser = scan_file( project_dir "components/parser.cpp" );
Code parsing_interface = scan_file( project_dir "components/interface.parsing.cpp" );
Code untyped = scan_file( project_dir "components/interface.untyped.cpp" ); Code untyped = scan_file( project_dir "components/interface.untyped.cpp" );
CodeBody etoktype = gen_etoktype( project_dir "enums/ETokType.csv", project_dir "enums/AttributeTokens.csv" ); CodeBody etoktype = gen_etoktype( project_dir "enums/ETokType.csv", project_dir "enums/AttributeTokens.csv" );
@ -221,7 +223,9 @@ int gen_main()
header.print( upfront ); header.print( upfront );
header.print_fmt( "\n#pragma region Parsing\n\n" ); header.print_fmt( "\n#pragma region Parsing\n\n" );
header.print( parser_nspace ); header.print( parser_nspace );
header.print( parsing ); header.print( lexer );
header.print( parser );
header.print( parsing_interface );
header.print_fmt( "\n#pragma endregion Parsing\n" ); header.print_fmt( "\n#pragma endregion Parsing\n" );
header.print( untyped ); header.print( untyped );
header.print_fmt( "\n#pragma endregion Interface\n\n"); header.print_fmt( "\n#pragma endregion Interface\n\n");