From 31e1c38c1833eee5d30650ff1324938e0e030542 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 28 Jul 2023 21:44:31 -0400 Subject: [PATCH] Started to implement context stack for parser. --- Readme.md | 30 +- project/components/ETokType.csv | 8 +- project/components/gen.ast.cpp | 14 +- project/components/gen.ast_case_macros.cpp | 152 +++++---- project/components/gen.data.cpp | 4 + project/components/gen.etoktype.cpp | 14 +- project/components/gen.header_end.hpp | 9 + project/components/gen.interface.parsing.cpp | 335 ++++++++++++------- project/components/gen.interface.upfront.cpp | 28 +- project/components/gen.types.hpp | 7 - project/helpers/gen.helper.hpp | 9 +- 11 files changed, 350 insertions(+), 260 deletions(-) diff --git a/Readme.md b/Readme.md index 49f6ef6..9d91958 100644 --- a/Readme.md +++ b/Readme.md @@ -5,11 +5,6 @@ An attempt at simple staged metaprogramming for c/c++. The library API is a composition of code element constructors. These build up a code AST to then serialize with a file builder. -General goal is to have a less than 15k sloc library that takes at most a couple of hours to learn and make use of. - -*Why 15k ?* Assuming a seasoned coder of C++ can read and understand around 1000-2000 lines of code per hour, 15,000 could be understood in under 16-18 hours -and have confidence in modifying for their use case. - This code base attempts follow the [handmade philosophy](https://handmade.network/manifesto), its not meant to be a black box metaprogramming utility, its meant for the user to extend for their project domain. @@ -30,17 +25,17 @@ The project has reached an *alpha* state, all the current functionality works fo The project has no external dependencies beyond: -* `errno.h` (gen.dep.cpp) -* `stat.h` (gen.dep.cpp) -* `stdarg.h` (gen.dep.hpp) -* `stddef.h` (gen.dep.hpp -* `stdio.h` (gen.dep.cpp) -* `copyfile.h` (Mac, gen.dep.cpp) -* `types.h` (Linux, gen.dep.cpp) -* `unistd.h` (Linux/Mac, gen.dep.cpp) -* `intrin.h` (Windows, gen.dep.hpp) -* `io.h` (Windows with gcc, gen.dep.cpp) -* `windows.h` (Windows, gen.dep.cpp) +* `errno.h` +* `stat.h` +* `stdarg.h` +* `stddef.h` +* `stdio.h` +* `copyfile.h` (Mac) +* `types.h` (Linux) +* `unistd.h` (Linux/Mac) +* `intrin.h` (Windows) +* `io.h` (Windows with gcc) +* `windows.h` (Windows) Dependencies for the project are wrapped within `GENCPP_ROLL_OWN_DEPENDENCIES` (Defining it will disable them). The majority of the dependency's implementation was derived from the [c-zpl library](https://github.com/zpl-c/zpl). @@ -561,6 +556,7 @@ The following are provided predefined by the library as they are commonly used: * `access_private` * `module_global_fragment` * `module_private_fragment` +* `param_varaidc` (Used for varadic definitions) * `pragma_once` * `spec_const` * `spec_consteval` @@ -586,7 +582,7 @@ The following are provided predefined by the library as they are commonly used: * `spec_type_unsigned` * `spec_type_short` * `spec_type_long` -* `t_empty` +* `t_empty` (Used for varaidc macros) * `t_auto` * `t_void` * `t_int` diff --git a/project/components/ETokType.csv b/project/components/ETokType.csv index be89940..2056739 100644 --- a/project/components/ETokType.csv +++ b/project/components/ETokType.csv @@ -37,8 +37,12 @@ Module_Import, "import" Module_Export, "export" Number, "number" Operator, "operator" -Preprocessor_Directive, "#" -Preprocessor_Include, "include" +Preprocess_Define, "#define" +Preprocess_Include, "#include" +Preprocess_If, "#if" +Preprocess_ElIF, "#elif" +Preprocess_Else, "#else" +Preprocess_EndIf, "#endif" Spec_Alignas, "alignas" Spec_Const, "const" Spec_Consteval, "consteval" diff --git a/project/components/gen.ast.cpp b/project/components/gen.ast.cpp index c576ec3..e9054aa 100644 --- a/project/components/gen.ast.cpp +++ b/project/components/gen.ast.cpp @@ -778,7 +778,7 @@ bool AST::validate_body() switch ( Type ) { case Class_Body: - CheckEntries( AST_BODY_CLASS_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_CLASS_UNALLOWED_TYPES ); break; case Enum_Body: for ( Code entry : cast() ) @@ -791,22 +791,22 @@ bool AST::validate_body() } break; case Export_Body: - CheckEntries( AST_BODY_CLASS_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_CLASS_UNALLOWED_TYPES ); break; case Extern_Linkage: - CheckEntries( AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES ); break; case Function_Body: - CheckEntries( AST_BODY_FUNCTION_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES ); break; case Global_Body: - CheckEntries( AST_BODY_GLOBAL_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES ); break; case Namespace_Body: - CheckEntries( AST_BODY_NAMESPACE_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES ); break; case Struct_Body: - CheckEntries( AST_BODY_STRUCT_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_STRUCT_UNALLOWED_TYPES ); break; case Union_Body: for ( Code entry : Body->cast() ) diff --git a/project/components/gen.ast_case_macros.cpp b/project/components/gen.ast_case_macros.cpp index 83e18e1..fa47e8b 100644 --- a/project/components/gen.ast_case_macros.cpp +++ b/project/components/gen.ast_case_macros.cpp @@ -1,81 +1,79 @@ -# define AST_BODY_CLASS_UNALLOWED_TYPES \ - case PlatformAttributes: \ - case Class_Body: \ - case Enum_Body: \ - case Extern_Linkage: \ - case Function_Body: \ - case Function_Fwd: \ - case Global_Body: \ - case Namespace: \ - case Namespace_Body: \ - case Operator: \ - case Operator_Fwd: \ - case Parameters: \ - case Specifiers: \ - case Struct_Body: \ - case Typename: - -# define AST_BODY_FUNCTION_UNALLOWED_TYPES \ - case Access_Public: \ - case Access_Protected: \ - case Access_Private: \ - case PlatformAttributes: \ - case Class_Body: \ - case Enum_Body: \ - case Extern_Linkage: \ - case Friend: \ - case Function_Body: \ - case Function_Fwd: \ - case Global_Body: \ - case Namespace: \ - case Namespace_Body: \ - case Operator: \ - case Operator_Fwd: \ - case Operator_Member: \ - case Operator_Member_Fwd: \ - case Parameters: \ - case Specifiers: \ - case Struct_Body: \ - case Typename: - -# define AST_BODY_GLOBAL_UNALLOWED_TYPES \ - case Access_Public: \ - case Access_Protected: \ - case Access_Private: \ - case PlatformAttributes: \ - case Class_Body: \ - case Enum_Body: \ - case Execution: \ - case Friend: \ - case Function_Body: \ - case Global_Body: \ - case Namespace_Body: \ - case Operator_Member: \ - case Operator_Member_Fwd: \ - case Parameters: \ - case Specifiers: \ - case Struct_Body: \ - case Typename: - -# define AST_BODY_EXPORT_UNALLOWED_TYPES AST_BODY_GLOBAL_UNALLOWED_TYPES -# define AST_BODY_NAMESPACE_UNALLOWED_TYPES \ - case Access_Public: \ - case Access_Protected: \ - case Access_Private: \ +# define GEN_AST_BODY_CLASS_UNALLOWED_TYPES \ case PlatformAttributes: \ - case Class_Body: \ - case Enum_Body: \ - case Execution: \ - case Friend: \ - case Function_Body: \ - case Namespace_Body: \ - case Operator_Member: \ - case Operator_Member_Fwd: \ - case Parameters: \ - case Specifiers: \ - case Struct_Body: \ + case Class_Body: \ + case Enum_Body: \ + case Extern_Linkage: \ + case Function_Body: \ + case Function_Fwd: \ + case Global_Body: \ + case Namespace: \ + case Namespace_Body: \ + case Operator: \ + case Operator_Fwd: \ + case Parameters: \ + case Specifiers: \ + case Struct_Body: \ + case Typename: +# define GEN_AST_BODY_STRUCT_UNALLOWED_TYPES GEN_AST_BODY_CLASS_UNALLOWED_TYPES + +# define GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES \ + case Access_Public: \ + case Access_Protected: \ + case Access_Private: \ + case PlatformAttributes: \ + case Class_Body: \ + case Enum_Body: \ + case Extern_Linkage: \ + case Friend: \ + case Function_Body: \ + case Function_Fwd: \ + case Global_Body: \ + case Namespace: \ + case Namespace_Body: \ + case Operator: \ + case Operator_Fwd: \ + case Operator_Member: \ + case Operator_Member_Fwd: \ + case Parameters: \ + case Specifiers: \ + case Struct_Body: \ case Typename: -# define AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES AST_BODY_GLOBAL_UNALLOWED_TYPES +# define GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES \ + case Access_Public: \ + case Access_Protected: \ + case Access_Private: \ + case PlatformAttributes: \ + case Class_Body: \ + case Enum_Body: \ + case Execution: \ + case Friend: \ + case Function_Body: \ + case Global_Body: \ + case Namespace_Body: \ + case Operator_Member: \ + case Operator_Member_Fwd: \ + case Parameters: \ + case Specifiers: \ + case Struct_Body: \ + case Typename: +# define GEN_AST_BODY_EXPORT_UNALLOWED_TYPES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES +# define GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES -# define AST_BODY_STRUCT_UNALLOWED_TYPES AST_BODY_CLASS_UNALLOWED_TYPES +# define GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES \ + case Access_Public: \ + case Access_Protected: \ + case Access_Private: \ + case PlatformAttributes: \ + case Class_Body: \ + case Enum_Body: \ + case Execution: \ + case Friend: \ + case Function_Body: \ + case Namespace_Body: \ + case Operator_Member: \ + case Operator_Member_Fwd: \ + case Parameters: \ + case Specifiers: \ + case Struct_Body: \ + case Typename: diff --git a/project/components/gen.data.cpp b/project/components/gen.data.cpp index 313a485..99cbdcd 100644 --- a/project/components/gen.data.cpp +++ b/project/components/gen.data.cpp @@ -1,4 +1,5 @@ #pragma region StaticData + // TODO : Convert global allocation strategy to use a slab allocation strategy. global AllocatorInfo GlobalAllocator; global Array Global_AllocatorBuckets; @@ -16,9 +17,11 @@ global AllocatorInfo Allocator_Lexer = heap(); global AllocatorInfo Allocator_StringArena = heap(); global AllocatorInfo Allocator_StringTable = heap(); global AllocatorInfo Allocator_TypeTable = heap(); + #pragma endregion StaticData #pragma region Constants + global CodeType t_empty; global CodeType t_auto; global CodeType t_void; @@ -83,4 +86,5 @@ global CodeSpecifiers spec_static_member; global CodeSpecifiers spec_thread_local; global CodeSpecifiers spec_virtual; global CodeSpecifiers spec_volatile; + #pragma endregion Constants diff --git a/project/components/gen.etoktype.cpp b/project/components/gen.etoktype.cpp index 39c50a9..c20a285 100644 --- a/project/components/gen.etoktype.cpp +++ b/project/components/gen.etoktype.cpp @@ -9,6 +9,12 @@ namespace Parser Attributes_Start is only used to indicate the start of the user_defined attribute list. */ +#ifndef GEN_Define_Attribute_Tokens +# define GEN_Define_Attribute_Tokens \ + Entry( API_Export, "GEN_API_Export_Code" ) \ + Entry( API_Import, "GEN_API_Import_Code" ) +#endif + # define Define_TokType \ Entry( Invalid, "INVALID" ) \ Entry( Access_Private, "private" ) \ @@ -49,8 +55,12 @@ namespace Parser Entry( Module_Export, "export" ) \ Entry( Number, "number" ) \ Entry( Operator, "operator" ) \ - Entry( Preprocessor_Directive, "#") \ - Entry( Preprocessor_Include, "include" ) \ + Entry( Preprocess_Define, "#define") \ + Entry( Preproces_Include, "include" ) \ + Entry( Preprocess_If, "#if") \ + Entry( Preprocess_Elif, "#elif") \ + Entry( Preprocess_Else, "#else") \ + Entry( Preprocess_EndIf, "#endif") \ Entry( Spec_Alignas, "alignas" ) \ Entry( Spec_Const, "const" ) \ Entry( Spec_Consteval, "consteval" ) \ diff --git a/project/components/gen.header_end.hpp b/project/components/gen.header_end.hpp index ac408f9..d40961d 100644 --- a/project/components/gen.header_end.hpp +++ b/project/components/gen.header_end.hpp @@ -84,6 +84,7 @@ Code& Code::operator ++() } #pragma region AST & Code Gen Common + #define Define_CodeImpl( Typename ) \ char const* Typename::debug_str() \ { \ @@ -243,6 +244,7 @@ Define_CodeCast( Using ); Define_CodeCast( Var ); Define_CodeCast( Body); #undef Define_CodeCast + #pragma endregion AST & Code Gen Common void CodeClass::add_interface( CodeType type ) @@ -361,9 +363,11 @@ StrC token_fmt_impl( sw num, ... ) return { result, buf }; } + #pragma endregion Inlines #pragma region Constants + #ifdef GEN_DEFINE_LIBRARY_CODE_CONSTANTS // Predefined typename codes. Are set to readonly and are setup during gen::init() @@ -477,9 +481,11 @@ extern CodeSpecifiers spec_static_member; extern CodeSpecifiers spec_thread_local; extern CodeSpecifiers spec_virtual; extern CodeSpecifiers spec_volatile; + #pragma endregion Constants #pragma region Macros + # define gen_main main # define __ NoCode @@ -498,9 +504,11 @@ extern CodeSpecifiers spec_volatile; // Takes a format string (char const*) and a list of tokens (StrC) and returns a StrC of the formatted string. # define token_fmt( ... ) gen::token_fmt_impl( (num_args( __VA_ARGS__ ) + 1) / 2, __VA_ARGS__ ) + #pragma endregion Macros #ifdef GEN_EXPOSE_BACKEND + // Global allocator used for data with process lifetime. extern AllocatorInfo GlobalAllocator; extern Array< Arena > Global_AllocatorBuckets; @@ -517,4 +525,5 @@ extern CodeSpecifiers spec_volatile; extern AllocatorInfo Allocator_StringArena; extern AllocatorInfo Allocator_StringTable; extern AllocatorInfo Allocator_TypeTable; + #endif diff --git a/project/components/gen.interface.parsing.cpp b/project/components/gen.interface.parsing.cpp index ac7dcdc..f7f4445 100644 --- a/project/components/gen.interface.parsing.cpp +++ b/project/components/gen.interface.parsing.cpp @@ -4,13 +4,21 @@ These constructors are the most implementation intensive other than the editor o namespace Parser { - struct Token { + // TokType Type; + // s32 Start; + // s32 End; + // s32 Line; + // s32 Column; + // TokFlags Flags; + char const* Text; sptr Length; TokType Type; bool IsAssign; + s32 Line; + s32 Column; operator bool() { @@ -21,60 +29,42 @@ namespace Parser { return { Length, Text }; } + + bool is_access_specifier() + { + return Type >= TokType::Access_Private && Type <= TokType::Access_Public; + } + + bool is_attribute() + { + return Type > TokType::Attributes_Start; + } + + bool is_preprocessor() + { + return Type >= TokType::Preprocess_Define && Type <= TokType::Preprocess_EndIf; + } + + bool is_specifier() + { + return (Type <= TokType::Star && Type >= TokType::Spec_Alignas) + || Type == TokType::Ampersand + || Type == TokType::Ampersand_DBL + ; + } + + AccessSpec to_access_specifier() + { + return scast(AccessSpec, Type); + } }; - internal inline - bool tok_is_specifier( Token const& tok ) - { - return (tok.Type <= TokType::Star && tok.Type >= TokType::Spec_Alignas) - || tok.Type == TokType::Ampersand - || tok.Type == TokType::Ampersand_DBL - ; - } - - internal inline - bool tok_is_access_specifier( Token const& tok ) - { - return tok.Type >= TokType::Access_Private && tok.Type <= TokType::Access_Public; - } - - internal inline - AccessSpec tok_to_access_specifier( Token const& tok ) - { - return scast(AccessSpec, tok.Type); - } - - internal inline - bool tok_is_attribute( Token const& tok ) - { - return tok.Type > TokType::Attributes_Start; - } - struct TokArray { Array Arr; s32 Idx; - bool __eat( TokType type, char const* context ) - { - if ( Arr.num() - Idx <= 0 ) - { - log_failure( "gen::%s: No tokens left", context ); - return false; - } - - if ( Arr[Idx].Type != type ) - { - String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } ); - - log_failure( "gen::%s: expected %s, got %s", context, ETokType::to_str(type), ETokType::to_str(Arr[Idx].Type) ); - - return false; - } - - Idx++; - return true; - } + bool __eat( TokType type ); Token& current() { @@ -92,13 +82,87 @@ namespace Parser } }; - TokArray lex( StrC content, bool keep_preprocess_directives = false ) + struct StackNode + { + StackNode* Prev; + + Token Name; // The name of the AST node (if parsed) + StrC ProcName; // The name of the procedure + }; + + struct ParseContext + { + TokArray Tokens; + StackNode* Scope; + + String to_string() + { + String result = String::make_reserve( GlobalAllocator, kilobytes(4) ); + + result.append_fmt("\tContext:\n"); + + StackNode* current = Scope; + do + { + String name = String::make( GlobalAllocator, current->Name ? (StrC)current->Name : txt_StrC("Unresolved") ); + + result.append_fmt("\tProcedure: %s, AST Name: %s\n\t(%d, %d):", current->ProcName, name ); + current = current->Prev; + + name.free(); + } + while ( current ); + + return result; + } + }; + + global ParseContext Context; + + bool TokArray::__eat( TokType type ) + { + if ( Arr.num() - Idx <= 0 ) + { + log_failure( "No tokens left\n", Context.Scope->ProcName ); + return false; + } + + if ( Arr[Idx].Type != type ) + { + String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } ); + + log_failure( "gen::%s: expected %s, got %s", Context.Scope->ProcName, ETokType::to_str(type), ETokType::to_str(Arr[Idx].Type) ); + + return false; + } + + Idx++; + return true; + } + + enum TokFlags : u32 + { + IsAssign = bit(0), + }; + + TokArray lex( StrC content, bool keep_preprocess_directives = true ) { # define current ( * scanner ) - # define move_forward() \ - left--; \ - scanner++ + # define move_forward() \ + { \ + if ( current == '\n' ) \ + { \ + line++; \ + column = 0; \ + } \ + else \ + { \ + column++; \ + } \ + left--; \ + scanner++; \ + } # define SkipWhitespace() \ while ( left && char_is_space( current ) ) \ @@ -126,6 +190,9 @@ namespace Parser char const* word = scanner; s32 word_length = 0; + s32 line = 0; + s32 column = 0; + SkipWhitespace(); if ( left <= 0 ) { @@ -142,7 +209,7 @@ namespace Parser while (left ) { - Token token = { nullptr, 0, TokType::Invalid, false }; + Token token = { nullptr, 0, TokType::Invalid, false, line, column }; SkipWhitespace(); if ( left <= 0 ) @@ -153,11 +220,15 @@ namespace Parser case '#': token.Text = scanner; token.Length = 1; - token.Type = TokType::Preprocessor_Directive; move_forward(); while (left && current != '\n' ) { + if ( token.Type == ETokType::Invalid && current == ' ' ) + { + token.Type = ETokType::to_type( token ); + } + if ( current == '\\' ) { move_forward(); @@ -178,8 +249,9 @@ namespace Parser token.Length = 1; token.Type = TokType::Access_MemberSymbol; - if (left) + if (left) { move_forward(); + } if ( current == '.' ) { @@ -577,7 +649,7 @@ namespace Parser if ( token.Type != TokType::Invalid ) { - if ( token.Type == TokType::Preprocessor_Directive && keep_preprocess_directives == false ) + if ( token.is_preprocessor() && keep_preprocess_directives == false ) continue; Tokens.append( token ); @@ -608,6 +680,7 @@ namespace Parser } #pragma region Helper Macros + # define check_parse_args( func, def ) \ if ( def.Len <= 0 ) \ { \ @@ -620,13 +693,16 @@ if ( def.Ptr == nullptr ) \ return CodeInvalid; \ } -# define nexttok toks.next() -# define currtok toks.current() -# define prevtok toks.previous() -# define eat( Type_ ) toks.__eat( Type_, context ) -# define left ( toks.Arr.num() - toks.Idx ) +# define nexttok Context.Tokens.next() +# define currtok Context.Tokens.current() +# define prevtok Context.Tokens.previous() +# define eat( Type_ ) Context.Tokens.__eat( Type_ ) +# define left ( Context.Tokens.Arr.num() - Context.Tokens.Idx ) # define check( Type_ ) ( left && currtok.Type == Type_ ) + +// # define + #pragma endregion Helper Macros struct ParseContext @@ -635,28 +711,28 @@ struct ParseContext char const* Fn; }; -internal Code parse_function_body( Parser::TokArray& toks, char const* context ); -internal Code parse_global_nspace( Parser::TokArray& toks, char const* context ); +internal Code parse_function_body(); +internal Code parse_global_nspace(); -internal CodeClass parse_class ( Parser::TokArray& toks, char const* context ); -internal CodeEnum parse_enum ( Parser::TokArray& toks, char const* context ); -internal CodeBody parse_export_body ( Parser::TokArray& toks, char const* context ); -internal CodeBody parse_extern_link_body( Parser::TokArray& toks, char const* context ); -internal CodeExtern parse_exten_link ( Parser::TokArray& toks, char const* context ); -internal CodeFriend parse_friend ( Parser::TokArray& toks, char const* context ); -internal CodeFn parse_function ( Parser::TokArray& toks, char const* context ); -internal CodeNamespace parse_namespace ( Parser::TokArray& toks, char const* context ); -internal CodeOpCast parse_operator_cast ( Parser::TokArray& toks, char const* context ); -internal CodeStruct parse_struct ( Parser::TokArray& toks, char const* context ); -internal CodeVar parse_variable ( Parser::TokArray& toks, char const* context ); -internal CodeTemplate parse_template ( Parser::TokArray& toks, char const* context ); -internal CodeType parse_type ( Parser::TokArray& toks, char const* context ); -internal CodeTypedef parse_typedef ( Parser::TokArray& toks, char const* context ); -internal CodeUnion parse_union ( Parser::TokArray& toks, char const* context ); -internal CodeUsing parse_using ( Parser::TokArray& toks, char const* context ); +internal CodeClass parse_class (); +internal CodeEnum parse_enum (); +internal CodeBody parse_export_body (); +internal CodeBody parse_extern_link_body(); +internal CodeExtern parse_exten_link (); +internal CodeFriend parse_friend (); +internal CodeFn parse_function (); +internal CodeNamespace parse_namespace (); +internal CodeOpCast parse_operator_cast (); +internal CodeStruct parse_struct (); +internal CodeVar parse_variable (); +internal CodeTemplate parse_template (); +internal CodeType parse_type (); +internal CodeTypedef parse_typedef (); +internal CodeUnion parse_union (); +internal CodeUsing parse_using (); internal inline -Code parse_array_decl( Parser::TokArray& toks, char const* context ) +Code parse_array_decl() { using namespace Parser; @@ -707,7 +783,7 @@ Code parse_array_decl( Parser::TokArray& toks, char const* context ) } internal inline -CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) +CodeAttributes parse_attributes() { using namespace Parser; @@ -759,7 +835,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; } - else if ( tok_is_attribute( currtok ) ) + else if ( currtok.is_attribute() ) { eat(currtok.Type); s32 len = start.Length; @@ -775,7 +851,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) } internal inline -Parser::Token parse_identifier( Parser::TokArray& toks, char const* context ) +Parser::Token parse_identifier() { using namespace Parser; Token name = currtok; @@ -788,7 +864,7 @@ Parser::Token parse_identifier( Parser::TokArray& toks, char const* context ) if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of type definition, expected identifier", context ); + log_failure( "%s: Error, unexpected end of type definition, expected identifier", Context.to_string() ); return { nullptr, 0, TokType::Invalid }; } @@ -806,7 +882,7 @@ Parser::Token parse_identifier( Parser::TokArray& toks, char const* context ) } internal -CodeParam parse_params( Parser::TokArray& toks, char const* context, bool use_template_capture = false ) +CodeParam parse_params( bool use_template_capture = false ) { using namespace Parser; using namespace ECode; @@ -973,15 +1049,13 @@ CodeFn parse_function_after_name( , CodeSpecifiers specifiers , CodeType ret_type , StrC name - , Parser::TokArray& toks - , char const* context ) { using namespace Parser; CodeParam params = parse_params( toks, stringize(parse_function) ); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { specifiers.append( ESpecifier::to_type(currtok) ); eat( currtok.Type ); @@ -1041,12 +1115,12 @@ CodeFn parse_function_after_name( } internal inline -CodeOperator parse_operator_after_ret_type( ModuleFlag mflags +CodeOperator parse_operator_after_ret_type( + ModuleFlag mflags , CodeAttributes attributes , CodeSpecifiers specifiers , CodeType ret_type - , Parser::TokArray& toks - , char const* context ) +) { using namespace Parser; using namespace EOperator; @@ -1251,7 +1325,7 @@ CodeOperator parse_operator_after_ret_type( ModuleFlag mflags // Parse Params CodeParam params = parse_params( toks, stringize(parse_operator) ); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { specifiers.append( ESpecifier::to_type(currtok) ); eat( currtok.Type ); @@ -1341,7 +1415,7 @@ CodeVar parse_variable_after_name( } internal inline -Code parse_variable_assignment( Parser::TokArray& toks, char const* context ) +Code parse_variable_assignment() { using namespace Parser; @@ -1372,7 +1446,7 @@ Code parse_variable_assignment( Parser::TokArray& toks, char const* context ) } internal inline -Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers, Parser::TokArray& toks, char const* context ) +Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ) { using namespace Parser; @@ -1416,7 +1490,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes } internal -CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks, char const* context ) +CodeBody parse_class_struct_body( Parser::TokType which ) { using namespace Parser; using namespace ECode; @@ -1508,7 +1582,7 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks, GEN_Define_Attribute_Tokens #undef Entry { - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); } //! Fallthrough intended case TokType::Spec_Consteval: @@ -1522,7 +1596,7 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks, SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -1597,7 +1671,7 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks, } internal -Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char const* context ) +Code parse_class_struct( Parser::TokType which ) { using namespace Parser; @@ -1625,7 +1699,7 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con eat( which ); - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); if ( check( TokType::Identifier ) ) name = parse_identifier( toks, context ); @@ -1638,9 +1712,9 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con { eat( TokType::Assign_Classifer ); - if ( tok_is_access_specifier( currtok ) ) + if ( currtok.is_access_specifier() ) { - access = tok_to_access_specifier( currtok ); + access = currtok.to_access_specifier(); } Token parent_tok = parse_identifier( toks, context ); @@ -1650,7 +1724,7 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con { eat(TokType::Access_Public); - if ( tok_is_access_specifier( currtok ) ) + if ( currtok.is_access_specifier() ) { eat(currtok.Type); } @@ -1685,7 +1759,7 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con } internal -Code parse_function_body( Parser::TokArray& toks, char const* context ) +Code parse_function_body() { using namespace Parser; using namespace ECode; @@ -1724,7 +1798,7 @@ Code parse_function_body( Parser::TokArray& toks, char const* context ) } internal -CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* context ) +CodeBody parse_global_nspace( CodeT which ) { using namespace Parser; using namespace ECode; @@ -1811,7 +1885,7 @@ CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* c GEN_Define_Attribute_Tokens #undef Entry { - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); } //! Fallthrough intentional case TokType::Spec_Consteval: @@ -1826,7 +1900,7 @@ CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* c SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -1890,7 +1964,7 @@ CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* c } internal -CodeClass parse_class( Parser::TokArray& toks, char const* context ) +CodeClass parse_class() { return (CodeClass) parse_class_struct( Parser::TokType::Decl_Class, toks, context ); } @@ -1908,7 +1982,7 @@ CodeClass parse_class( StrC def ) } internal -CodeEnum parse_enum( Parser::TokArray& toks, char const* context ) +CodeEnum parse_enum() { using namespace Parser; using namespace ECode; @@ -2027,7 +2101,7 @@ CodeEnum parse_enum( StrC def ) } internal inline -CodeBody parse_export_body( Parser::TokArray& toks, char const* context ) +CodeBody parse_export_body() { return parse_global_nspace( ECode::Export_Body, toks, context ); } @@ -2170,9 +2244,9 @@ CodeFn parse_functon( Parser::TokArray& toks, char const* context ) eat( TokType::Module_Export ); } - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2294,9 +2368,9 @@ CodeOperator parse_operator( Parser::TokArray& toks, char const* context ) eat( TokType::Module_Export ); } - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2492,9 +2566,9 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context ) SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; - attributes = parse_attributes( toks, stringize(parse_template) ); + attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2576,9 +2650,9 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) Token name = { nullptr, 0, TokType::Invalid }; Token brute_sig = { currtok.Text, 0, TokType::Invalid }; - CodeAttributes attributes = parse_attributes( toks, context ); + CodeAttributes attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2651,7 +2725,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) } } - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2842,7 +2916,7 @@ CodeUnion parse_union( Parser::TokArray& toks, char const* context ) eat( TokType::Decl_Union ); - CodeAttributes attributes = parse_attributes( toks, context ); + CodeAttributes attributes = parse_attributes(); StrC name = { 0, nullptr }; @@ -2986,12 +3060,10 @@ CodeUsing parse_using( StrC def ) } internal -CodeVar parse_variable( Parser::TokArray& toks, char const* context ) +CodeVar parse_variable() { using namespace Parser; - Token name = { nullptr, 0, TokType::Invalid }; - SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; @@ -3005,7 +3077,7 @@ CodeVar parse_variable( Parser::TokArray& toks, char const* context ) eat( TokType::Module_Export ); } - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); while ( left && tok_is_specifier( currtok ) ) { @@ -3050,24 +3122,32 @@ CodeVar parse_variable( Parser::TokArray& toks, char const* context ) if ( type == Code::Invalid ) return CodeInvalid; - name = currtok; + Context.Scope->Name = current; eat( TokType::Identifier ); - CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, name, toks, context ); + CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, Context.Scope->Name, Context.Tokens, Context.Scope->ProcName ); return result; } CodeVar parse_variable( StrC def ) { - check_parse_args( parse_variable, def ); using namespace Parser; + check_parse_args( parse_variable, def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_variable( toks, stringize(parse_variable) ); + Context.Tokens = toks; + Parser::StackNode root + { + toks.current(), + { nullptr, 0, TokType::Invalid }, + name(parse_variable) + }; + + return parse_variable(); } // Undef helper macros @@ -3075,4 +3155,3 @@ CodeVar parse_variable( StrC def ) # undef curr_tok # undef eat # undef left - diff --git a/project/components/gen.interface.upfront.cpp b/project/components/gen.interface.upfront.cpp index 90adc67..8245157 100644 --- a/project/components/gen.interface.upfront.cpp +++ b/project/components/gen.interface.upfront.cpp @@ -1338,7 +1338,7 @@ CodeBody def_class_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_class_body ); - AST_BODY_CLASS_UNALLOWED_TYPES + GEN_AST_BODY_CLASS_UNALLOWED_TYPES def_body_code_validation_end( def_class_body ); va_end(va); @@ -1354,7 +1354,7 @@ CodeBody def_class_body( s32 num, Code* codes ) result->Type = Function_Body; def_body_code_array_validation_start( def_class_body ); - AST_BODY_CLASS_UNALLOWED_TYPES + GEN_AST_BODY_CLASS_UNALLOWED_TYPES def_body_code_validation_end( def_class_body ); return result; @@ -1437,7 +1437,7 @@ CodeBody def_export_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_export_body ); - AST_BODY_EXPORT_UNALLOWED_TYPES + GEN_AST_BODY_EXPORT_UNALLOWED_TYPES def_body_code_validation_end( def_export_body ); va_end(va); @@ -1453,7 +1453,7 @@ CodeBody def_export_body( s32 num, Code* codes ) result->Type = Export_Body; def_body_code_array_validation_start( def_export_body ); - AST_BODY_EXPORT_UNALLOWED_TYPES + GEN_AST_BODY_EXPORT_UNALLOWED_TYPES def_body_code_validation_end( def_export_body ); return result; @@ -1470,7 +1470,7 @@ CodeBody def_extern_link_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_extern_linkage_body ); - AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES + GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES def_body_code_validation_end( def_extern_linkage_body ); va_end(va); @@ -1486,7 +1486,7 @@ CodeBody def_extern_link_body( s32 num, Code* codes ) result->Type = Extern_Linkage_Body; def_body_code_array_validation_start( def_extern_linkage_body ); - AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES + GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES def_body_code_validation_end( def_extern_linkage_body ); return result; @@ -1503,7 +1503,7 @@ CodeBody def_function_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_function_body ); - AST_BODY_FUNCTION_UNALLOWED_TYPES + GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES def_body_code_validation_end( def_function_body ); va_end(va); @@ -1519,7 +1519,7 @@ CodeBody def_function_body( s32 num, Code* codes ) result->Type = Function_Body; def_body_code_array_validation_start( def_function_body ); - AST_BODY_FUNCTION_UNALLOWED_TYPES + GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES def_body_code_validation_end( def_function_body ); return result; @@ -1536,7 +1536,7 @@ CodeBody def_global_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_global_body ); - AST_BODY_GLOBAL_UNALLOWED_TYPES + GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES def_body_code_validation_end( def_global_body ); va_end(va); @@ -1552,7 +1552,7 @@ CodeBody def_global_body( s32 num, Code* codes ) result->Type = Global_Body; def_body_code_array_validation_start( def_global_body ); - AST_BODY_GLOBAL_UNALLOWED_TYPES + GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES def_body_code_validation_end( def_global_body ); return result; @@ -1569,7 +1569,7 @@ CodeBody def_namespace_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_namespace_body ); - AST_BODY_NAMESPACE_UNALLOWED_TYPES + GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES def_body_code_validation_end( def_namespace_body ); va_end(va); @@ -1585,7 +1585,7 @@ CodeBody def_namespace_body( s32 num, Code* codes ) result->Type = Global_Body; def_body_code_array_validation_start( def_namespace_body ); - AST_BODY_NAMESPACE_UNALLOWED_TYPES + GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES def_body_code_validation_end( def_namespace_body ); return result; @@ -1737,7 +1737,7 @@ CodeBody def_struct_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_struct_body ); - AST_BODY_STRUCT_UNALLOWED_TYPES + GEN_AST_BODY_STRUCT_UNALLOWED_TYPES def_body_code_validation_end( def_struct_body ); va_end(va); @@ -1753,7 +1753,7 @@ CodeBody def_struct_body( s32 num, Code* codes ) result->Type = Struct_Body; def_body_code_array_validation_start( def_struct_body ); - AST_BODY_STRUCT_UNALLOWED_TYPES + GEN_AST_BODY_STRUCT_UNALLOWED_TYPES def_body_code_validation_end( def_struct_body ); return result; diff --git a/project/components/gen.types.hpp b/project/components/gen.types.hpp index bda942a..20f201f 100644 --- a/project/components/gen.types.hpp +++ b/project/components/gen.types.hpp @@ -100,10 +100,3 @@ constexpr char const* Attribute_Keyword = stringize( GEN_Attribute_Keyword ); constexpr char const* Attribute_Keyword = ""; #endif - -#ifndef GEN_Define_Attribute_Tokens -# define GEN_Define_Attribute_Tokens \ - Entry( API_Export, "GEN_API_Export_Code" ) \ - Entry( API_Import, "GEN_API_Import_Code" ) -#endif - diff --git a/project/helpers/gen.helper.hpp b/project/helpers/gen.helper.hpp index d0e257f..2270899 100644 --- a/project/helpers/gen.helper.hpp +++ b/project/helpers/gen.helper.hpp @@ -6,7 +6,7 @@ using namespace gen; CodeBody gen_ecode( char const* path ) { - char scratch_mem[kilobytes(1)]; + char scratch_mem[kilobytes(1)]; Arena scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) ); file_read_contents( scratch, zero_terminate, path ); @@ -50,8 +50,7 @@ CodeBody gen_ecode( char const* path ) #pragma pop_macro( "local_persist" ) CodeNamespace nspace = def_namespace( name(ECode), def_namespace_body( args( enum_code, to_str ) ) ); - - CodeUsing code_t = def_using( name(CodeT), def_type( name(ECode::Type) ) ); + CodeUsing code_t = def_using( name(CodeT), def_type( name(ECode::Type) ) ); return def_global_body( args( nspace, code_t ) ); } @@ -209,7 +208,7 @@ CodeBody gen_especifier( char const* path ) CodeBody gen_etoktype( char const* etok_path, char const* attr_path ) { - char scratch_mem[kilobytes(64)]; + char scratch_mem[kilobytes(64)]; Arena scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) ); FileContents enum_content = file_read_contents( scratch, zero_terminate, etok_path ); @@ -217,8 +216,6 @@ CodeBody gen_etoktype( char const* etok_path, char const* attr_path ) CSV_Object csv_enum_nodes; csv_parse( &csv_enum_nodes, rcast(char*, enum_content.data), GlobalAllocator, false ); - // memset( scratch_mem, 0, sizeof(scratch_mem) ); - // scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) ); FileContents attrib_content = file_read_contents( scratch, zero_terminate, attr_path ); CSV_Object csv_attr_nodes;