From 31e1c38c1833eee5d30650ff1324938e0e030542 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 28 Jul 2023 21:44:31 -0400 Subject: [PATCH 01/10] Started to implement context stack for parser. --- Readme.md | 30 +- project/components/ETokType.csv | 8 +- project/components/gen.ast.cpp | 14 +- project/components/gen.ast_case_macros.cpp | 152 +++++---- project/components/gen.data.cpp | 4 + project/components/gen.etoktype.cpp | 14 +- project/components/gen.header_end.hpp | 9 + project/components/gen.interface.parsing.cpp | 335 ++++++++++++------- project/components/gen.interface.upfront.cpp | 28 +- project/components/gen.types.hpp | 7 - project/helpers/gen.helper.hpp | 9 +- 11 files changed, 350 insertions(+), 260 deletions(-) diff --git a/Readme.md b/Readme.md index 49f6ef6..9d91958 100644 --- a/Readme.md +++ b/Readme.md @@ -5,11 +5,6 @@ An attempt at simple staged metaprogramming for c/c++. The library API is a composition of code element constructors. These build up a code AST to then serialize with a file builder. -General goal is to have a less than 15k sloc library that takes at most a couple of hours to learn and make use of. - -*Why 15k ?* Assuming a seasoned coder of C++ can read and understand around 1000-2000 lines of code per hour, 15,000 could be understood in under 16-18 hours -and have confidence in modifying for their use case. - This code base attempts follow the [handmade philosophy](https://handmade.network/manifesto), its not meant to be a black box metaprogramming utility, its meant for the user to extend for their project domain. @@ -30,17 +25,17 @@ The project has reached an *alpha* state, all the current functionality works fo The project has no external dependencies beyond: -* `errno.h` (gen.dep.cpp) -* `stat.h` (gen.dep.cpp) -* `stdarg.h` (gen.dep.hpp) -* `stddef.h` (gen.dep.hpp -* `stdio.h` (gen.dep.cpp) -* `copyfile.h` (Mac, gen.dep.cpp) -* `types.h` (Linux, gen.dep.cpp) -* `unistd.h` (Linux/Mac, gen.dep.cpp) -* `intrin.h` (Windows, gen.dep.hpp) -* `io.h` (Windows with gcc, gen.dep.cpp) -* `windows.h` (Windows, gen.dep.cpp) +* `errno.h` +* `stat.h` +* `stdarg.h` +* `stddef.h` +* `stdio.h` +* `copyfile.h` (Mac) +* `types.h` (Linux) +* `unistd.h` (Linux/Mac) +* `intrin.h` (Windows) +* `io.h` (Windows with gcc) +* `windows.h` (Windows) Dependencies for the project are wrapped within `GENCPP_ROLL_OWN_DEPENDENCIES` (Defining it will disable them). The majority of the dependency's implementation was derived from the [c-zpl library](https://github.com/zpl-c/zpl). @@ -561,6 +556,7 @@ The following are provided predefined by the library as they are commonly used: * `access_private` * `module_global_fragment` * `module_private_fragment` +* `param_varaidc` (Used for varadic definitions) * `pragma_once` * `spec_const` * `spec_consteval` @@ -586,7 +582,7 @@ The following are provided predefined by the library as they are commonly used: * `spec_type_unsigned` * `spec_type_short` * `spec_type_long` -* `t_empty` +* `t_empty` (Used for varaidc macros) * `t_auto` * `t_void` * `t_int` diff --git a/project/components/ETokType.csv b/project/components/ETokType.csv index be89940..2056739 100644 --- a/project/components/ETokType.csv +++ b/project/components/ETokType.csv @@ -37,8 +37,12 @@ Module_Import, "import" Module_Export, "export" Number, "number" Operator, "operator" -Preprocessor_Directive, "#" -Preprocessor_Include, "include" +Preprocess_Define, "#define" +Preprocess_Include, "#include" +Preprocess_If, "#if" +Preprocess_ElIF, "#elif" +Preprocess_Else, "#else" +Preprocess_EndIf, "#endif" Spec_Alignas, "alignas" Spec_Const, "const" Spec_Consteval, "consteval" diff --git a/project/components/gen.ast.cpp b/project/components/gen.ast.cpp index c576ec3..e9054aa 100644 --- a/project/components/gen.ast.cpp +++ b/project/components/gen.ast.cpp @@ -778,7 +778,7 @@ bool AST::validate_body() switch ( Type ) { case Class_Body: - CheckEntries( AST_BODY_CLASS_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_CLASS_UNALLOWED_TYPES ); break; case Enum_Body: for ( Code entry : cast() ) @@ -791,22 +791,22 @@ bool AST::validate_body() } break; case Export_Body: - CheckEntries( AST_BODY_CLASS_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_CLASS_UNALLOWED_TYPES ); break; case Extern_Linkage: - CheckEntries( AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES ); break; case Function_Body: - CheckEntries( AST_BODY_FUNCTION_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES ); break; case Global_Body: - CheckEntries( AST_BODY_GLOBAL_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES ); break; case Namespace_Body: - CheckEntries( AST_BODY_NAMESPACE_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES ); break; case Struct_Body: - CheckEntries( AST_BODY_STRUCT_UNALLOWED_TYPES ); + CheckEntries( GEN_AST_BODY_STRUCT_UNALLOWED_TYPES ); break; case Union_Body: for ( Code entry : Body->cast() ) diff --git a/project/components/gen.ast_case_macros.cpp b/project/components/gen.ast_case_macros.cpp index 83e18e1..fa47e8b 100644 --- a/project/components/gen.ast_case_macros.cpp +++ b/project/components/gen.ast_case_macros.cpp @@ -1,81 +1,79 @@ -# define AST_BODY_CLASS_UNALLOWED_TYPES \ - case PlatformAttributes: \ - case Class_Body: \ - case Enum_Body: \ - case Extern_Linkage: \ - case Function_Body: \ - case Function_Fwd: \ - case Global_Body: \ - case Namespace: \ - case Namespace_Body: \ - case Operator: \ - case Operator_Fwd: \ - case Parameters: \ - case Specifiers: \ - case Struct_Body: \ - case Typename: - -# define AST_BODY_FUNCTION_UNALLOWED_TYPES \ - case Access_Public: \ - case Access_Protected: \ - case Access_Private: \ - case PlatformAttributes: \ - case Class_Body: \ - case Enum_Body: \ - case Extern_Linkage: \ - case Friend: \ - case Function_Body: \ - case Function_Fwd: \ - case Global_Body: \ - case Namespace: \ - case Namespace_Body: \ - case Operator: \ - case Operator_Fwd: \ - case Operator_Member: \ - case Operator_Member_Fwd: \ - case Parameters: \ - case Specifiers: \ - case Struct_Body: \ - case Typename: - -# define AST_BODY_GLOBAL_UNALLOWED_TYPES \ - case Access_Public: \ - case Access_Protected: \ - case Access_Private: \ - case PlatformAttributes: \ - case Class_Body: \ - case Enum_Body: \ - case Execution: \ - case Friend: \ - case Function_Body: \ - case Global_Body: \ - case Namespace_Body: \ - case Operator_Member: \ - case Operator_Member_Fwd: \ - case Parameters: \ - case Specifiers: \ - case Struct_Body: \ - case Typename: - -# define AST_BODY_EXPORT_UNALLOWED_TYPES AST_BODY_GLOBAL_UNALLOWED_TYPES -# define AST_BODY_NAMESPACE_UNALLOWED_TYPES \ - case Access_Public: \ - case Access_Protected: \ - case Access_Private: \ +# define GEN_AST_BODY_CLASS_UNALLOWED_TYPES \ case PlatformAttributes: \ - case Class_Body: \ - case Enum_Body: \ - case Execution: \ - case Friend: \ - case Function_Body: \ - case Namespace_Body: \ - case Operator_Member: \ - case Operator_Member_Fwd: \ - case Parameters: \ - case Specifiers: \ - case Struct_Body: \ + case Class_Body: \ + case Enum_Body: \ + case Extern_Linkage: \ + case Function_Body: \ + case Function_Fwd: \ + case Global_Body: \ + case Namespace: \ + case Namespace_Body: \ + case Operator: \ + case Operator_Fwd: \ + case Parameters: \ + case Specifiers: \ + case Struct_Body: \ + case Typename: +# define GEN_AST_BODY_STRUCT_UNALLOWED_TYPES GEN_AST_BODY_CLASS_UNALLOWED_TYPES + +# define GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES \ + case Access_Public: \ + case Access_Protected: \ + case Access_Private: \ + case PlatformAttributes: \ + case Class_Body: \ + case Enum_Body: \ + case Extern_Linkage: \ + case Friend: \ + case Function_Body: \ + case Function_Fwd: \ + case Global_Body: \ + case Namespace: \ + case Namespace_Body: \ + case Operator: \ + case Operator_Fwd: \ + case Operator_Member: \ + case Operator_Member_Fwd: \ + case Parameters: \ + case Specifiers: \ + case Struct_Body: \ case Typename: -# define AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES AST_BODY_GLOBAL_UNALLOWED_TYPES +# define GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES \ + case Access_Public: \ + case Access_Protected: \ + case Access_Private: \ + case PlatformAttributes: \ + case Class_Body: \ + case Enum_Body: \ + case Execution: \ + case Friend: \ + case Function_Body: \ + case Global_Body: \ + case Namespace_Body: \ + case Operator_Member: \ + case Operator_Member_Fwd: \ + case Parameters: \ + case Specifiers: \ + case Struct_Body: \ + case Typename: +# define GEN_AST_BODY_EXPORT_UNALLOWED_TYPES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES +# define GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES -# define AST_BODY_STRUCT_UNALLOWED_TYPES AST_BODY_CLASS_UNALLOWED_TYPES +# define GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES \ + case Access_Public: \ + case Access_Protected: \ + case Access_Private: \ + case PlatformAttributes: \ + case Class_Body: \ + case Enum_Body: \ + case Execution: \ + case Friend: \ + case Function_Body: \ + case Namespace_Body: \ + case Operator_Member: \ + case Operator_Member_Fwd: \ + case Parameters: \ + case Specifiers: \ + case Struct_Body: \ + case Typename: diff --git a/project/components/gen.data.cpp b/project/components/gen.data.cpp index 313a485..99cbdcd 100644 --- a/project/components/gen.data.cpp +++ b/project/components/gen.data.cpp @@ -1,4 +1,5 @@ #pragma region StaticData + // TODO : Convert global allocation strategy to use a slab allocation strategy. global AllocatorInfo GlobalAllocator; global Array Global_AllocatorBuckets; @@ -16,9 +17,11 @@ global AllocatorInfo Allocator_Lexer = heap(); global AllocatorInfo Allocator_StringArena = heap(); global AllocatorInfo Allocator_StringTable = heap(); global AllocatorInfo Allocator_TypeTable = heap(); + #pragma endregion StaticData #pragma region Constants + global CodeType t_empty; global CodeType t_auto; global CodeType t_void; @@ -83,4 +86,5 @@ global CodeSpecifiers spec_static_member; global CodeSpecifiers spec_thread_local; global CodeSpecifiers spec_virtual; global CodeSpecifiers spec_volatile; + #pragma endregion Constants diff --git a/project/components/gen.etoktype.cpp b/project/components/gen.etoktype.cpp index 39c50a9..c20a285 100644 --- a/project/components/gen.etoktype.cpp +++ b/project/components/gen.etoktype.cpp @@ -9,6 +9,12 @@ namespace Parser Attributes_Start is only used to indicate the start of the user_defined attribute list. */ +#ifndef GEN_Define_Attribute_Tokens +# define GEN_Define_Attribute_Tokens \ + Entry( API_Export, "GEN_API_Export_Code" ) \ + Entry( API_Import, "GEN_API_Import_Code" ) +#endif + # define Define_TokType \ Entry( Invalid, "INVALID" ) \ Entry( Access_Private, "private" ) \ @@ -49,8 +55,12 @@ namespace Parser Entry( Module_Export, "export" ) \ Entry( Number, "number" ) \ Entry( Operator, "operator" ) \ - Entry( Preprocessor_Directive, "#") \ - Entry( Preprocessor_Include, "include" ) \ + Entry( Preprocess_Define, "#define") \ + Entry( Preproces_Include, "include" ) \ + Entry( Preprocess_If, "#if") \ + Entry( Preprocess_Elif, "#elif") \ + Entry( Preprocess_Else, "#else") \ + Entry( Preprocess_EndIf, "#endif") \ Entry( Spec_Alignas, "alignas" ) \ Entry( Spec_Const, "const" ) \ Entry( Spec_Consteval, "consteval" ) \ diff --git a/project/components/gen.header_end.hpp b/project/components/gen.header_end.hpp index ac408f9..d40961d 100644 --- a/project/components/gen.header_end.hpp +++ b/project/components/gen.header_end.hpp @@ -84,6 +84,7 @@ Code& Code::operator ++() } #pragma region AST & Code Gen Common + #define Define_CodeImpl( Typename ) \ char const* Typename::debug_str() \ { \ @@ -243,6 +244,7 @@ Define_CodeCast( Using ); Define_CodeCast( Var ); Define_CodeCast( Body); #undef Define_CodeCast + #pragma endregion AST & Code Gen Common void CodeClass::add_interface( CodeType type ) @@ -361,9 +363,11 @@ StrC token_fmt_impl( sw num, ... ) return { result, buf }; } + #pragma endregion Inlines #pragma region Constants + #ifdef GEN_DEFINE_LIBRARY_CODE_CONSTANTS // Predefined typename codes. Are set to readonly and are setup during gen::init() @@ -477,9 +481,11 @@ extern CodeSpecifiers spec_static_member; extern CodeSpecifiers spec_thread_local; extern CodeSpecifiers spec_virtual; extern CodeSpecifiers spec_volatile; + #pragma endregion Constants #pragma region Macros + # define gen_main main # define __ NoCode @@ -498,9 +504,11 @@ extern CodeSpecifiers spec_volatile; // Takes a format string (char const*) and a list of tokens (StrC) and returns a StrC of the formatted string. # define token_fmt( ... ) gen::token_fmt_impl( (num_args( __VA_ARGS__ ) + 1) / 2, __VA_ARGS__ ) + #pragma endregion Macros #ifdef GEN_EXPOSE_BACKEND + // Global allocator used for data with process lifetime. extern AllocatorInfo GlobalAllocator; extern Array< Arena > Global_AllocatorBuckets; @@ -517,4 +525,5 @@ extern CodeSpecifiers spec_volatile; extern AllocatorInfo Allocator_StringArena; extern AllocatorInfo Allocator_StringTable; extern AllocatorInfo Allocator_TypeTable; + #endif diff --git a/project/components/gen.interface.parsing.cpp b/project/components/gen.interface.parsing.cpp index ac7dcdc..f7f4445 100644 --- a/project/components/gen.interface.parsing.cpp +++ b/project/components/gen.interface.parsing.cpp @@ -4,13 +4,21 @@ These constructors are the most implementation intensive other than the editor o namespace Parser { - struct Token { + // TokType Type; + // s32 Start; + // s32 End; + // s32 Line; + // s32 Column; + // TokFlags Flags; + char const* Text; sptr Length; TokType Type; bool IsAssign; + s32 Line; + s32 Column; operator bool() { @@ -21,60 +29,42 @@ namespace Parser { return { Length, Text }; } + + bool is_access_specifier() + { + return Type >= TokType::Access_Private && Type <= TokType::Access_Public; + } + + bool is_attribute() + { + return Type > TokType::Attributes_Start; + } + + bool is_preprocessor() + { + return Type >= TokType::Preprocess_Define && Type <= TokType::Preprocess_EndIf; + } + + bool is_specifier() + { + return (Type <= TokType::Star && Type >= TokType::Spec_Alignas) + || Type == TokType::Ampersand + || Type == TokType::Ampersand_DBL + ; + } + + AccessSpec to_access_specifier() + { + return scast(AccessSpec, Type); + } }; - internal inline - bool tok_is_specifier( Token const& tok ) - { - return (tok.Type <= TokType::Star && tok.Type >= TokType::Spec_Alignas) - || tok.Type == TokType::Ampersand - || tok.Type == TokType::Ampersand_DBL - ; - } - - internal inline - bool tok_is_access_specifier( Token const& tok ) - { - return tok.Type >= TokType::Access_Private && tok.Type <= TokType::Access_Public; - } - - internal inline - AccessSpec tok_to_access_specifier( Token const& tok ) - { - return scast(AccessSpec, tok.Type); - } - - internal inline - bool tok_is_attribute( Token const& tok ) - { - return tok.Type > TokType::Attributes_Start; - } - struct TokArray { Array Arr; s32 Idx; - bool __eat( TokType type, char const* context ) - { - if ( Arr.num() - Idx <= 0 ) - { - log_failure( "gen::%s: No tokens left", context ); - return false; - } - - if ( Arr[Idx].Type != type ) - { - String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } ); - - log_failure( "gen::%s: expected %s, got %s", context, ETokType::to_str(type), ETokType::to_str(Arr[Idx].Type) ); - - return false; - } - - Idx++; - return true; - } + bool __eat( TokType type ); Token& current() { @@ -92,13 +82,87 @@ namespace Parser } }; - TokArray lex( StrC content, bool keep_preprocess_directives = false ) + struct StackNode + { + StackNode* Prev; + + Token Name; // The name of the AST node (if parsed) + StrC ProcName; // The name of the procedure + }; + + struct ParseContext + { + TokArray Tokens; + StackNode* Scope; + + String to_string() + { + String result = String::make_reserve( GlobalAllocator, kilobytes(4) ); + + result.append_fmt("\tContext:\n"); + + StackNode* current = Scope; + do + { + String name = String::make( GlobalAllocator, current->Name ? (StrC)current->Name : txt_StrC("Unresolved") ); + + result.append_fmt("\tProcedure: %s, AST Name: %s\n\t(%d, %d):", current->ProcName, name ); + current = current->Prev; + + name.free(); + } + while ( current ); + + return result; + } + }; + + global ParseContext Context; + + bool TokArray::__eat( TokType type ) + { + if ( Arr.num() - Idx <= 0 ) + { + log_failure( "No tokens left\n", Context.Scope->ProcName ); + return false; + } + + if ( Arr[Idx].Type != type ) + { + String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } ); + + log_failure( "gen::%s: expected %s, got %s", Context.Scope->ProcName, ETokType::to_str(type), ETokType::to_str(Arr[Idx].Type) ); + + return false; + } + + Idx++; + return true; + } + + enum TokFlags : u32 + { + IsAssign = bit(0), + }; + + TokArray lex( StrC content, bool keep_preprocess_directives = true ) { # define current ( * scanner ) - # define move_forward() \ - left--; \ - scanner++ + # define move_forward() \ + { \ + if ( current == '\n' ) \ + { \ + line++; \ + column = 0; \ + } \ + else \ + { \ + column++; \ + } \ + left--; \ + scanner++; \ + } # define SkipWhitespace() \ while ( left && char_is_space( current ) ) \ @@ -126,6 +190,9 @@ namespace Parser char const* word = scanner; s32 word_length = 0; + s32 line = 0; + s32 column = 0; + SkipWhitespace(); if ( left <= 0 ) { @@ -142,7 +209,7 @@ namespace Parser while (left ) { - Token token = { nullptr, 0, TokType::Invalid, false }; + Token token = { nullptr, 0, TokType::Invalid, false, line, column }; SkipWhitespace(); if ( left <= 0 ) @@ -153,11 +220,15 @@ namespace Parser case '#': token.Text = scanner; token.Length = 1; - token.Type = TokType::Preprocessor_Directive; move_forward(); while (left && current != '\n' ) { + if ( token.Type == ETokType::Invalid && current == ' ' ) + { + token.Type = ETokType::to_type( token ); + } + if ( current == '\\' ) { move_forward(); @@ -178,8 +249,9 @@ namespace Parser token.Length = 1; token.Type = TokType::Access_MemberSymbol; - if (left) + if (left) { move_forward(); + } if ( current == '.' ) { @@ -577,7 +649,7 @@ namespace Parser if ( token.Type != TokType::Invalid ) { - if ( token.Type == TokType::Preprocessor_Directive && keep_preprocess_directives == false ) + if ( token.is_preprocessor() && keep_preprocess_directives == false ) continue; Tokens.append( token ); @@ -608,6 +680,7 @@ namespace Parser } #pragma region Helper Macros + # define check_parse_args( func, def ) \ if ( def.Len <= 0 ) \ { \ @@ -620,13 +693,16 @@ if ( def.Ptr == nullptr ) \ return CodeInvalid; \ } -# define nexttok toks.next() -# define currtok toks.current() -# define prevtok toks.previous() -# define eat( Type_ ) toks.__eat( Type_, context ) -# define left ( toks.Arr.num() - toks.Idx ) +# define nexttok Context.Tokens.next() +# define currtok Context.Tokens.current() +# define prevtok Context.Tokens.previous() +# define eat( Type_ ) Context.Tokens.__eat( Type_ ) +# define left ( Context.Tokens.Arr.num() - Context.Tokens.Idx ) # define check( Type_ ) ( left && currtok.Type == Type_ ) + +// # define + #pragma endregion Helper Macros struct ParseContext @@ -635,28 +711,28 @@ struct ParseContext char const* Fn; }; -internal Code parse_function_body( Parser::TokArray& toks, char const* context ); -internal Code parse_global_nspace( Parser::TokArray& toks, char const* context ); +internal Code parse_function_body(); +internal Code parse_global_nspace(); -internal CodeClass parse_class ( Parser::TokArray& toks, char const* context ); -internal CodeEnum parse_enum ( Parser::TokArray& toks, char const* context ); -internal CodeBody parse_export_body ( Parser::TokArray& toks, char const* context ); -internal CodeBody parse_extern_link_body( Parser::TokArray& toks, char const* context ); -internal CodeExtern parse_exten_link ( Parser::TokArray& toks, char const* context ); -internal CodeFriend parse_friend ( Parser::TokArray& toks, char const* context ); -internal CodeFn parse_function ( Parser::TokArray& toks, char const* context ); -internal CodeNamespace parse_namespace ( Parser::TokArray& toks, char const* context ); -internal CodeOpCast parse_operator_cast ( Parser::TokArray& toks, char const* context ); -internal CodeStruct parse_struct ( Parser::TokArray& toks, char const* context ); -internal CodeVar parse_variable ( Parser::TokArray& toks, char const* context ); -internal CodeTemplate parse_template ( Parser::TokArray& toks, char const* context ); -internal CodeType parse_type ( Parser::TokArray& toks, char const* context ); -internal CodeTypedef parse_typedef ( Parser::TokArray& toks, char const* context ); -internal CodeUnion parse_union ( Parser::TokArray& toks, char const* context ); -internal CodeUsing parse_using ( Parser::TokArray& toks, char const* context ); +internal CodeClass parse_class (); +internal CodeEnum parse_enum (); +internal CodeBody parse_export_body (); +internal CodeBody parse_extern_link_body(); +internal CodeExtern parse_exten_link (); +internal CodeFriend parse_friend (); +internal CodeFn parse_function (); +internal CodeNamespace parse_namespace (); +internal CodeOpCast parse_operator_cast (); +internal CodeStruct parse_struct (); +internal CodeVar parse_variable (); +internal CodeTemplate parse_template (); +internal CodeType parse_type (); +internal CodeTypedef parse_typedef (); +internal CodeUnion parse_union (); +internal CodeUsing parse_using (); internal inline -Code parse_array_decl( Parser::TokArray& toks, char const* context ) +Code parse_array_decl() { using namespace Parser; @@ -707,7 +783,7 @@ Code parse_array_decl( Parser::TokArray& toks, char const* context ) } internal inline -CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) +CodeAttributes parse_attributes() { using namespace Parser; @@ -759,7 +835,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; } - else if ( tok_is_attribute( currtok ) ) + else if ( currtok.is_attribute() ) { eat(currtok.Type); s32 len = start.Length; @@ -775,7 +851,7 @@ CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context ) } internal inline -Parser::Token parse_identifier( Parser::TokArray& toks, char const* context ) +Parser::Token parse_identifier() { using namespace Parser; Token name = currtok; @@ -788,7 +864,7 @@ Parser::Token parse_identifier( Parser::TokArray& toks, char const* context ) if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of type definition, expected identifier", context ); + log_failure( "%s: Error, unexpected end of type definition, expected identifier", Context.to_string() ); return { nullptr, 0, TokType::Invalid }; } @@ -806,7 +882,7 @@ Parser::Token parse_identifier( Parser::TokArray& toks, char const* context ) } internal -CodeParam parse_params( Parser::TokArray& toks, char const* context, bool use_template_capture = false ) +CodeParam parse_params( bool use_template_capture = false ) { using namespace Parser; using namespace ECode; @@ -973,15 +1049,13 @@ CodeFn parse_function_after_name( , CodeSpecifiers specifiers , CodeType ret_type , StrC name - , Parser::TokArray& toks - , char const* context ) { using namespace Parser; CodeParam params = parse_params( toks, stringize(parse_function) ); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { specifiers.append( ESpecifier::to_type(currtok) ); eat( currtok.Type ); @@ -1041,12 +1115,12 @@ CodeFn parse_function_after_name( } internal inline -CodeOperator parse_operator_after_ret_type( ModuleFlag mflags +CodeOperator parse_operator_after_ret_type( + ModuleFlag mflags , CodeAttributes attributes , CodeSpecifiers specifiers , CodeType ret_type - , Parser::TokArray& toks - , char const* context ) +) { using namespace Parser; using namespace EOperator; @@ -1251,7 +1325,7 @@ CodeOperator parse_operator_after_ret_type( ModuleFlag mflags // Parse Params CodeParam params = parse_params( toks, stringize(parse_operator) ); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { specifiers.append( ESpecifier::to_type(currtok) ); eat( currtok.Type ); @@ -1341,7 +1415,7 @@ CodeVar parse_variable_after_name( } internal inline -Code parse_variable_assignment( Parser::TokArray& toks, char const* context ) +Code parse_variable_assignment() { using namespace Parser; @@ -1372,7 +1446,7 @@ Code parse_variable_assignment( Parser::TokArray& toks, char const* context ) } internal inline -Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers, Parser::TokArray& toks, char const* context ) +Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ) { using namespace Parser; @@ -1416,7 +1490,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes } internal -CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks, char const* context ) +CodeBody parse_class_struct_body( Parser::TokType which ) { using namespace Parser; using namespace ECode; @@ -1508,7 +1582,7 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks, GEN_Define_Attribute_Tokens #undef Entry { - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); } //! Fallthrough intended case TokType::Spec_Consteval: @@ -1522,7 +1596,7 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks, SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -1597,7 +1671,7 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks, } internal -Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char const* context ) +Code parse_class_struct( Parser::TokType which ) { using namespace Parser; @@ -1625,7 +1699,7 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con eat( which ); - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); if ( check( TokType::Identifier ) ) name = parse_identifier( toks, context ); @@ -1638,9 +1712,9 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con { eat( TokType::Assign_Classifer ); - if ( tok_is_access_specifier( currtok ) ) + if ( currtok.is_access_specifier() ) { - access = tok_to_access_specifier( currtok ); + access = currtok.to_access_specifier(); } Token parent_tok = parse_identifier( toks, context ); @@ -1650,7 +1724,7 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con { eat(TokType::Access_Public); - if ( tok_is_access_specifier( currtok ) ) + if ( currtok.is_access_specifier() ) { eat(currtok.Type); } @@ -1685,7 +1759,7 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con } internal -Code parse_function_body( Parser::TokArray& toks, char const* context ) +Code parse_function_body() { using namespace Parser; using namespace ECode; @@ -1724,7 +1798,7 @@ Code parse_function_body( Parser::TokArray& toks, char const* context ) } internal -CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* context ) +CodeBody parse_global_nspace( CodeT which ) { using namespace Parser; using namespace ECode; @@ -1811,7 +1885,7 @@ CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* c GEN_Define_Attribute_Tokens #undef Entry { - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); } //! Fallthrough intentional case TokType::Spec_Consteval: @@ -1826,7 +1900,7 @@ CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* c SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -1890,7 +1964,7 @@ CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* c } internal -CodeClass parse_class( Parser::TokArray& toks, char const* context ) +CodeClass parse_class() { return (CodeClass) parse_class_struct( Parser::TokType::Decl_Class, toks, context ); } @@ -1908,7 +1982,7 @@ CodeClass parse_class( StrC def ) } internal -CodeEnum parse_enum( Parser::TokArray& toks, char const* context ) +CodeEnum parse_enum() { using namespace Parser; using namespace ECode; @@ -2027,7 +2101,7 @@ CodeEnum parse_enum( StrC def ) } internal inline -CodeBody parse_export_body( Parser::TokArray& toks, char const* context ) +CodeBody parse_export_body() { return parse_global_nspace( ECode::Export_Body, toks, context ); } @@ -2170,9 +2244,9 @@ CodeFn parse_functon( Parser::TokArray& toks, char const* context ) eat( TokType::Module_Export ); } - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2294,9 +2368,9 @@ CodeOperator parse_operator( Parser::TokArray& toks, char const* context ) eat( TokType::Module_Export ); } - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2492,9 +2566,9 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context ) SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; - attributes = parse_attributes( toks, stringize(parse_template) ); + attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2576,9 +2650,9 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) Token name = { nullptr, 0, TokType::Invalid }; Token brute_sig = { currtok.Text, 0, TokType::Invalid }; - CodeAttributes attributes = parse_attributes( toks, context ); + CodeAttributes attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2651,7 +2725,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) } } - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -2842,7 +2916,7 @@ CodeUnion parse_union( Parser::TokArray& toks, char const* context ) eat( TokType::Decl_Union ); - CodeAttributes attributes = parse_attributes( toks, context ); + CodeAttributes attributes = parse_attributes(); StrC name = { 0, nullptr }; @@ -2986,12 +3060,10 @@ CodeUsing parse_using( StrC def ) } internal -CodeVar parse_variable( Parser::TokArray& toks, char const* context ) +CodeVar parse_variable() { using namespace Parser; - Token name = { nullptr, 0, TokType::Invalid }; - SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; @@ -3005,7 +3077,7 @@ CodeVar parse_variable( Parser::TokArray& toks, char const* context ) eat( TokType::Module_Export ); } - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); while ( left && tok_is_specifier( currtok ) ) { @@ -3050,24 +3122,32 @@ CodeVar parse_variable( Parser::TokArray& toks, char const* context ) if ( type == Code::Invalid ) return CodeInvalid; - name = currtok; + Context.Scope->Name = current; eat( TokType::Identifier ); - CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, name, toks, context ); + CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, Context.Scope->Name, Context.Tokens, Context.Scope->ProcName ); return result; } CodeVar parse_variable( StrC def ) { - check_parse_args( parse_variable, def ); using namespace Parser; + check_parse_args( parse_variable, def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_variable( toks, stringize(parse_variable) ); + Context.Tokens = toks; + Parser::StackNode root + { + toks.current(), + { nullptr, 0, TokType::Invalid }, + name(parse_variable) + }; + + return parse_variable(); } // Undef helper macros @@ -3075,4 +3155,3 @@ CodeVar parse_variable( StrC def ) # undef curr_tok # undef eat # undef left - diff --git a/project/components/gen.interface.upfront.cpp b/project/components/gen.interface.upfront.cpp index 90adc67..8245157 100644 --- a/project/components/gen.interface.upfront.cpp +++ b/project/components/gen.interface.upfront.cpp @@ -1338,7 +1338,7 @@ CodeBody def_class_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_class_body ); - AST_BODY_CLASS_UNALLOWED_TYPES + GEN_AST_BODY_CLASS_UNALLOWED_TYPES def_body_code_validation_end( def_class_body ); va_end(va); @@ -1354,7 +1354,7 @@ CodeBody def_class_body( s32 num, Code* codes ) result->Type = Function_Body; def_body_code_array_validation_start( def_class_body ); - AST_BODY_CLASS_UNALLOWED_TYPES + GEN_AST_BODY_CLASS_UNALLOWED_TYPES def_body_code_validation_end( def_class_body ); return result; @@ -1437,7 +1437,7 @@ CodeBody def_export_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_export_body ); - AST_BODY_EXPORT_UNALLOWED_TYPES + GEN_AST_BODY_EXPORT_UNALLOWED_TYPES def_body_code_validation_end( def_export_body ); va_end(va); @@ -1453,7 +1453,7 @@ CodeBody def_export_body( s32 num, Code* codes ) result->Type = Export_Body; def_body_code_array_validation_start( def_export_body ); - AST_BODY_EXPORT_UNALLOWED_TYPES + GEN_AST_BODY_EXPORT_UNALLOWED_TYPES def_body_code_validation_end( def_export_body ); return result; @@ -1470,7 +1470,7 @@ CodeBody def_extern_link_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_extern_linkage_body ); - AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES + GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES def_body_code_validation_end( def_extern_linkage_body ); va_end(va); @@ -1486,7 +1486,7 @@ CodeBody def_extern_link_body( s32 num, Code* codes ) result->Type = Extern_Linkage_Body; def_body_code_array_validation_start( def_extern_linkage_body ); - AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES + GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES def_body_code_validation_end( def_extern_linkage_body ); return result; @@ -1503,7 +1503,7 @@ CodeBody def_function_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_function_body ); - AST_BODY_FUNCTION_UNALLOWED_TYPES + GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES def_body_code_validation_end( def_function_body ); va_end(va); @@ -1519,7 +1519,7 @@ CodeBody def_function_body( s32 num, Code* codes ) result->Type = Function_Body; def_body_code_array_validation_start( def_function_body ); - AST_BODY_FUNCTION_UNALLOWED_TYPES + GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES def_body_code_validation_end( def_function_body ); return result; @@ -1536,7 +1536,7 @@ CodeBody def_global_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_global_body ); - AST_BODY_GLOBAL_UNALLOWED_TYPES + GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES def_body_code_validation_end( def_global_body ); va_end(va); @@ -1552,7 +1552,7 @@ CodeBody def_global_body( s32 num, Code* codes ) result->Type = Global_Body; def_body_code_array_validation_start( def_global_body ); - AST_BODY_GLOBAL_UNALLOWED_TYPES + GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES def_body_code_validation_end( def_global_body ); return result; @@ -1569,7 +1569,7 @@ CodeBody def_namespace_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_namespace_body ); - AST_BODY_NAMESPACE_UNALLOWED_TYPES + GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES def_body_code_validation_end( def_namespace_body ); va_end(va); @@ -1585,7 +1585,7 @@ CodeBody def_namespace_body( s32 num, Code* codes ) result->Type = Global_Body; def_body_code_array_validation_start( def_namespace_body ); - AST_BODY_NAMESPACE_UNALLOWED_TYPES + GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES def_body_code_validation_end( def_namespace_body ); return result; @@ -1737,7 +1737,7 @@ CodeBody def_struct_body( s32 num, ... ) va_list va; va_start(va, num); def_body_code_validation_start( def_struct_body ); - AST_BODY_STRUCT_UNALLOWED_TYPES + GEN_AST_BODY_STRUCT_UNALLOWED_TYPES def_body_code_validation_end( def_struct_body ); va_end(va); @@ -1753,7 +1753,7 @@ CodeBody def_struct_body( s32 num, Code* codes ) result->Type = Struct_Body; def_body_code_array_validation_start( def_struct_body ); - AST_BODY_STRUCT_UNALLOWED_TYPES + GEN_AST_BODY_STRUCT_UNALLOWED_TYPES def_body_code_validation_end( def_struct_body ); return result; diff --git a/project/components/gen.types.hpp b/project/components/gen.types.hpp index bda942a..20f201f 100644 --- a/project/components/gen.types.hpp +++ b/project/components/gen.types.hpp @@ -100,10 +100,3 @@ constexpr char const* Attribute_Keyword = stringize( GEN_Attribute_Keyword ); constexpr char const* Attribute_Keyword = ""; #endif - -#ifndef GEN_Define_Attribute_Tokens -# define GEN_Define_Attribute_Tokens \ - Entry( API_Export, "GEN_API_Export_Code" ) \ - Entry( API_Import, "GEN_API_Import_Code" ) -#endif - diff --git a/project/helpers/gen.helper.hpp b/project/helpers/gen.helper.hpp index d0e257f..2270899 100644 --- a/project/helpers/gen.helper.hpp +++ b/project/helpers/gen.helper.hpp @@ -6,7 +6,7 @@ using namespace gen; CodeBody gen_ecode( char const* path ) { - char scratch_mem[kilobytes(1)]; + char scratch_mem[kilobytes(1)]; Arena scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) ); file_read_contents( scratch, zero_terminate, path ); @@ -50,8 +50,7 @@ CodeBody gen_ecode( char const* path ) #pragma pop_macro( "local_persist" ) CodeNamespace nspace = def_namespace( name(ECode), def_namespace_body( args( enum_code, to_str ) ) ); - - CodeUsing code_t = def_using( name(CodeT), def_type( name(ECode::Type) ) ); + CodeUsing code_t = def_using( name(CodeT), def_type( name(ECode::Type) ) ); return def_global_body( args( nspace, code_t ) ); } @@ -209,7 +208,7 @@ CodeBody gen_especifier( char const* path ) CodeBody gen_etoktype( char const* etok_path, char const* attr_path ) { - char scratch_mem[kilobytes(64)]; + char scratch_mem[kilobytes(64)]; Arena scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) ); FileContents enum_content = file_read_contents( scratch, zero_terminate, etok_path ); @@ -217,8 +216,6 @@ CodeBody gen_etoktype( char const* etok_path, char const* attr_path ) CSV_Object csv_enum_nodes; csv_parse( &csv_enum_nodes, rcast(char*, enum_content.data), GlobalAllocator, false ); - // memset( scratch_mem, 0, sizeof(scratch_mem) ); - // scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) ); FileContents attrib_content = file_read_contents( scratch, zero_terminate, attr_path ); CSV_Object csv_attr_nodes; From c5afede7b5b77526fb432a088633b6e5c290fb4c Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 05:52:06 -0400 Subject: [PATCH 02/10] Reorganization of files, refactors, doc updates (WIP) Removing the gen. namespace from the files for components, dependencies, and file_processors. They are only necessary if the include directory is transparent, and in my case those are not. Made a docs directory. I'm offloading information from the main readme to there along with additional informationn I end up elaborating on down the line. Enum tables were moved to their own directory (project/enums). Library will not compile for now. Major refactor occuring with parsing related components. --- Docs/Parsing.md | 49 ++ Docs/Readme.md | 537 +++++++++++++++++ Readme.md | 564 +----------------- gencpp.vcxproj | 55 +- project/Readme.md | 40 +- project/components/{gen.ast.cpp => ast.cpp} | 97 +-- ...st_case_macros.cpp => ast_case_macros.cpp} | 0 ...ata_structures.hpp => data_structures.hpp} | 0 .../components/{gen.ecode.hpp => ecode.hpp} | 0 .../{gen.eoperator.hpp => eoperator.hpp} | 2 +- .../{gen.especifier.hpp => especifier.hpp} | 1 - .../{gen.etoktype.cpp => etoktype.cpp} | 10 +- .../{gen.header_end.hpp => header_end.hpp} | 0 ...{gen.header_start.hpp => header_start.hpp} | 0 .../{gen.impl_start.cpp => impl_start.cpp} | 0 .../{gen.interface.cpp => interface.cpp} | 6 +- .../{gen.interface.hpp => interface.hpp} | 0 ...face.parsing.cpp => interface.parsing.cpp} | 363 +++++++---- ...face.upfront.cpp => interface.upfront.cpp} | 0 .../{gen.data.cpp => static_data.cpp} | 0 .../components/{gen.types.hpp => types.hpp} | 2 +- .../{gen.untyped.cpp => untyped.cpp} | 3 +- .../{gen.basic_types.hpp => basic_types.hpp} | 0 .../{gen.containers.hpp => containers.hpp} | 0 .../dependencies/{gen.debug.cpp => debug.cpp} | 0 .../dependencies/{gen.debug.hpp => debug.hpp} | 0 ...en.file_handling.cpp => file_handling.cpp} | 0 ...en.file_handling.hpp => file_handling.hpp} | 0 .../{gen.hashing.cpp => hashing.cpp} | 0 .../{gen.hashing.hpp => hashing.hpp} | 0 ...{gen.header_start.hpp => header_start.hpp} | 0 .../{gen.impl_start.cpp => impl_start.cpp} | 0 .../{gen.macros.hpp => macros.hpp} | 0 .../{gen.memory.cpp => memory.cpp} | 0 .../{gen.memory.hpp => memory.hpp} | 0 .../{gen.parsing.cpp => parsing.cpp} | 0 .../{gen.parsing.hpp => parsing.hpp} | 0 .../{gen.printing.cpp => printing.cpp} | 0 .../{gen.printing.hpp => printing.hpp} | 7 +- .../{gen.string.cpp => string.cpp} | 0 .../{gen.string.hpp => string.hpp} | 0 .../{gen.string_ops.cpp => string_ops.cpp} | 0 .../{gen.string_ops.hpp => string_ops.hpp} | 0 .../{gen.timing.cpp => timing.cpp} | 0 .../{gen.timing.hpp => timing.hpp} | 0 .../{components => enums}/AttributeTokens.csv | 0 project/{components => enums}/ECode.csv | 0 project/{components => enums}/EOperator.csv | 0 project/{components => enums}/ESpecifier.csv | 0 project/{components => enums}/ETokType.csv | 0 .../builder.cpp} | 0 .../builder.hpp} | 0 .../editor.hpp} | 0 .../scanner.hpp} | 0 project/gen.bootstrap.cpp | 92 +-- project/gen.cpp | 22 +- project/gen.dep.cpp | 20 +- project/gen.dep.hpp | 25 +- project/gen.hpp | 22 +- .../helpers/{gen.helper.hpp => helper.hpp} | 0 ...ores.inline.hpp => pop_ignores.inline.hpp} | 0 ...res.inline.hpp => push_ignores.inline.hpp} | 0 ...{gen.undef.macros.hpp => undef.macros.hpp} | 0 scripts/Readme.md | 36 +- scripts/get_sources.ps1 | 12 - .../{build.ci.ps1 => test.gen_build.ci.ps1} | 0 scripts/{build.ps1 => test.gen_build.ps1} | 0 scripts/{gen.ps1 => test.gen_run.ps1} | 0 68 files changed, 1077 insertions(+), 888 deletions(-) create mode 100644 Docs/Parsing.md create mode 100644 Docs/Readme.md rename project/components/{gen.ast.cpp => ast.cpp} (87%) rename project/components/{gen.ast_case_macros.cpp => ast_case_macros.cpp} (100%) rename project/components/{gen.data_structures.hpp => data_structures.hpp} (100%) rename project/components/{gen.ecode.hpp => ecode.hpp} (100%) rename project/components/{gen.eoperator.hpp => eoperator.hpp} (98%) rename project/components/{gen.especifier.hpp => especifier.hpp} (99%) rename project/components/{gen.etoktype.cpp => etoktype.cpp} (96%) rename project/components/{gen.header_end.hpp => header_end.hpp} (100%) rename project/components/{gen.header_start.hpp => header_start.hpp} (100%) rename project/components/{gen.impl_start.cpp => impl_start.cpp} (100%) rename project/components/{gen.interface.cpp => interface.cpp} (98%) rename project/components/{gen.interface.hpp => interface.hpp} (100%) rename project/components/{gen.interface.parsing.cpp => interface.parsing.cpp} (87%) rename project/components/{gen.interface.upfront.cpp => interface.upfront.cpp} (100%) rename project/components/{gen.data.cpp => static_data.cpp} (100%) rename project/components/{gen.types.hpp => types.hpp} (96%) rename project/components/{gen.untyped.cpp => untyped.cpp} (97%) rename project/dependencies/{gen.basic_types.hpp => basic_types.hpp} (100%) rename project/dependencies/{gen.containers.hpp => containers.hpp} (100%) rename project/dependencies/{gen.debug.cpp => debug.cpp} (100%) rename project/dependencies/{gen.debug.hpp => debug.hpp} (100%) rename project/dependencies/{gen.file_handling.cpp => file_handling.cpp} (100%) rename project/dependencies/{gen.file_handling.hpp => file_handling.hpp} (100%) rename project/dependencies/{gen.hashing.cpp => hashing.cpp} (100%) rename project/dependencies/{gen.hashing.hpp => hashing.hpp} (100%) rename project/dependencies/{gen.header_start.hpp => header_start.hpp} (100%) rename project/dependencies/{gen.impl_start.cpp => impl_start.cpp} (100%) rename project/dependencies/{gen.macros.hpp => macros.hpp} (100%) rename project/dependencies/{gen.memory.cpp => memory.cpp} (100%) rename project/dependencies/{gen.memory.hpp => memory.hpp} (100%) rename project/dependencies/{gen.parsing.cpp => parsing.cpp} (100%) rename project/dependencies/{gen.parsing.hpp => parsing.hpp} (100%) rename project/dependencies/{gen.printing.cpp => printing.cpp} (100%) rename project/dependencies/{gen.printing.hpp => printing.hpp} (86%) rename project/dependencies/{gen.string.cpp => string.cpp} (100%) rename project/dependencies/{gen.string.hpp => string.hpp} (100%) rename project/dependencies/{gen.string_ops.cpp => string_ops.cpp} (100%) rename project/dependencies/{gen.string_ops.hpp => string_ops.hpp} (100%) rename project/dependencies/{gen.timing.cpp => timing.cpp} (100%) rename project/dependencies/{gen.timing.hpp => timing.hpp} (100%) rename project/{components => enums}/AttributeTokens.csv (100%) rename project/{components => enums}/ECode.csv (100%) rename project/{components => enums}/EOperator.csv (100%) rename project/{components => enums}/ESpecifier.csv (100%) rename project/{components => enums}/ETokType.csv (100%) rename project/{filesystem/gen.builder.cpp => file_processors/builder.cpp} (100%) rename project/{filesystem/gen.builder.hpp => file_processors/builder.hpp} (100%) rename project/{filesystem/gen.editor.hpp => file_processors/editor.hpp} (100%) rename project/{filesystem/gen.scanner.hpp => file_processors/scanner.hpp} (100%) rename project/helpers/{gen.helper.hpp => helper.hpp} (100%) rename project/helpers/{gen.pop_ignores.inline.hpp => pop_ignores.inline.hpp} (100%) rename project/helpers/{gen.push_ignores.inline.hpp => push_ignores.inline.hpp} (100%) rename project/helpers/{gen.undef.macros.hpp => undef.macros.hpp} (100%) delete mode 100644 scripts/get_sources.ps1 rename scripts/{build.ci.ps1 => test.gen_build.ci.ps1} (100%) rename scripts/{build.ps1 => test.gen_build.ps1} (100%) rename scripts/{gen.ps1 => test.gen_run.ps1} (100%) diff --git a/Docs/Parsing.md b/Docs/Parsing.md new file mode 100644 index 0000000..e3e23e0 --- /dev/null +++ b/Docs/Parsing.md @@ -0,0 +1,49 @@ +# Parsing + +The library features a naive parser tailored for only what the library needs to construct the supported syntax of C++ into its AST. +This parser does not, and should not do the compiler's job. By only supporting this minimal set of features, the parser is kept under 5000 loc. + + +Everything is done in one pass for both the preprocessor directives and the rest of the language. +The parser performs no macro expansion as the scope of gencpp feature-set is to only support the preprocessor for the goal of having rudimentary awareness of preprocessor ***conditionals***, ***defines***, and ***includes***. +*(Conditionals and defines are a TODO)* + +The keywords supported for the preprocessor are: + +* include +* define +* if +* ifdef +* elif +* endif +* undef + +Just like with actual preprocessor, each directive # line is considered one preproecessor unit, and will be treated as one Preprocessor AST. *These ASTs will be considered members or entries of braced scope they reside within*. +All keywords except *include* are suppported as members of a scope for a class/struct, global, or namespace body. + +Any preprocessor definition abuse that changes the syntax of the core language is unsupported and will fail to parse if not kept within an execution scope (function body, or expression assignment). + +The parsing implementation supports the following for the user: + +```cpp +CodeClass parse_class ( StrC class_def ); +CodeEnum parse_enum ( StrC enum_def ); +CodeBody parse_export_body ( StrC export_def ); +CodeExtern parse_extern_link ( StrC exten_link_def); +CodeFriend parse_friend ( StrC friend_def ); +CodeFn parse_function ( StrC fn_def ); +CodeBody parse_global_body ( StrC body_def ); +CodeNamespace parse_namespace ( StrC namespace_def ); +CodeOperator parse_operator ( StrC operator_def ); +CodeOpCast parse_operator_cast( StrC operator_def ); +CodeStruct parse_struct ( StrC struct_def ); +CodeTemplate parse_template ( StrC template_def ); +CodeType parse_type ( StrC type_def ); +CodeTypedef parse_typedef ( StrC typedef_def ); +CodeUnion parse_union ( StrC union_def ); +CodeUsing parse_using ( StrC using_def ); +CodeVar parse_variable ( StrC var_def ); +``` + +***Parsing will aggregate any tokens within a function body or expression statement to an untyped Code AST.*** + diff --git a/Docs/Readme.md b/Docs/Readme.md new file mode 100644 index 0000000..1787d75 --- /dev/null +++ b/Docs/Readme.md @@ -0,0 +1,537 @@ +## Documentation + +The project has no external dependencies beyond: + +* `errno.h` +* `stat.h` +* `stdarg.h` +* `stddef.h` +* `stdio.h` +* `copyfile.h` (Mac) +* `types.h` (Linux) +* `unistd.h` (Linux/Mac) +* `intrin.h` (Windows) +* `io.h` (Windows with gcc) +* `windows.h` (Windows) + +Dependencies for the project are wrapped within `GENCPP_ROLL_OWN_DEPENDENCIES` (Defining it will disable them). +The majority of the dependency's implementation was derived from the [c-zpl library](https://github.com/zpl-c/zpl). + +This library was written in a subset of C++ where the following are not used at all: + +* RAII (Constructors/Destructors), lifetimes are managed using named static or regular functions. +* Language provide dynamic dispatch, RTTI +* Object-Oriented Inheritance +* Exceptions + +Polymorphic & Member-functions are used as an ergonomic choice, along with a conserative use of operator overloads. +There are only 4 template definitions in the entire library. (`Array`, `Hashtable`, `swap`, and `AST/Code::cast`) + +Two generic templated containers are used throughout the library: + +* `template< class Type> struct Array` +* `template< class Type> struct HashTable` + +Both Code and AST definitions have a `template< class Type> Code/AST cast()`. Its just an alternative way to explicitly cast to each other. + +`template< class Type> swap( Type& a, Type& b)` is used over a macro. + +Otherwise the library is free of any templates. + +### *WHAT IS NOT PROVIDED* + +* Execution statement validation : Execution expressions are defined using the untyped AST. + * Lambdas (This naturally means its unsupported) +* RAII : This needs support for constructors/destructor parsing + * Haven't gotten around to yet (its in the github issues) + +Keywords kept from "Modern C++": + +* constexpr : Great to store compile-time constants. +* consteval : Technically fine, need to make sure to execute in moderation. +* constinit : Better than constexpr at doing its job, however, its only c++ 20. +* export : Useful if c++ modules ever come around to actually being usable. +* import : ^^ +* module : ^^ + +When it comes to expressions: + +**There is no support for validating expressions.** +Its difficult to parse without enough benefits (At the metaprogramming level). + +When it comes to templates: + +**Only trivial template support is provided.** +The intention is for only simple, non-recursive substitution. +The parameters of the template are treated like regular parameter AST entries. +This means that the typename entry for the parameter AST would be either: + +* `class` +* `typename` +* A fundamental type, function, or pointer type. + +Anything beyond this usage is not supported by parse_template for arguments (at least not intentionally). +Use at your own mental peril. + +*Concepts and Constraints are not supported, its usage is non-trivial substitution.* + +### The Data & Interface + +As mentioned in [Usage](#usage), the user is provided Code objects by calling the constructor's functions to generate them or find existing matches. + +The AST is managed by the library and provided the user via its interface. +However, the user may specifiy memory configuration. + +Data layout of AST struct: + +```cpp +union { + struct + { + AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable + AST* Specs; // Function, Operator, Type symbol, Variable + union { + AST* ParentType; // Class, Struct + AST* ReturnType; // Function, Operator + AST* UnderlyingType; // Enum, Typedef + AST* ValueType; // Parameter, Variable + }; + AST* Params; // Function, Operator, Template + union { + AST* ArrExpr; // Type Symbol + AST* Body; // Class, Enum, Function, Namespace, Struct, Union + AST* Declaration; // Friend, Template + AST* Value; // Parameter, Variable + }; + }; + StringCached Content; // Attributes, Comment, Execution, Include + SpecifierT ArrSpecs[AST::ArrSpecs_Cap]; // Specifiers +}; +union { + AST* Prev; + AST* Front; // Used by CodeBody + AST* Last; // Used by CodeParam +}; +union { + AST* Next; + AST* Back; // Used by CodeBody +}; +AST* Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +union { + OperatorT Op; + AccessSpec ParentAccess; + s32 NumEntries; +}; +``` + +*`CodeT` is a typedef for `ECode::Type` which has an underlying type of `u32`* +*`OperatorT` is a typedef for `EOperator::Type` which has an underlying type of `u32`* +*`StringCahced` is a typedef for `String const`, to denote it is an interned string* +*`String` is the dynamically allocated string type for the library* + +AST widths are setup to be AST_POD_Size. +The width dictates how much the static array can hold before it must give way to using an allocated array: + +```cpp +constexpr static +uw ArrSpecs_Cap = +( + AST_POD_Size + - sizeof(AST*) * 3 + - sizeof(StringCached) + - sizeof(CodeT) + - sizeof(ModuleFlag) + - sizeof(s32) +) +/ sizeof(SpecifierT) -1; // -1 for 4 extra bytes (Odd num of AST*) +``` + +*Ex: If the AST_POD_Size is 128 the capacity of the static array is 20.* + +Data Notes: + +* The allocator definitions used are exposed to the user incase they want to dictate memory usage + * You'll find the memory handling in `init`, `gen_string_allocator`, `get_cached_string`, `make_code`. +* ASTs are wrapped for the user in a Code struct which is a wrapper for a AST* type. +* Both AST and Code have member symbols but their data layout is enforced to be POD types. +* This library treats memory failures as fatal. +* Cached Strings are stored in their own set of arenas. AST constructors use cached strings for names, and content. + * `StringArenas`, `StringCache`, `Allocator_StringArena`, and `Allocator_StringTable` are the associated containers or allocators. +* Strings used for serialization and file buffers are not contained by those used for cached strings. + * They are currently using `GlobalAllocator`, which are tracked array of arenas that grows as needed (adds buckets when one runs out). + * Memory within the buckets is not reused, so its inherently wasteful. + * I will be augmenting the single arena with a simple slag allocator. +* Linked lists used children nodes on bodies, and parameters. +* Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup. +* Allocations can be tuned by defining the folloiwng macros: + * `GEN_BUILDER_STR_BUFFER_RESERVE` + * `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator + * `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator + * `GEN_LEX_ALLOCATOR_SIZE` + * `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line. + * `GEN_MAX_NAME_LENGTH` : Max length of any identifier. + * `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code. + * `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching. + * `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable. + +The following CodeTypes are used which the user may optionally use strong typing with if they enable: `GEN_ENFORCE_STRONG_CODE_TYPES` + +* CodeBody : Has support for `for-range` iterating across Code objects. +* CodeAttributes +* CodeComment +* CodeClass +* CodeConstructor +* CodeDefine +* CodeDestructor +* CodePreprocessCond +* CodeEnum +* CodeExec +* CodeExtern +* CodeInclude +* CodeFriend +* CodeFn +* CodeModule +* CodeNamespace +* CodeOperator +* CodeOpCast +* CodeParam : Has support for `for-range` iterating across parameters. +* CodeSpecifiers : Has support for `for-range` iterating across specifiers. +* CodeStruct +* CodeTemplate +* CodeType +* CodeTypedef +* CodeUnion +* CodeUsing +* CodeVar + +Each Code boy has an associated "filtered AST" with the naming convention: `AST_` +Unrelated fields of the AST for that node type are omitted and only necessary padding members are defined otherwise. +Retrieving a raw version of the ast can be done using the `raw()` function defined in each AST. + +## There are three sets of interfaces for Code AST generation the library provides + +* Upfront +* Parsing +* Untyped + +### Upfront Construction + +All component ASTs must be previously constructed, and provided on creation of the code AST. +The construction will fail and return Code::Invalid otherwise. + +Interface :`` + +* def_attributes + * *This is pre-appended right before the function symbol, or placed after the class or struct keyword for any flavor of attributes used.* + * *Its up to the user to use the desired attribute formatting: `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU).* +* def_comment +* def_class +* def_constructor +* def_destructor +* def_enum +* def_execution + * *This is equivalent to untyped_str, except that its intended for use only in execution scopes.* +* def_extern_link +* def_friend +* def_function +* def_include +* def_module +* def_namespace +* def_operator +* def_operator_cast +* def_param +* def_params +* def_specifier +* def_specifiers +* def_struct +* def_template +* def_type +* def_typedef +* def_union +* def_using +* def_using_namespace +* def_variable + +Bodies: + +* def_body +* def_class_body +* def_enum_body +* def_export_body +* def_extern_link_body +* def_function_body + * *Use this for operator bodies as well* +* def_global_body +* def_namespace_body +* def_struct_body +* def_union_body + +Usage: + +```cpp + = def_( ... ); + +Code +{ + ... + = def_( ... ); +} + +``` + +When using the body functions, its recommended to use the args macro to auto determine the number of arguments for the varadic: +```cpp +def_global_body( args( ht_entry, array_ht_entry, hashtable )); + +// instead of: +def_global_body( 3, ht_entry, array_ht_entry, hashtable ); +``` + +If a more incremental approach is desired for the body ASTs, `Code def_body( CodeT type )` can be used to create an empty body. +When the members have been populated use: `AST::validate_body` to verify that the members are valid entires for that type. + +### Parse construction + +A string provided to the API is parsed for the intended language construct. + +Interface : + +* parse_class +* parse_constructor +* parse_destructor +* parse_enum +* parse_export_body +* parse_extern_link +* parse_friend + * Purposefully are only support forward declares with this constructor. +* parse_function +* parse_global_body +* parse_namespace +* parse_operator +* parse_operator_cast +* parse_struct +* parse_template +* parse_type +* parse_typedef +* parse_union +* parse_using +* parse_variable + +The lexing and parsing takes shortcuts from whats expected in the standard. + +* Numeric literals are not check for validity. +* The parse API treats any execution scope definitions with no validation and are turned into untyped Code ASTs. + * *This includes the assignment of variables.* +* Attributes ( `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU) ) + * Assumed to *come before specifiers* (`const`, `constexpr`, `extern`, `static`, etc) for a function + * Or in the usual spot for class, structs, (*right after the declaration keyword*) + * typedefs have attributes with the type (`parse_type`) +* As a general rule; if its not available from the upfront constructors, its not available in the parsing constructors. + * *Upfront constructors are not necessarily used in the parsing constructors, this is just a good metric to know what can be parsed.* +* Parsing attributes can be extended to support user defined macros by defining `GEN_DEFINE_ATTRIBUTE_TOKENS` (see `gen.hpp` for the formatting) + +Usage: + +```cpp +Code = parse_( string with code ); + +Code = def_( ..., parse_( + +)); + +Code = make_( ... ) +{ + ->add( parse_( + + )); +} +``` + +### Untyped constructions + +Code ASTs are constructed using unvalidated strings. + +Interface : + +* token_fmt_va +* token_fmt +* untyped_str +* untyped_fmt +* untyped_token_fmt + +During serialization any untyped Code AST has its string value directly injected inline of whatever context the content existed as an entry within. +Even though these are not validated from somewhat correct c/c++ syntax or components, it doesn't mean that Untyped code can be added as any component of a Code AST: + +* Untyped code cannot have children, thus there cannot be recursive injection this way. +* Untyped code can only be a child of a parent of body AST, or for values of an assignment (ex: variable assignment). + +These restrictions help prevent abuse of untyped code to some extent. + +Usage Conventions: + +```cpp +Code = def_variable( , , untyped_( + +)); + +Code = untyped_str( code( + +)); +``` + +Optionally, `code_str`, and `code_fmt` macros can be used so that the code macro doesn't have to be used: +```cpp +Code = code_str( ) +``` + +Template metaprogramming in the traditional sense becomes possible with the use of `token_fmt` and parse constructors: + +```cpp +StrC value = txt_StrC("Something"); + +char const* template_str = txt( + Code with to replace with token_values + ... +); +char const* gen_code_str = token_fmt( "key", value, template_str ); +Code = parse_( gen_code_str ); +``` + +## Predefined Codes + +The following are provided predefined by the library as they are commonly used: + +* `access_public` +* `access_protected` +* `access_private` +* `module_global_fragment` +* `module_private_fragment` +* `param_varaidc` (Used for varadic definitions) +* `preprocess_else` +* `preprocess_endif` +* `pragma_once` +* `spec_const` +* `spec_consteval` +* `spec_constexpr` +* `spec_constinit` +* `spec_extern_linkage` (extern) +* `spec_final` +* `spec_global` (global macro) +* `spec_inline` +* `spec_internal_linkage` (internal macro) +* `spec_local_persist` (local_persist macro) +* `spec_mutable` +* `spec_override` +* `spec_ptr` +* `spec_ref` +* `spec_register` +* `spec_rvalue` +* `spec_static_member` (static) +* `spec_thread_local` +* `spec_virtual` +* `spec_volatile` +* `spec_type_signed` +* `spec_type_unsigned` +* `spec_type_short` +* `spec_type_long` +* `t_empty` (Used for varaidc macros) +* `t_auto` +* `t_void` +* `t_int` +* `t_bool` +* `t_char` +* `t_wchar_t` +* `t_class` +* `t_typename` + +Optionally the following may be defined if `GEN_DEFINE_LIBRARY_CODE_CONSTANTS` is defined + +* `t_b32` +* `t_s8` +* `t_s16` +* `t_s32` +* `t_s64` +* `t_u8` +* `t_u16` +* `t_u32` +* `t_u64` +* `t_sw` +* `t_uw` +* `t_f32` +* `t_f64` + +## Extent of operator overload validation + +The AST and constructors will be able to validate that the arguments provided for the operator type match the expected form: + +* If return type must match a parameter +* If number of parameters is correct +* If added as a member symbol to a class or struct, that operator matches the requirements for the class (types match up) + +The user is responsible for making sure the code types provided are correct +and have the desired specifiers assigned to them beforehand. + +## Code generation and modification + +There are three provided file interfaces: + +* Builder +* Editor +* Scanner + +Editor and Scanner are disabled by default, use `GEN_FEATURE_EDITOR` and `GEN_FEATURE_SCANNER` to enable them. + +### Builder is a similar object to the jai language's string_builder + +* The purpose of it is to generate a file. +* A file is specified and opened for writing using the open( file_path) ) function. +* The code is provided via print( code ) function will be serialized to its buffer. +* When all serialization is finished, use the write() command to write the buffer to the file. + +### Editor is for editing a series of files based on a set of requests provided to it + +**Note: Not implemented yet** + +* The purpose is to overrite a specific file, it places its contents in a buffer to scan. +* Requests are populated using the following interface: + * add : Add code. + * remove : Remove code. + * replace: Replace code. + +All three have the same parameters with exception to remove which only has SymbolInfo and Policy: + +* SymbolInfo: + * File : The file the symbol resides in. Leave null to indicate to search all files. Leave null to indicated all-file search. + * Marker : #define symbol that indicates a location or following signature is valid to manipulate. Leave null to indicate the signature should only be used. + * Signature : Use a Code symbol to find a valid location to manipulate, can be further filtered with the marker. Leave null to indicate the marker should only be used. + +* Policy : Additional policy info for completing the request (empty for now) +* Code : Code to inject if adding, or replace existing code with. + +Additionally if `GEN_FEATURE_EDITOR_REFACTOR` is defined, refactor( file_path, specification_path ) wil be made available. +Refactor is based of the refactor library and uses its interface. +It will on call add a request to the queue to run the refactor script on the file. + +### Scanner allows the user to generate Code ASTs by reading files + +**Note: Not implemented yet** + +* The purpose is to grab definitions to generate metadata or generate new code from these definitions. +* Requests are populated using the add( SymbolInfo, Policy ) function. The symbol info is the same as the one used for the editor. So is the case with Policy. + +The file will only be read from, no writing supported. + +One great use case is for example: generating the single-header library for gencpp! + +### Additional Info (Editor and Scanner) + +When all requests have been populated, call process_requests(). +It will provide an output of receipt data of the results when it completes. + +Files may be added to the Editor and Scanner additionally with add_files( num, files ). +This is intended for when you have requests that are for multiple files. + +Request queue in both Editor and Scanner are cleared once process_requests completes. diff --git a/Readme.md b/Readme.md index 9d91958..f1aab97 100644 --- a/Readme.md +++ b/Readme.md @@ -12,43 +12,12 @@ its not meant to be a black box metaprogramming utility, its meant for the user * [Notes](#notes) * [Usage](#usage) -* [Building](#notes) -* [Outline](#outline) -* [What is not provided](#what-is-not-provided) -* [The three constructors ](#there-are-three-sets-of-interfaces-for-code-ast-generation-the-library-provides) -* [Predefined Codes](#predefined-codes) -* [Code generation and modification](#code-generation-and-modification) +* [Building](#building) ## Notes -The project has reached an *alpha* state, all the current functionality works for the test cases but it will most likely break in many other cases. - -The project has no external dependencies beyond: - -* `errno.h` -* `stat.h` -* `stdarg.h` -* `stddef.h` -* `stdio.h` -* `copyfile.h` (Mac) -* `types.h` (Linux) -* `unistd.h` (Linux/Mac) -* `intrin.h` (Windows) -* `io.h` (Windows with gcc) -* `windows.h` (Windows) - -Dependencies for the project are wrapped within `GENCPP_ROLL_OWN_DEPENDENCIES` (Defining it will disable them). -The majority of the dependency's implementation was derived from the [c-zpl library](https://github.com/zpl-c/zpl). - -This library was written a subset of C++ where the following are not used at all: - -* RAII (Constructors/Destructors), lifetimes are managed using named static or regular functions. -* Language provide dynamic dispatch, RTTI -* Object-Oriented Inheritance -* Exceptions - -Polymorphic & Member-functions are used as an ergonomic choice, along with a conserative use of operator overloads. -There are only 4 template definitions in the entire library. (`Array`, `Hashtable`, `swap`, and `AST/Code::cast`) +The project has reached an *alpha* state, all the current functionality works for the test cases but it will most likely break in many other cases. +The [issues](https://github.com/Ed94/gencpp/issues) marked with v1.0 Feature indicate whats left before the library is considered feature complete. A `natvis` and `natstepfilter` are provided in the scripts directory. @@ -87,7 +56,7 @@ u32 gen_main() ``` The design uses a constructive builder API for the code to generate. -The user is given `Code` objects that are used to build up the AST. +The user is provided `Code` objects that are used to build up the AST. Example using each construction interface: @@ -154,529 +123,8 @@ struct ArrayHeader ``` **Note: The formatting shown here is not how it will look. For your desired formatting its recommended to run a pass through the files with an auto-formatter.** +*(The library currently uses clang-format for formatting, beaware its pretty slow...)* ## Building -An example of building is provided within project, singleheader, and test. - -(All generated files go to a corresponding `*/gen` directory) - -**Project** - -`gen.bootstrap.cpp` generates a segmented version of the library where code is divided into `gen.hpp`, `gen.cpp`, `gen_dep.hpp`, and `gen_dep.cpp`. -(Similar whats there already, but with the includes injected) - -**Singleheader** - -`gen.singleheader.cpp` generates a single-header version of the library following the convention shown in popular libraries such as: gb, stb, and zpl. - -**Test** - -There are two meson build files the one within test is the program's build specification. -The other one in the gen directory within test is the metaprogram's build specification. - -Both of them build the same source file: `test.cpp`. The only differences are that gen needs a different relative path to the include directories and defines the macro definition: `GEN_TIME`. - -This method is setup where all the metaprogram's code are the within the same files as the regular program's code. - -## Outline - -### *WHAT IS NOT PROVIDED* - -* Exceptions -* Execution statement validation : Execution expressions are defined using the untyped API. - * Lambdas (This naturally means its unsupported) -* RAII : This needs support for constructors/destructor parsing - * Haven't gotten around to yet, only useful (to me) for third-party scanning - -Keywords kept from "Modern C++": - -* constexpr : Great to store compile-time constants. -* consteval : Technically fine, need to make sure to execute in moderation. -* constinit : Better than constexpr at doing its job, however, its only c++ 20. -* export : Useful if c++ modules ever come around to actually being usable. -* import : ^^ -* module : ^^ - -When it comes to expressions: - -**There is no support for validating expressions.** -Its difficult to parse without enough benefits (At the metaprogramming level). - -When it comes to templates: - -Only trivial template support is provided. the intention is for only simple, non-recursive substitution. -The parameters of the template are treated like regular parameter AST entries. -This means that the typename entry for the parameter AST would be either: - -* `class` -* `typename` -* A fundamental type, function, or pointer type. - -Anything beyond this usage is not supported by parse_template for arguments (at least not intentionally). -Use at your own mental peril. - -*Concepts and Constraints are not supported, its usage is non-trivial substitution.* - -### The Data & Interface - -As mentioned in [Usage](#usage), the user is provided Code objects by calling the constructor's functions to generate them or find existing matches. - -The AST is managed by the library and provided the user via its interface. -However, the user may specifiy memory configuration. - -Data layout of AST struct: - -```cpp -union { - struct - { - AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable - AST* Specs; // Function, Operator, Type symbol, Variable - union { - AST* ParentType; // Class, Struct - AST* ReturnType; // Function, Operator - AST* UnderlyingType; // Enum, Typedef - AST* ValueType; // Parameter, Variable - }; - AST* Params; // Function, Operator, Template - union { - AST* ArrExpr; // Type Symbol - AST* Body; // Class, Enum, Function, Namespace, Struct, Union - AST* Declaration; // Friend, Template - AST* Value; // Parameter, Variable - }; - }; - StringCached Content; // Attributes, Comment, Execution, Include - SpecifierT ArrSpecs[AST::ArrSpecs_Cap]; // Specifiers -}; -union { - AST* Prev; - AST* Front; // Used by CodeBody - AST* Last; // Used by CodeParam -}; -union { - AST* Next; - AST* Back; // Used by CodeBody -}; -AST* Parent; -StringCached Name; -CodeT Type; -ModuleFlag ModuleFlags; -union { - OperatorT Op; - AccessSpec ParentAccess; - s32 NumEntries; -}; -``` - -*`CodeT` is a typedef for `ECode::Type` which has an underlying type of `u32`* -*`OperatorT` is a typedef for `EOperator::Type` which has an underlying type of `u32`* -*`StringCahced` is a typedef for `String const`, to denote it is an interned string* -*`String` is the dynamically allocated string type for the library* - -AST widths are setup to be AST_POD_Size. -The width dictates how much the static array can hold before it must give way to using an allocated array: - -```cpp -constexpr static -uw ArrSpecs_Cap = -( - AST_POD_Size - - sizeof(AST*) * 3 - - sizeof(StringCached) - - sizeof(CodeT) - - sizeof(ModuleFlag) - - sizeof(s32) -) -/ sizeof(SpecifierT) -1; // -1 for 4 extra bytes (Odd num of AST*) -``` - -*Ex: If the AST_POD_Size is 128 the capacity of the static array is 20.* - -Data Notes: - -* The allocator definitions used are exposed to the user incase they want to dictate memory usage - * You'll find the memory handling in `init`, `gen_string_allocator`, `get_cached_string`, `make_code`. -* ASTs are wrapped for the user in a Code struct which is a wrapper for a AST* type. -* Both AST and Code have member symbols but their data layout is enforced to be POD types. -* This library treats memory failures as fatal. -* Cached Strings are stored in their own set of arenas. AST constructors use cached strings for names, and content. - * `StringArenas`, `StringCache`, `Allocator_StringArena`, and `Allocator_StringTable` are the associated containers or allocators. -* Strings used for serialization and file buffers are not contained by those used for cached strings. - * They are currently using `GlobalAllocator`, which are tracked array of arenas that grows as needed (adds buckets when one runs out). - * Memory within the buckets is not reused, so its inherently wasteful. - * I will be augmenting the single arena with a simple slag allocator. -* Linked lists used children nodes on bodies, and parameters. -* Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup. -* Allocations can be tuned by defining the folloiwng macros: - * `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator - * `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator - * `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching. - * `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line. - * `GEN_MAX_NAME_LENGTH` : Max length of any identifier. - * `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code. - * `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable. - * `GEN_LEX_ALLOCATOR_SIZE` - * `GEN_BUILDER_STR_BUFFER_RESERVE` - -Two generic templated containers are used throughout the library: - -* `template< class Type> struct Array` -* `template< class Type> struct HashTable` - -Both Code and AST definitions have a `template< class Type> Code/AST cast()`. Its just an alternative way to explicitly cast to each other. - -`template< class Type> swap( Type& a, Type& b)` is used over a macro. - -Otherwise the library is free of any templates. - -The following CodeTypes are used which the user may optionally use strong typing with if they enable: `GEN_ENFORCE_STRONG_CODE_TYPES` - -* CodeBody : Has support for `for-range` iterating across Code objects. -* CodeAttributes -* CodeComment -* CodeClass -* CodeEnum -* CodeExec -* CodeExtern -* CodeInclude -* CodeFriend -* CodeFn -* CodeModule -* CodeNamespace -* CodeOperator -* CodeOpCast -* CodeParam : Has support for `for-range` iterating across parameters. -* CodeSpecifiers : Has support for `for-range` iterating across specifiers. -* CodeStruct -* CodeTemplate -* CodeType -* CodeTypedef -* CodeUnion -* CodeUsing -* CodeUsingNamespace -* CodeVar - -Each Code boy has an associated "filtered AST" with the naming convention: `AST_` -Unrelated fields of the AST for that node type are omitted and only necessary padding members are defined otherwise. -Retrieving a raw version of the ast can be done using the `raw()` function defined in each AST. - -## There are three sets of interfaces for Code AST generation the library provides - -* Upfront -* Parsing -* Untyped - -### Upfront Construction - -All component ASTs must be previously constructed, and provided on creation of the code AST. -The construction will fail and return Code::Invalid otherwise. - -Interface :`` - -* def_attributes - * *This is pre-appended right before the function symbol, or placed after the class or struct keyword for any flavor of attributes used.* - * *Its up to the user to use the desired attribute formatting: `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU).* -* def_comment -* def_class -* def_enum -* def_execution - * *This is equivalent to untyped_str, except that its intended for use only in execution scopes.* -* def_extern_link -* def_friend -* def_function -* def_include -* def_module -* def_namespace -* def_operator -* def_operator_cast -* def_param -* def_params -* def_specifier -* def_specifiers -* def_struct -* def_template -* def_type -* def_typedef -* def_union -* def_using -* def_using_namespace -* def_variable - -Bodies: - -* def_body -* def_class_body -* def_enum_body -* def_export_body -* def_extern_link_body -* def_function_body - * *Use this for operator bodies as well* -* def_global_body -* def_namespace_body -* def_struct_body -* def_union_body - -Usage: - -```cpp - = def_( ... ); - -Code -{ - ... - = def_( ... ); -} - -``` - -When using the body functions, its recommended to use the args macro to auto determine the number of arguments for the varadic: -```cpp -def_global_body( args( ht_entry, array_ht_entry, hashtable )); - -// instead of: -def_global_body( 3, ht_entry, array_ht_entry, hashtable ); -``` - -If a more incremental approach is desired for the body ASTs, `Code def_body( CodeT type )` can be used to create an empty body. -When the members have been populated use: `AST::validate_body` to verify that the members are valid entires for that type. - -### Parse construction - -A string provided to the API is parsed for the intended language construct. - -Interface : - -* parse_class -* parse_enum -* parse_export_body -* parse_extern_link -* parse_friend - * Purposefully are only support forward declares with this constructor. -* parse_function -* parse_global_body -* parse_namespace -* parse_operator -* parse_operator_cast -* parse_struct -* parse_template -* parse_type -* parse_typedef -* parse_union -* parse_using -* parse_variable - -The lexing and parsing takes shortcuts from whats expected in the standard. - -* Numeric literals are not check for validity. -* The parse API treats any execution scope definitions with no validation and are turned into untyped Code ASTs. - * *This includes the assignment of variables.* -* Attributes ( `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU) ) - * Assumed to *come before specifiers* (`const`, `constexpr`, `extern`, `static`, etc) for a function - * Or in the usual spot for class, structs, (*right after the declaration keyword*) - * typedefs have attributes with the type (`parse_type`) -* As a general rule; if its not available from the upfront constructors, its not available in the parsing constructors. - * *Upfront constructors are not necessarily used in the parsing constructors, this is just a good metric to know what can be parsed.* -* Parsing attributes can be extended to support user defined macros by defining `GEN_Define_Attribute_Tokens` (see `gen.hpp` for the formatting) - -Usage: - -```cpp -Code = parse_( string with code ); - -Code = def_( ..., parse_( - -)); - -Code = make_( ... ) -{ - ->add( parse_( - - )); -} -``` - -### Untyped constructions - -Code ASTs are constructed using unvalidated strings. - -Interface : - -* token_fmt_va -* token_fmt -* untyped_str -* untyped_fmt -* untyped_token_fmt - -During serialization any untyped Code AST has its string value directly injected inline of whatever context the content existed as an entry within. -Even though these are not validated from somewhat correct c/c++ syntax or components, it doesn't mean that Untyped code can be added as any component of a Code AST: - -* Untyped code cannot have children, thus there cannot be recursive injection this way. -* Untyped code can only be a child of a parent of body AST, or for values of an assignment (ex: variable assignment). - -These restrictions help prevent abuse of untyped code to some extent. - -Usage Conventions: - -```cpp -Code = def_variable( , , untyped_( - -)); - -Code = untyped_str( code( - -)); -``` - -Optionally, `code_str`, and `code_fmt` macros can be used so that the code macro doesn't have to be used: -```cpp -Code = code_str( ) -``` - -Template metaprogramming in the traditional sense becomes possible with the use of `token_fmt` and parse constructors: - -```cpp -StrC value = txt_StrC("Something"); - -char const* template_str = txt( - Code with to replace with token_values - ... -); -char const* gen_code_str = token_fmt( "key", value, template_str ); -Code = parse_( gen_code_str ); -``` - -## Predefined Codes - -The following are provided predefined by the library as they are commonly used: - -* `access_public` -* `access_protected` -* `access_private` -* `module_global_fragment` -* `module_private_fragment` -* `param_varaidc` (Used for varadic definitions) -* `pragma_once` -* `spec_const` -* `spec_consteval` -* `spec_constexpr` -* `spec_constinit` -* `spec_extern_linkage` (extern) -* `spec_final` -* `spec_global` (global macro) -* `spec_inline` -* `spec_internal_linkage` (internal macro) -* `spec_local_persist` (local_persist macro) -* `spec_mutable` -* `spec_override` -* `spec_ptr` -* `spec_ref` -* `spec_register` -* `spec_rvalue` -* `spec_static_member` (static) -* `spec_thread_local` -* `spec_virtual` -* `spec_volatile` -* `spec_type_signed` -* `spec_type_unsigned` -* `spec_type_short` -* `spec_type_long` -* `t_empty` (Used for varaidc macros) -* `t_auto` -* `t_void` -* `t_int` -* `t_bool` -* `t_char` -* `t_wchar_t` -* `t_class` -* `t_typename` - -Optionally the following may be defined if `GEN_DEFINE_LIBRARY_CODE_CONSTANTS` is defined - -* `t_b32` -* `t_s8` -* `t_s16` -* `t_s32` -* `t_s64` -* `t_u8` -* `t_u16` -* `t_u32` -* `t_u64` -* `t_sw` -* `t_uw` -* `t_f32` -* `t_f64` - -## Extent of operator overload validation - -The AST and constructors will be able to validate that the arguments provided for the operator type match the expected form: - -* If return type must match a parameter -* If number of parameters is correct -* If added as a member symbol to a class or struct, that operator matches the requirements for the class (types match up) - -The user is responsible for making sure the code types provided are correct -and have the desired specifiers assigned to them beforehand. - -## Code generation and modification - -There are three provided file interfaces: - -* Builder -* Editor -* Scanner - -Editor and Scanner are disabled by default, use `GEN_FEATURE_EDITOR` and `GEN_FEATURE_SCANNER` to enable them. - -### Builder is a similar object to the jai language's string_builder - -* The purpose of it is to generate a file. -* A file is specified and opened for writing using the open( file_path) ) function. -* The code is provided via print( code ) function will be serialized to its buffer. -* When all serialization is finished, use the write() command to write the buffer to the file. - -### Editor is for editing a series of files based on a set of requests provided to it - -**Note: Not implemented yet** - -* The purpose is to overrite a specific file, it places its contents in a buffer to scan. -* Requests are populated using the following interface: - * add : Add code. - * remove : Remove code. - * replace: Replace code. - -All three have the same parameters with exception to remove which only has SymbolInfo and Policy: - -* SymbolInfo: - * File : The file the symbol resides in. Leave null to indicate to search all files. Leave null to indicated all-file search. - * Marker : #define symbol that indicates a location or following signature is valid to manipulate. Leave null to indicate the signature should only be used. - * Signature : Use a Code symbol to find a valid location to manipulate, can be further filtered with the marker. Leave null to indicate the marker should only be used. - -* Policy : Additional policy info for completing the request (empty for now) -* Code : Code to inject if adding, or replace existing code with. - -Additionally if `GEN_FEATURE_EDITOR_REFACTOR` is defined, refactor( file_path, specification_path ) wil be made available. -Refactor is based of the refactor library and uses its interface. -It will on call add a request to the queue to run the refactor script on the file. - -### Scanner allows the user to generate Code ASTs by reading files - -**Note: Not implemented yet** - -* The purpose is to grab definitions to generate metadata or generate new code from these definitions. -* Requests are populated using the add( SymbolInfo, Policy ) function. The symbol info is the same as the one used for the editor. So is the case with Policy. - -The file will only be read from, no writing supported. - -One great use case is for example: generating the single-header library for gencpp! - -### Additional Info (Editor and Scanner) - -When all requests have been populated, call process_requests(). -It will provide an output of receipt data of the results when it completes. - -Files may be added to the Editor and Scanner additionally with add_files( num, files ). -This is intended for when you have requests that are for multiple files. - -Request queue in both Editor and Scanner are cleared once process_requests completes. +See the [scripts directory](scripts/). diff --git a/gencpp.vcxproj b/gencpp.vcxproj index 8057c56..0887f2c 100644 --- a/gencpp.vcxproj +++ b/gencpp.vcxproj @@ -112,12 +112,18 @@ + + + + + - + + @@ -125,25 +131,17 @@ + - - - - - + - - - - - @@ -160,18 +158,28 @@ + + + + + - + + + + + + + + - - - + @@ -184,6 +192,23 @@ + + + + + + + + + + + + + + + + + diff --git a/project/Readme.md b/project/Readme.md index b8b1ae0..5698371 100644 --- a/project/Readme.md +++ b/project/Readme.md @@ -1,47 +1,43 @@ # Documentation -The core library is contained within `gen.hpp` and `gen.cpp`. -Things related to the editor and scanner are in their own respective files. (Ex: `gen.scanner.` ) - -Dependencies are within `gen.dep.` - The library is fragmented into a series of headers and sources files meant to be scanned in and then generated to a tailored format for the target `gen` files. +The principal (user) files are `gen.hpp` and `gen.cpp`. +They contain includes for its various components: `components/.` + +Dependencies are bundled into `gen.dep.`. +Just like the `gen.` they include their components: `dependencies/.` + +The fle processors are in their own respective files. (Ex: `file_processors/.` ) +They directly include `depedencies/file_handling.` as the core library does not include file processing by defualt. + +**TODO : Right now the library is not finished structurally, as such the first self-hosting iteration is still WIP** Both libraries use *pre-generated* (self-hosting I guess) version of the library to then generate the latest version of itself. -(sort of a verification that the generated version is equivalent) +(sort of a verification that the generated version is equivalent). The default `gen.bootstrap.cpp` located in the project folder is meant to be produce a standard segmented library, where the components of the library -have relatively dedicated header and source files. With dependencies included at the top of the file and each header starting with a pragma once. -This will overwrite the existing library implementation in the immediate directory. +have relatively dedicated header and source files. Dependencies included at the top of the file and each header starting with a pragma once. +The output will be in the `project/gen` directory (if the directory does not exist, it will create it). Use those to get a general idea of how to make your own tailored version. -If the naming convention is undesired, the `gencpp.refactor` script can be used with the [refactor]() - Feature Macros: +* `GEN_DEFINE_ATTRIBUTE_TOKENS` : Allows user to define their own attribute macros for use in parsing. + * This is auto-generated if using the bootstrap or single-header generation + * *Note: The user will use the `AttributeTokens.csv` when the library is fully self-hosting.* +* `GEN_DEFINE_LIBRARY_CORE_CONSTANTS` : Optional typename codes as they are non-standard to C/C++ and not necessary to library usage * `GEN_DONT_USE_NAMESPACE` : By default, the library is wrapped in a `gen` namespace, this will disable that expose it to the global scope. * `GEN_DONT_ENFORCE_GEN_TIME_GUARD` : By default, the library ( gen.hpp/ gen.cpp ) expects the macro `GEN_TIME` to be defined, this disables that. -* `GEN_ROLL_OWN_DEPENDENCIES` : Optional override so that user may define the dependencies themselves. -* `GEN_DEFINE_LIBRARY_CORE_CONSTANTS` : Optional typename codes as they are non-standard to C/C++ and not necessary to library usage * `GEN_ENFORCE_STRONG_CODE_TYPES` : Enforces casts to filtered code types. * `GEN_EXPOSE_BACKEND` : Will expose symbols meant for internal use only. -* `GEN_Define_Attribute_Tokens` : Allows user to define their own attribute macros for use in parsing. - -`GEN_USE_RECURSIVE_AST_DUPLICATION` is available but its not well tested and should not need to be used. -If constructing ASTs properly. There should be no modification of ASTs, and thus this would never become an issue. -(I will probably remove down the line...) +* `GEN_ROLL_OWN_DEPENDENCIES` : Optional override so that user may define the dependencies themselves. ## On multi-threading Currently unsupported. The following changes would have to be made: -* Setup static data access with fences if more than one thread will generate ASTs ( or keep a different set for each thread) -* Make sure local persistent data of functions are also thread local. -* The builder should be done on a per-thread basis. -* Due to the design of the editor and scanner, it will most likely be best to make each file a job to process request entries on. Receipts should have an an array to store per thread. They can be combined to the final receipts array when all files have been processed. - ## Extending the library This library is relatively very small, and can be extended without much hassle. diff --git a/project/components/gen.ast.cpp b/project/components/ast.cpp similarity index 87% rename from project/components/gen.ast.cpp rename to project/components/ast.cpp index e9054aa..846af4d 100644 --- a/project/components/gen.ast.cpp +++ b/project/components/ast.cpp @@ -17,20 +17,9 @@ AST* AST::duplicate() String AST::to_string() { -# define ProcessModuleFlags() \ - if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) \ - result.append( "export " ); \ - \ - if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Import )) \ - result.append( "import " ); \ - local_persist thread_local char SerializationLevel = 0; -#if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION) - u64 time_start = time_rel_ms(); -#endif - // TODO : Need to refactor so that intermeidate strings are freed conviently. String result = String::make( GlobalAllocator, "" ); @@ -82,7 +71,8 @@ String AST::to_string() case Class: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes || ParentType ) { @@ -129,7 +119,8 @@ String AST::to_string() case Class_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "class %s %s;", Attributes->to_string(), Name ); @@ -140,7 +131,8 @@ String AST::to_string() case Enum: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes || UnderlyingType ) { @@ -170,7 +162,8 @@ String AST::to_string() case Enum_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -181,7 +174,8 @@ String AST::to_string() case Enum_Class: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes || UnderlyingType ) { @@ -219,7 +213,8 @@ String AST::to_string() case Enum_Class_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append( "enum class " ); @@ -262,7 +257,8 @@ String AST::to_string() case Function: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -301,7 +297,8 @@ String AST::to_string() case Function_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -347,7 +344,8 @@ String AST::to_string() break; case Namespace: - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append_fmt( "namespace %s\n{\n%s}" , Name @@ -358,7 +356,8 @@ String AST::to_string() case Operator: case Operator_Member: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -395,7 +394,8 @@ String AST::to_string() case Operator_Fwd: case Operator_Member_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -513,7 +513,8 @@ String AST::to_string() case Struct: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Name == nullptr) { @@ -566,7 +567,8 @@ String AST::to_string() case Struct_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "struct %s %s;", Attributes->to_string(), Name ); @@ -577,7 +579,8 @@ String AST::to_string() case Template: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append_fmt( "template< %s >\n%s", Params->to_string(), Declaration->to_string() ); } @@ -585,7 +588,8 @@ String AST::to_string() case Typedef: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append( "typedef "); @@ -624,7 +628,8 @@ String AST::to_string() case Union: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append( "union " ); @@ -650,7 +655,8 @@ String AST::to_string() case Using: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -675,7 +681,8 @@ String AST::to_string() case Variable: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes || Specs ) { @@ -726,11 +733,7 @@ String AST::to_string() break; } -#if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION) - log_fmt("AST::to_string() time taken: %llu for: %s\n", time_rel_ms() - time_start, result ); -#endif return result; -#undef ProcessModuleFlags } bool AST::is_equal( AST* other ) @@ -760,19 +763,19 @@ bool AST::validate_body() { using namespace ECode; -#define CheckEntries( Unallowed_Types ) \ - do \ - { \ - for ( Code entry : cast() ) \ - { \ - switch ( entry->Type ) \ - { \ - Unallowed_Types \ - log_failure( "AST::validate_body: Invalid entry in body %s", entry.debug_str() ); \ - return false; \ - } \ - } \ - } \ +#define CheckEntries( Unallowed_Types ) \ + do \ + { \ + for ( Code entry : cast() ) \ + { \ + switch ( entry->Type ) \ + { \ + Unallowed_Types \ + log_failure( "AST::validate_body: Invalid entry in body %s", entry.debug_str() ); \ + return false; \ + } \ + } \ + } \ while (0); switch ( Type ) @@ -825,6 +828,8 @@ bool AST::validate_body() } return false; + +#undef CheckEntries } #pragma endregion AST diff --git a/project/components/gen.ast_case_macros.cpp b/project/components/ast_case_macros.cpp similarity index 100% rename from project/components/gen.ast_case_macros.cpp rename to project/components/ast_case_macros.cpp diff --git a/project/components/gen.data_structures.hpp b/project/components/data_structures.hpp similarity index 100% rename from project/components/gen.data_structures.hpp rename to project/components/data_structures.hpp diff --git a/project/components/gen.ecode.hpp b/project/components/ecode.hpp similarity index 100% rename from project/components/gen.ecode.hpp rename to project/components/ecode.hpp diff --git a/project/components/gen.eoperator.hpp b/project/components/eoperator.hpp similarity index 98% rename from project/components/gen.eoperator.hpp rename to project/components/eoperator.hpp index 2d9ca26..70c99dd 100644 --- a/project/components/gen.eoperator.hpp +++ b/project/components/eoperator.hpp @@ -72,4 +72,4 @@ namespace EOperator # undef Define_Operators } -using OperatorT = EOperator::Type; \ No newline at end of file +using OperatorT = EOperator::Type; diff --git a/project/components/gen.especifier.hpp b/project/components/especifier.hpp similarity index 99% rename from project/components/gen.especifier.hpp rename to project/components/especifier.hpp index fea0f59..9affa84 100644 --- a/project/components/gen.especifier.hpp +++ b/project/components/especifier.hpp @@ -101,5 +101,4 @@ namespace ESpecifier # undef Define_Specifiers } - using SpecifierT = ESpecifier::Type; diff --git a/project/components/gen.etoktype.cpp b/project/components/etoktype.cpp similarity index 96% rename from project/components/gen.etoktype.cpp rename to project/components/etoktype.cpp index c20a285..5516221 100644 --- a/project/components/gen.etoktype.cpp +++ b/project/components/etoktype.cpp @@ -9,8 +9,8 @@ namespace Parser Attributes_Start is only used to indicate the start of the user_defined attribute list. */ -#ifndef GEN_Define_Attribute_Tokens -# define GEN_Define_Attribute_Tokens \ +#ifndef GEN_DEFINE_ATTRIBUTE_TOKENS +# define GEN_DEFINE_ATTRIBUTE_TOKENS \ Entry( API_Export, "GEN_API_Export_Code" ) \ Entry( API_Import, "GEN_API_Import_Code" ) #endif @@ -97,7 +97,7 @@ namespace Parser { # define Entry( Name_, Str_ ) Name_, Define_TokType - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS # undef Entry NumTokens, }; @@ -110,7 +110,7 @@ namespace Parser { # define Entry( Name_, Str_ ) { sizeof(Str_), Str_ }, Define_TokType - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS # undef Entry }; @@ -137,7 +137,7 @@ namespace Parser { # define Entry( Name_, Str_ ) Str_, Define_TokType - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS # undef Entry }; diff --git a/project/components/gen.header_end.hpp b/project/components/header_end.hpp similarity index 100% rename from project/components/gen.header_end.hpp rename to project/components/header_end.hpp diff --git a/project/components/gen.header_start.hpp b/project/components/header_start.hpp similarity index 100% rename from project/components/gen.header_start.hpp rename to project/components/header_start.hpp diff --git a/project/components/gen.impl_start.cpp b/project/components/impl_start.cpp similarity index 100% rename from project/components/gen.impl_start.cpp rename to project/components/impl_start.cpp diff --git a/project/components/gen.interface.cpp b/project/components/interface.cpp similarity index 98% rename from project/components/gen.interface.cpp rename to project/components/interface.cpp index f4be7a8..d1e7d56 100644 --- a/project/components/gen.interface.cpp +++ b/project/components/interface.cpp @@ -105,7 +105,7 @@ void define_constants() #endif # undef def_constant_code_type - t_empty = (CodeType) make_code(); + t_empty = (CodeType) make_code(); t_empty->Type = ECode::Typename; t_empty->Name = get_cached_string( txt_StrC("") ); t_empty.set_global(); @@ -127,12 +127,12 @@ void define_constants() access_private->Name = get_cached_string( txt_StrC("private:") ); access_private.set_global(); - access_protected = make_code(); + access_protected = make_code(); access_protected->Type = ECode::Access_Protected; access_protected->Name = get_cached_string( txt_StrC("protected:") ); access_protected.set_global(); - access_public = make_code(); + access_public = make_code(); access_public->Type = ECode::Access_Public; access_public->Name = get_cached_string( txt_StrC("public:") ); access_public.set_global(); diff --git a/project/components/gen.interface.hpp b/project/components/interface.hpp similarity index 100% rename from project/components/gen.interface.hpp rename to project/components/interface.hpp diff --git a/project/components/gen.interface.parsing.cpp b/project/components/interface.parsing.cpp similarity index 87% rename from project/components/gen.interface.parsing.cpp rename to project/components/interface.parsing.cpp index f7f4445..85dea41 100644 --- a/project/components/gen.interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -6,19 +6,13 @@ namespace Parser { struct Token { - // TokType Type; - // s32 Start; - // s32 End; - // s32 Line; - // s32 Column; - // TokFlags Flags; - char const* Text; sptr Length; TokType Type; bool IsAssign; s32 Line; s32 Column; + // TokFlags Flags; operator bool() { @@ -59,6 +53,8 @@ namespace Parser } }; + constexpr Token NullToken { nullptr, 0, TokType::Invalid, false, 0, 0 }; + struct TokArray { Array Arr; @@ -101,20 +97,48 @@ namespace Parser result.append_fmt("\tContext:\n"); + char* current = Tokens.current().Text; + sptr length = Tokens.current().Length; + while ( current != Tokens.back().Text && current != '\n' ) + { + current++; + length--; + } + + String line = String::make( GlobalAllocator, { length, Tokens.current().Text } ); + result.append_fmt("\t(%d, %d): %s", Tokens.current().Line, Tokens.current().Column, line ); + line.free(); + StackNode* current = Scope; do { - String name = String::make( GlobalAllocator, current->Name ? (StrC)current->Name : txt_StrC("Unresolved") ); + if ( current->Name ) + { + result.append_fmt("\tProcedure: %s, AST Name: %s\n", current->ProcName, (StrC)current->Name ); + } + else + { + result.append_fmt("\tProcedure: %s\n", current->ProcName ); + } - result.append_fmt("\tProcedure: %s, AST Name: %s\n\t(%d, %d):", current->ProcName, name ); current = current->Prev; name.free(); } while ( current ); - return result; } + + void push( StackNode* node ) + { + node->Prev = Scope; + Scope = node; + } + + void pop() + { + Context.Scope = Context.Scope->Prev; + } }; global ParseContext Context; @@ -123,7 +147,7 @@ namespace Parser { if ( Arr.num() - Idx <= 0 ) { - log_failure( "No tokens left\n", Context.Scope->ProcName ); + log_failure( "No tokens left.\n%s", Context.to_string() ); return false; } @@ -131,7 +155,7 @@ namespace Parser { String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } ); - log_failure( "gen::%s: expected %s, got %s", Context.Scope->ProcName, ETokType::to_str(type), ETokType::to_str(Arr[Idx].Type) ); + log_failure( "Parse Error, TokArray::eat, Expected: %s, not %s (%d, %d)`\n%s", ETokType::to_str(type), token_str, Context.to_string() ); return false; } @@ -217,6 +241,7 @@ namespace Parser switch ( current ) { + // TODO : Need to handle the preprocessor as a separate pass. case '#': token.Text = scanner; token.Length = 1; @@ -681,16 +706,16 @@ namespace Parser #pragma region Helper Macros -# define check_parse_args( func, def ) \ -if ( def.Len <= 0 ) \ -{ \ - log_failure( "gen::" stringize(func) ": length must greater than 0" ); \ - return CodeInvalid; \ -} \ -if ( def.Ptr == nullptr ) \ -{ \ - log_failure( "gen::" stringize(func) ": def was null" ); \ - return CodeInvalid; \ +# define check_parse_args( def ) \ +if ( def.Len <= 0 ) \ +{ \ + log_failure( "gen::" stringize( __func__ ) ": length must greater than 0" ); \ + return CodeInvalid; \ +} \ +if ( def.Ptr == nullptr ) \ +{ \ + log_failure( "gen::" stringize( __func__ ) ": def was null" ); \ + return CodeInvalid; \ } # define nexttok Context.Tokens.next() @@ -701,16 +726,12 @@ if ( def.Ptr == nullptr ) \ # define check( Type_ ) ( left && currtok.Type == Type_ ) -// # define +# define push_scope() \ + StackNode scope { nullptr, NullToken, txt_StrC( __func__ ) }; \ + Context.push( & scope ) #pragma endregion Helper Macros -struct ParseContext -{ - ParseContext* Parent; - char const* Fn; -}; - internal Code parse_function_body(); internal Code parse_global_nspace(); @@ -735,6 +756,7 @@ internal inline Code parse_array_decl() { using namespace Parser; + push_scope(); if ( check( TokType::BraceSquare_Open ) ) { @@ -742,13 +764,13 @@ Code parse_array_decl() if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of typedef definition ( '[]' scope started )", stringize(parse_typedef) ); + log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", Context.to_string() ); return Code::Invalid; } if ( currtok.Type == TokType::BraceSquare_Close ) { - log_failure( "%s: Error, empty array expression in typedef definition", stringize(parse_typedef) ); + log_failure( "Error, empty array expression in typedef definition\n%s", Context.to_string() ); return Code::Invalid; } @@ -765,13 +787,13 @@ Code parse_array_decl() if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of type definition, expected ]", stringize(parse_typedef) ); + log_failure( "Error, unexpected end of array declaration, expected ]\n%s", Context.to_string() ); return Code::Invalid; } if ( currtok.Type != TokType::BraceSquare_Close ) { - log_failure( "%s: Error, expected ] in type definition, not %s", stringize(parse_typedef), ETokType::to_str( currtok.Type ) ); + log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", ETokType::to_str( currtok.Type ), Context.to_string() ); return Code::Invalid; } @@ -779,6 +801,7 @@ Code parse_array_decl() return array_expr; } + Context.pop(); return { nullptr }; } @@ -786,6 +809,7 @@ internal inline CodeAttributes parse_attributes() { using namespace Parser; + push_scope(); Token start; s32 len = 0; @@ -847,6 +871,7 @@ CodeAttributes parse_attributes() return def_attributes( attribute_txt ); } + pop_context(); return { nullptr }; } @@ -854,6 +879,8 @@ internal inline Parser::Token parse_identifier() { using namespace Parser; + push_scope(); + Token name = currtok; eat( TokType::Identifier ); @@ -864,13 +891,13 @@ Parser::Token parse_identifier() if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of type definition, expected identifier", Context.to_string() ); + log_failure( "Error, unexpected end of static symbol identifier\n%s", Context.to_string() ); return { nullptr, 0, TokType::Invalid }; } if ( currtok.Type != TokType::Identifier ) { - log_failure( "%s: Error, expected identifier in type definition, not %s", context, ETokType::to_str( currtok.Type ) ); + log_failure( "Error, expected static symbol identifier, not %s\n%s", ETokType::to_str( currtok.Type ), Context.to_string() ); return { nullptr, 0, TokType::Invalid }; } @@ -878,6 +905,33 @@ Parser::Token parse_identifier() eat( TokType::Identifier ); } + if ( check( TokType::Operator ) && currtok.Text[0] == '<' ) + { + // Template arguments can be complex so were not validating if they are correct. + s32 level = 0; + while ( left && (currtok.Text[0] != '>' || level > 0 ) ) + { + if ( currtok.Text[0] == '<' ) + level++; + + else if ( currtok.Text[0] == '>' && level > 0 ) + level--; + + eat( currtok.Type ); + } + + if ( left == 0 ) + { + log_failure( "Error, unexpected end of template arguments\n%s", Context.to_string() ); + return { nullptr, 0, TokType::Invalid }; + } + + eat( TokType::Operator ); + + name.Length = ( (sptr)prevtok.Text + (sptr)prevtok.Length ) - (sptr)name.Text; + } + + pop_context(); return name; } @@ -886,6 +940,7 @@ CodeParam parse_params( bool use_template_capture = false ) { using namespace Parser; using namespace ECode; + push_scope(); if ( ! use_template_capture ) eat( TokType::Capture_Start ); @@ -912,7 +967,7 @@ CodeParam parse_params( bool use_template_capture = false ) return param_varadic; } - type = parse_type( toks, context ); + type = parse_type(); if ( type == Code::Invalid ) return CodeInvalid; @@ -931,7 +986,7 @@ CodeParam parse_params( bool use_template_capture = false ) if ( currtok.Type == TokType::Statement_End ) { - log_failure( "gen::%s: Expected value after assignment operator", context ); + log_failure( "Expected value after assignment operator\n%s.", Context.to_string() ); return CodeInvalid; } @@ -995,7 +1050,7 @@ CodeParam parse_params( bool use_template_capture = false ) if ( currtok.Type == TokType::Statement_End ) { - log_failure( "gen::%s: Expected value after assignment operator", context ); + log_failure( "Expected value after assignment operator\n%s", Context.to_string() ); return CodeInvalid; } @@ -1031,12 +1086,13 @@ CodeParam parse_params( bool use_template_capture = false ) { if ( ! check( TokType::Operator) || currtok.Text[0] != '>' ) { - log_failure("gen::parse_params: expected '<' after 'template' keyword. %s", ETokType::to_str( currtok.Type )); + log_failure("Expected '<' after 'template' keyword\n%s", Context.to_string() ); return CodeInvalid; } eat( TokType::Operator ); } + Context.pop(); return result; # undef context } @@ -1052,6 +1108,7 @@ CodeFn parse_function_after_name( ) { using namespace Parser; + push_scope(); CodeParam params = parse_params( toks, stringize(parse_function) ); @@ -1090,7 +1147,7 @@ CodeFn parse_function_after_name( default: { - log_failure("gen::def_function: body must be either of Function_Body or Untyped type. %s", body.debug_str()); + log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", body.debug_str(), Context.to_string()); return CodeInvalid; } } @@ -1111,26 +1168,28 @@ CodeFn parse_function_after_name( if ( params ) result->Params = params; + Context.pop(); return result; } internal inline -CodeOperator parse_operator_after_ret_type( - ModuleFlag mflags +CodeOperator parse_operator_after_ret_type( + ModuleFlag mflags , CodeAttributes attributes - , CodeSpecifiers specifiers + , CodeSpecifiers specifiers , CodeType ret_type ) { using namespace Parser; using namespace EOperator; + push_scope(); // Parse Operator eat( TokType::Decl_Operator ); if ( ! check( TokType::Operator ) ) { - log_failure( "gen::%s: Expected operator after 'operator' keyword", context ); + log_failure( "Expected operator after 'operator' keyword\n%s", Context.to_string() ); return CodeInvalid; } @@ -1316,7 +1375,7 @@ CodeOperator parse_operator_after_ret_type( if ( op == Invalid ) { - log_failure( "gen::%s: Invalid operator '%s'", context, currtok.Text ); + log_failure( "Invalid operator '%s'\n%s", currtok.Text, Context.to_string() ); return CodeInvalid; } @@ -1346,6 +1405,7 @@ CodeOperator parse_operator_after_ret_type( // OpValidateResult check_result = operator__validate( op, params, ret_type, specifiers ); CodeOperator result = def_operator( op, params, ret_type, body, specifiers, attributes, mflags ); + Context.pop(); return result; } @@ -1357,14 +1417,13 @@ CodeVar parse_variable_after_name( ,CodeSpecifiers specifiers , CodeType type , StrC name - , Parser::TokArray& toks - , char const* context ) +) { using namespace Parser; + push_scope(); - Code array_expr = parse_array_decl( toks, stringize(parse_variable) ); - - Code expr = { nullptr }; + Code array_expr = parse_array_decl(); + Code expr = { nullptr }; if ( currtok.IsAssign ) { @@ -1374,7 +1433,7 @@ CodeVar parse_variable_after_name( if ( currtok.Type == TokType::Statement_End ) { - log_failure( "gen::parse_variable: expected expression after assignment operator" ); + log_failure( "Expected expression after assignment operator\n%s", Context.to_string() ); return CodeInvalid; } @@ -1411,6 +1470,7 @@ CodeVar parse_variable_after_name( if ( expr ) result->Value = expr; + Context.pop(); return result; } @@ -1418,6 +1478,7 @@ internal inline Code parse_variable_assignment() { using namespace Parser; + push_scope(); Code expr = Code::Invalid; @@ -1429,7 +1490,7 @@ Code parse_variable_assignment() if ( currtok.Type == TokType::Statement_End ) { - log_failure( "gen::parse_variable: expected expression after assignment operator" ); + log_failure( "Expected expression after assignment operator\n%s", Context.to_string() ); return Code::Invalid; } @@ -1442,6 +1503,7 @@ Code parse_variable_assignment() expr = untyped_str( expr_tok ); } + Context.pop(); return expr; } @@ -1449,6 +1511,7 @@ internal inline Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ) { using namespace Parser; + push_scope(); Code result = Code::Invalid; @@ -1460,7 +1523,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes if ( check( TokType::Operator) ) { // Dealing with an operator overload - result = parse_operator_after_ret_type( ModuleFlag::None, attributes, specifiers, type, toks, context ); + result = parse_operator_after_ret_type( ModuleFlag::None, attributes, specifiers, type ); } else { @@ -1471,21 +1534,22 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes { // Dealing with a function - result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, type, name, toks, context ); + result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, type, name ); } else { if ( expects_function ) { - log_failure( "gen::parse_operator_function_or_variable: expected function declaration (consteval was used)" ); + log_failure( "Expected function declaration (consteval was used)\n%s", Context.to_string() ); return Code::Invalid; } // Dealing with a variable - result = parse_variable_after_name( ModuleFlag::None, attributes, specifiers, type, name, toks, context ); + result = parse_variable_after_name( ModuleFlag::None, attributes, specifiers, type, name ); } } + Context.pop(); return result; } @@ -1494,6 +1558,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) { using namespace Parser; using namespace ECode; + push_scope(); eat( TokType::BraceCurly_Open ); @@ -1579,7 +1644,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) case TokType::Decl_GNU_Attribute: case TokType::Decl_MSVC_Attribute: #define Entry( attribute, str ) case TokType::attribute: - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { attributes = parse_attributes(); @@ -1615,7 +1680,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) break; default: - log_failure( "gen::parse_class_struct_body: invalid specifier " "%s" " for variable", ESpecifier::to_str(spec) ); + log_failure( "Invalid specifier %s for variable\n%s", ESpecifier::to_str(spec), Context.to_string() ); return CodeInvalid; } @@ -1640,7 +1705,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) case TokType::Type_int: case TokType::Type_double: { - member = parse_operator_function_or_variable( expects_function, attributes, specifiers, toks, context ); + member = parse_operator_function_or_variable( expects_function, attributes, specifiers ); } break; @@ -1659,7 +1724,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) if ( member == Code::Invalid ) { - log_failure( "gen::parse_variable: failed to parse member" ); + log_failure( "Failed to parse member\n%s", Context.to_string() ); return CodeInvalid; } @@ -1667,6 +1732,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) } eat( TokType::BraceCurly_Close ); + Context.pop(); return result; } @@ -1674,10 +1740,11 @@ internal Code parse_class_struct( Parser::TokType which ) { using namespace Parser; + push_scope(); if ( which != TokType::Decl_Class && which != TokType::Decl_Struct ) { - log_failure( "%s: Error, expected class or struct, not %s", context, ETokType::to_str( which ) ); + log_failure( "Error, expected class or struct, not %s\n%s", ETokType::to_str( which ), Context.to_string() ); return CodeInvalid; } @@ -1755,6 +1822,7 @@ Code parse_class_struct( Parser::TokType which ) ); interfaces.free(); + Context.pop(); return result; } @@ -1763,6 +1831,7 @@ Code parse_function_body() { using namespace Parser; using namespace ECode; + push_scope(); eat( TokType::BraceCurly_Open ); @@ -1794,6 +1863,8 @@ Code parse_function_body() } eat( TokType::BraceCurly_Close ); + + Context.pop(); return result; } @@ -1802,6 +1873,7 @@ CodeBody parse_global_nspace( CodeT which ) { using namespace Parser; using namespace ECode; + push_scope(); if ( which != Namespace_Body && which != Global_Body && which != Export_Body && which != Extern_Linkage_Body ) return CodeInvalid; @@ -1838,7 +1910,7 @@ CodeBody parse_global_nspace( CodeT which ) case TokType::Decl_Extern_Linkage: if ( which == Extern_Linkage_Body ) - log_failure( "gen::parse_global_nspace: nested extern linkage" ); + log_failure( "Nested extern linkage\n%s", Context.to_string() ); member = parse_extern_link_body( toks, context ); break; @@ -1869,7 +1941,7 @@ CodeBody parse_global_nspace( CodeT which ) case TokType::Module_Export: if ( which == Export_Body ) - log_failure( "gen::parse_global_nspace: nested export declaration" ); + log_failure( "Nested export declaration\n%s", Context.to_string() ); member = parse_export_body( toks, context ); break; @@ -1882,7 +1954,7 @@ CodeBody parse_global_nspace( CodeT which ) case TokType::Decl_GNU_Attribute: case TokType::Decl_MSVC_Attribute: #define Entry( attribute, str ) case TokType::attribute: - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { attributes = parse_attributes(); @@ -1919,7 +1991,7 @@ CodeBody parse_global_nspace( CodeT which ) break; default: - log_failure( "gen::parse_global_nspace: invalid specifier " "%s" " for variable", ESpecifier::to_str(spec) ); + log_failure( "Invalid specifier %s for variable\n%s", ESpecifier::to_str(spec), Context.to_string() ); return CodeInvalid; } @@ -1950,7 +2022,7 @@ CodeBody parse_global_nspace( CodeT which ) if ( member == Code::Invalid ) { - log_failure( "gen::%s: failed to parse member", context ); + log_failure( "Failed to parse member\n%s", Context.to_string() ); return CodeInvalid; } @@ -1960,13 +2032,17 @@ CodeBody parse_global_nspace( CodeT which ) if ( which != Global_Body ) eat( TokType::BraceCurly_Close ); + Context.pop(); return result; } internal CodeClass parse_class() { - return (CodeClass) parse_class_struct( Parser::TokType::Decl_Class, toks, context ); + push_scope(); + CodeClass result = (CodeClass) parse_class_struct( Parser::TokType::Decl_Class, toks, context ); + Context.pop(); + return result; } CodeClass parse_class( StrC def ) @@ -1978,7 +2054,10 @@ CodeClass parse_class( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return (CodeClass) parse_class_struct( TokType::Decl_Class, toks, stringize(parse_class) ); + push_scope(); + CodeClass result = (CodeClass) parse_class_struct( TokType::Decl_Class ); + Context.pop(); + return result; } internal @@ -1986,6 +2065,7 @@ CodeEnum parse_enum() { using namespace Parser; using namespace ECode; + push_scope(); SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; @@ -2012,7 +2092,7 @@ CodeEnum parse_enum() if ( currtok.Type != TokType::Identifier ) { - log_failure( "gen::parse_enum: expected identifier for enum name" ); + log_failure( "Expected identifier for enum name\n%s", Context.to_string() ); return CodeInvalid; } @@ -2085,6 +2165,7 @@ CodeEnum parse_enum() if ( type ) result->UnderlyingType = type; + Context.pop(); return result; } @@ -2097,13 +2178,16 @@ CodeEnum parse_enum( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_enum( toks, stringize(parse_enum) ); + return parse_enum(); } internal inline CodeBody parse_export_body() { - return parse_global_nspace( ECode::Export_Body, toks, context ); + push_scope(); + CodeBody result = parse_global_nspace( ECode::Export_Body ); + Context.pop(); + return result; } CodeBody parse_export_body( StrC def ) @@ -2115,19 +2199,23 @@ CodeBody parse_export_body( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_export_body( toks, stringize(parse_export_body) ); + return parse_export_body(); } internal inline -CodeBody parse_extern_link_body( Parser::TokArray& toks, char const* context ) +CodeBody parse_extern_link_body() { - return parse_global_nspace( ECode::Extern_Linkage_Body, toks, context ); + push_scope(); + CodeBody result = parse_global_nspace( ECode::Extern_Linkage_Body ); + Context.pop(); + return result; } internal -CodeExtern parse_extern_link( Parser::TokArray& toks, char const* context ) +CodeExtern parse_extern_link() { using namespace Parser; + push_scope(); eat( TokType::Decl_Extern_Linkage ); @@ -2145,12 +2233,13 @@ CodeExtern parse_extern_link( Parser::TokArray& toks, char const* context ) Code entry = parse_extern_link_body( toks, context ); if ( entry == Code::Invalid ) { - log_failure( "gen::parse_extern_link: failed to parse body" ); + log_failure( "Failed to parse body\n%s", Context.to_string() ); return result; } result->Body = entry; + Context.pop(); return result; } @@ -2163,14 +2252,15 @@ CodeExtern parse_extern_link( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_extern_link( toks, stringize(parse_extern_link) ); + return parse_extern_link(); } internal -CodeFriend parse_friend( Parser::TokArray& toks, char const* context ) +CodeFriend parse_friend() { using namespace Parser; using namespace ECode; + push_scope(); eat( TokType::Decl_Friend ); @@ -2211,6 +2301,7 @@ CodeFriend parse_friend( Parser::TokArray& toks, char const* context ) else result->Declaration = type; + Context.pop(); return result; } @@ -2223,13 +2314,14 @@ CodeFriend parse_friend( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_friend( toks, stringize(parse_friend) ); + return parse_friend(); } internal -CodeFn parse_functon( Parser::TokArray& toks, char const* context ) +CodeFn parse_functon() { using namespace Parser; + push_scope(); SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; @@ -2261,7 +2353,7 @@ CodeFn parse_functon( Parser::TokArray& toks, char const* context ) break; default: - log_failure( "gen::parse_functon: invalid specifier " "%s" " for functon", ESpecifier::to_str(spec) ); + log_failure( "Invalid specifier %s for functon\n%s", ESpecifier::to_str(spec), Context.to_string() ); return CodeInvalid; } @@ -2286,22 +2378,22 @@ CodeFn parse_functon( Parser::TokArray& toks, char const* context ) if ( ! name ) return CodeInvalid; - CodeFn result = parse_function_after_name( mflags, attributes, specifiers, ret_type, name, toks, context ); + CodeFn result = parse_function_after_name( mflags, attributes, specifiers, ret_type, name ); + Context.pop(); return result; } CodeFn parse_function( StrC def ) { - using namespace Parser; - check_parse_args( parse_function, def ); + using namespace Parser; TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return CodeInvalid; - return (CodeFn) parse_functon( toks, stringize(parse_function) ); + return (CodeFn) parse_functon(); } CodeBody parse_global_body( StrC def ) @@ -2313,13 +2405,18 @@ CodeBody parse_global_body( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_global_nspace( ECode::Global_Body, toks, stringize(parse_global_body) ); + push_scope(); + CodeBody result = parse_global_nspace( ECode::Global_Body ); + Context.pop(); + return result; } internal CodeNamespace parse_namespace( Parser::TokArray& toks, char const* context ) { using namespace Parser; + push_scope(); + eat( TokType::Decl_Namespace ); Token name = parse_identifier( toks, stringize(parse_namespace) ); @@ -2335,6 +2432,7 @@ CodeNamespace parse_namespace( Parser::TokArray& toks, char const* context ) result->Body = body; + Context.pop(); return result; } @@ -2347,13 +2445,14 @@ CodeNamespace parse_namespace( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_namespace( toks, stringize(parse_namespace) ); + return parse_namespace(); } internal -CodeOperator parse_operator( Parser::TokArray& toks, char const* context ) +CodeOperator parse_operator() { using namespace Parser; + push_scope(); CodeAttributes attributes = { nullptr }; CodeSpecifiers specifiers = { nullptr }; @@ -2383,7 +2482,7 @@ CodeOperator parse_operator( Parser::TokArray& toks, char const* context ) break; default: - log_failure( "gen::parse_operator: invalid specifier " "%s" " for operator", ESpecifier::to_str(spec) ); + log_failure( "Invalid specifier " "%s" " for operator\n%s", ESpecifier::to_str(spec), Context.to_string() ); return CodeInvalid; } @@ -2403,7 +2502,9 @@ CodeOperator parse_operator( Parser::TokArray& toks, char const* context ) // Parse Return Type CodeType ret_type = parse_type( toks, stringize(parse_operator) ); - CodeOperator result = parse_operator_after_ret_type( mflags, attributes, specifiers, ret_type, toks, context ); + CodeOperator result = parse_operator_after_ret_type( mflags, attributes, specifiers, ret_type ); + + Context.pop(); return result; } @@ -2416,12 +2517,13 @@ CodeOperator parse_operator( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return (CodeOperator) parse_operator( toks, stringize(parse_operator) ); + return (CodeOperator) parse_operator(); } -CodeOpCast parse_operator_cast( Parser::TokArray& toks, char const* context ) +CodeOpCast parse_operator_cast() { using namespace Parser; + push_scope(); eat( TokType::Decl_Operator ); @@ -2479,6 +2581,7 @@ CodeOpCast parse_operator_cast( Parser::TokArray& toks, char const* context ) result->ValueType = type; + Context.pop(); return result; } @@ -2491,13 +2594,16 @@ CodeOpCast parse_operator_cast( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_operator_cast( toks, stringize(parse_operator_cast) ); + return parse_operator_cast(); } internal inline -CodeStruct parse_struct( Parser::TokArray& toks, char const* context ) +CodeStruct parse_struct() { - return (CodeStruct) parse_class_struct( Parser::TokType::Decl_Struct, toks, stringize(parse_struct) ); + push_scope(); + CodeStruct result = (CodeStruct) parse_class_struct(); + Context.pop(); + return result; } CodeStruct parse_struct( StrC def ) @@ -2509,15 +2615,16 @@ CodeStruct parse_struct( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return (CodeStruct) parse_class_struct( TokType::Decl_Struct, toks, stringize(parse_struct) ); + return (CodeStruct) parse_class_struct( TokType::Decl_Struct ); } internal -CodeTemplate parse_template( Parser::TokArray& toks, char const* context ) +CodeTemplate parse_template() { # define UseTemplateCapture true using namespace Parser; + push_scope(); ModuleFlag mflags = ModuleFlag::None; @@ -2592,7 +2699,7 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context ) break; default: - log_failure( "gen::parse_template: invalid specifier %s for variable or function", ESpecifier::to_str( spec ) ); + log_failure( "Invalid specifier %s for variable or function\n%s", ESpecifier::to_str( spec ), Context.to_string() ); return CodeInvalid; } @@ -2621,6 +2728,7 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context ) result->Declaration = definition; result->ModuleFlags = mflags; + Context.pop(); return result; # undef UseTemplateCapture } @@ -2634,13 +2742,14 @@ CodeTemplate parse_template( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_template( toks, stringize(parse_template) ); + return parse_template(); } internal -CodeType parse_type( Parser::TokArray& toks, char const* context ) +CodeType parse_type() { using namespace Parser; + push_scope(); Token context_tok = prevtok; @@ -2658,7 +2767,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) if ( spec != ESpecifier::Const ) { - log_failure( "gen::parse_type: Error, invalid specifier used in type definition: %s", currtok.Text ); + log_failure( "Error, invalid specifier used in type definition: %s\n%s", currtok.Text, Context.to_string() ); return CodeInvalid; } @@ -2669,7 +2778,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of type definition", context ); + log_failure( "Error, unexpected end of type definition\n%s", Context.to_string() ); return CodeInvalid; } @@ -2734,7 +2843,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) && spec != ESpecifier::Ref && spec != ESpecifier::RValue ) { - log_failure( "%s: Error, invalid specifier used in type definition: %s", context, currtok.Text ); + log_failure( "Error, invalid specifier used in type definition: %s\n%s", currtok.Text, Context.to_string() ); return CodeInvalid; } @@ -2809,6 +2918,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) if ( attributes ) result->Attributes = attributes; + Context.pop(); return result; } @@ -2821,15 +2931,14 @@ CodeType parse_type( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - CodeType result = parse_type( toks, stringize(parse_type) ); - - return result; + return parse_type(); } internal -CodeTypedef parse_typedef( Parser::TokArray& toks, char const* context ) +CodeTypedef parse_typedef() { using namespace Parser; + push_scope(); Token name = { nullptr, 0, TokType::Invalid }; Code array_expr = { nullptr }; @@ -2862,7 +2971,7 @@ CodeTypedef parse_typedef( Parser::TokArray& toks, char const* context ) if ( ! check( TokType::Identifier ) ) { - log_failure( "gen::parse_typedef: Error, expected identifier for typedef" ); + log_failure( "Error, expected identifier for typedef\n%s", Context.to_string() ); return CodeInvalid; } @@ -2886,6 +2995,7 @@ CodeTypedef parse_typedef( Parser::TokArray& toks, char const* context ) if ( type->Type == Typename && array_expr && array_expr->Type != Invalid ) type.cast()->ArrExpr = array_expr; + Context.pop(); return result; } @@ -2898,13 +3008,14 @@ CodeTypedef parse_typedef( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_typedef( toks, stringize(parse_typedef) ); + return parse_typedef(); } internal -CodeUnion parse_union( Parser::TokArray& toks, char const* context ) +CodeUnion parse_union() { using namespace Parser; + push_scope(); ModuleFlag mflags = ModuleFlag::None; @@ -2958,6 +3069,7 @@ CodeUnion parse_union( Parser::TokArray& toks, char const* context ) if ( attributes ) result->Attributes = attributes; + Context.pop(); return result; } @@ -2970,13 +3082,14 @@ CodeUnion parse_union( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_union( toks, stringize(parse_union) ); + return parse_union(); } internal -CodeUsing parse_using( Parser::TokArray& toks, char const* context ) +CodeUsing parse_using() { using namespace Parser; + push_scope(); SpecifierT specs_found[16] { ESpecifier::Invalid }; s32 NumSpecifiers = 0; @@ -3044,6 +3157,8 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context ) if ( attributes ) result->Attributes = attributes; } + + Context.pop(); return result; } @@ -3056,13 +3171,14 @@ CodeUsing parse_using( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_using( toks, stringize(parse_using) ); + return parse_using(); } internal CodeVar parse_variable() { using namespace Parser; + push_scope(); SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; @@ -3099,7 +3215,7 @@ CodeVar parse_variable() break; default: - log_failure( "gen::parse_variable: invalid specifier %s for variable", ESpecifier::to_str( spec ) ); + log_failure( "Invalid specifier %s for variable\n%s", ESpecifier::to_str( spec ), Context.to_string() ); return CodeInvalid; } @@ -3125,33 +3241,30 @@ CodeVar parse_variable() Context.Scope->Name = current; eat( TokType::Identifier ); - CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, Context.Scope->Name, Context.Tokens, Context.Scope->ProcName ); + CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, Context.Scope->Name ); + Context.pop(); return result; } CodeVar parse_variable( StrC def ) { - using namespace Parser; check_parse_args( parse_variable, def ); + using namespace Parser; TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return CodeInvalid; - Context.Tokens = toks; - Parser::StackNode root - { - toks.current(), - { nullptr, 0, TokType::Invalid }, - name(parse_variable) - }; - return parse_variable(); } // Undef helper macros # undef check_parse_args -# undef curr_tok +# undef nexttok +# undef currtok +# undef prevtok # undef eat # undef left +# undef check +# undef push_scope diff --git a/project/components/gen.interface.upfront.cpp b/project/components/interface.upfront.cpp similarity index 100% rename from project/components/gen.interface.upfront.cpp rename to project/components/interface.upfront.cpp diff --git a/project/components/gen.data.cpp b/project/components/static_data.cpp similarity index 100% rename from project/components/gen.data.cpp rename to project/components/static_data.cpp diff --git a/project/components/gen.types.hpp b/project/components/types.hpp similarity index 96% rename from project/components/gen.types.hpp rename to project/components/types.hpp index 20f201f..0e9d6a3 100644 --- a/project/components/gen.types.hpp +++ b/project/components/types.hpp @@ -68,7 +68,7 @@ ModuleFlag operator|( ModuleFlag A, ModuleFlag B) Override these to change the attribute to your own unique identifier convention. - The tokenizer identifies attribute defines with the GEN_Define_Attribute_Tokens macros. + The tokenizer identifies attribute defines with the GEN_DEFINE_ATTRIBUTE_TOKENS macros. See the example below and the Define_TokType macro used in gen.cpp to know the format. While the library can parse raw attributes, most projects use defines to wrap them for compiler platform indendence. The token define allows support for them without having to modify the library. diff --git a/project/components/gen.untyped.cpp b/project/components/untyped.cpp similarity index 97% rename from project/components/gen.untyped.cpp rename to project/components/untyped.cpp index d33ca23..12f4971 100644 --- a/project/components/gen.untyped.cpp +++ b/project/components/untyped.cpp @@ -12,8 +12,7 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va ) char tok_map_mem[ TokenFmt_TokenMap_MemSize ]; tok_map_arena = Arena::init_from_memory( tok_map_mem, sizeof(tok_map_mem) ); - - tok_map = HashTable::init( tok_map_arena ); + tok_map = HashTable::init( tok_map_arena ); s32 left = num_tokens - 1; diff --git a/project/dependencies/gen.basic_types.hpp b/project/dependencies/basic_types.hpp similarity index 100% rename from project/dependencies/gen.basic_types.hpp rename to project/dependencies/basic_types.hpp diff --git a/project/dependencies/gen.containers.hpp b/project/dependencies/containers.hpp similarity index 100% rename from project/dependencies/gen.containers.hpp rename to project/dependencies/containers.hpp diff --git a/project/dependencies/gen.debug.cpp b/project/dependencies/debug.cpp similarity index 100% rename from project/dependencies/gen.debug.cpp rename to project/dependencies/debug.cpp diff --git a/project/dependencies/gen.debug.hpp b/project/dependencies/debug.hpp similarity index 100% rename from project/dependencies/gen.debug.hpp rename to project/dependencies/debug.hpp diff --git a/project/dependencies/gen.file_handling.cpp b/project/dependencies/file_handling.cpp similarity index 100% rename from project/dependencies/gen.file_handling.cpp rename to project/dependencies/file_handling.cpp diff --git a/project/dependencies/gen.file_handling.hpp b/project/dependencies/file_handling.hpp similarity index 100% rename from project/dependencies/gen.file_handling.hpp rename to project/dependencies/file_handling.hpp diff --git a/project/dependencies/gen.hashing.cpp b/project/dependencies/hashing.cpp similarity index 100% rename from project/dependencies/gen.hashing.cpp rename to project/dependencies/hashing.cpp diff --git a/project/dependencies/gen.hashing.hpp b/project/dependencies/hashing.hpp similarity index 100% rename from project/dependencies/gen.hashing.hpp rename to project/dependencies/hashing.hpp diff --git a/project/dependencies/gen.header_start.hpp b/project/dependencies/header_start.hpp similarity index 100% rename from project/dependencies/gen.header_start.hpp rename to project/dependencies/header_start.hpp diff --git a/project/dependencies/gen.impl_start.cpp b/project/dependencies/impl_start.cpp similarity index 100% rename from project/dependencies/gen.impl_start.cpp rename to project/dependencies/impl_start.cpp diff --git a/project/dependencies/gen.macros.hpp b/project/dependencies/macros.hpp similarity index 100% rename from project/dependencies/gen.macros.hpp rename to project/dependencies/macros.hpp diff --git a/project/dependencies/gen.memory.cpp b/project/dependencies/memory.cpp similarity index 100% rename from project/dependencies/gen.memory.cpp rename to project/dependencies/memory.cpp diff --git a/project/dependencies/gen.memory.hpp b/project/dependencies/memory.hpp similarity index 100% rename from project/dependencies/gen.memory.hpp rename to project/dependencies/memory.hpp diff --git a/project/dependencies/gen.parsing.cpp b/project/dependencies/parsing.cpp similarity index 100% rename from project/dependencies/gen.parsing.cpp rename to project/dependencies/parsing.cpp diff --git a/project/dependencies/gen.parsing.hpp b/project/dependencies/parsing.hpp similarity index 100% rename from project/dependencies/gen.parsing.hpp rename to project/dependencies/parsing.hpp diff --git a/project/dependencies/gen.printing.cpp b/project/dependencies/printing.cpp similarity index 100% rename from project/dependencies/gen.printing.cpp rename to project/dependencies/printing.cpp diff --git a/project/dependencies/gen.printing.hpp b/project/dependencies/printing.hpp similarity index 86% rename from project/dependencies/gen.printing.hpp rename to project/dependencies/printing.hpp index 9914e97..567bee9 100644 --- a/project/dependencies/gen.printing.hpp +++ b/project/dependencies/printing.hpp @@ -10,12 +10,15 @@ struct FileInfo; char* str_fmt_buf ( char const* fmt, ... ); char* str_fmt_buf_va ( char const* fmt, va_list va ); sw str_fmt_va ( char* str, sw n, char const* fmt, va_list va ); -sw str_fmt_file ( FileInfo* f, char const* fmt, ... ); -sw str_fmt_file_va ( FileInfo* f, char const* fmt, va_list va ); sw str_fmt_out_va ( char const* fmt, va_list va ); sw str_fmt_out_err ( char const* fmt, ... ); sw str_fmt_out_err_va( char const* fmt, va_list va ); +// TODO : Move these to file handling. + +sw str_fmt_file ( FileInfo* f, char const* fmt, ... ); +sw str_fmt_file_va ( FileInfo* f, char const* fmt, va_list va ) + constexpr char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED"; diff --git a/project/dependencies/gen.string.cpp b/project/dependencies/string.cpp similarity index 100% rename from project/dependencies/gen.string.cpp rename to project/dependencies/string.cpp diff --git a/project/dependencies/gen.string.hpp b/project/dependencies/string.hpp similarity index 100% rename from project/dependencies/gen.string.hpp rename to project/dependencies/string.hpp diff --git a/project/dependencies/gen.string_ops.cpp b/project/dependencies/string_ops.cpp similarity index 100% rename from project/dependencies/gen.string_ops.cpp rename to project/dependencies/string_ops.cpp diff --git a/project/dependencies/gen.string_ops.hpp b/project/dependencies/string_ops.hpp similarity index 100% rename from project/dependencies/gen.string_ops.hpp rename to project/dependencies/string_ops.hpp diff --git a/project/dependencies/gen.timing.cpp b/project/dependencies/timing.cpp similarity index 100% rename from project/dependencies/gen.timing.cpp rename to project/dependencies/timing.cpp diff --git a/project/dependencies/gen.timing.hpp b/project/dependencies/timing.hpp similarity index 100% rename from project/dependencies/gen.timing.hpp rename to project/dependencies/timing.hpp diff --git a/project/components/AttributeTokens.csv b/project/enums/AttributeTokens.csv similarity index 100% rename from project/components/AttributeTokens.csv rename to project/enums/AttributeTokens.csv diff --git a/project/components/ECode.csv b/project/enums/ECode.csv similarity index 100% rename from project/components/ECode.csv rename to project/enums/ECode.csv diff --git a/project/components/EOperator.csv b/project/enums/EOperator.csv similarity index 100% rename from project/components/EOperator.csv rename to project/enums/EOperator.csv diff --git a/project/components/ESpecifier.csv b/project/enums/ESpecifier.csv similarity index 100% rename from project/components/ESpecifier.csv rename to project/enums/ESpecifier.csv diff --git a/project/components/ETokType.csv b/project/enums/ETokType.csv similarity index 100% rename from project/components/ETokType.csv rename to project/enums/ETokType.csv diff --git a/project/filesystem/gen.builder.cpp b/project/file_processors/builder.cpp similarity index 100% rename from project/filesystem/gen.builder.cpp rename to project/file_processors/builder.cpp diff --git a/project/filesystem/gen.builder.hpp b/project/file_processors/builder.hpp similarity index 100% rename from project/filesystem/gen.builder.hpp rename to project/file_processors/builder.hpp diff --git a/project/filesystem/gen.editor.hpp b/project/file_processors/editor.hpp similarity index 100% rename from project/filesystem/gen.editor.hpp rename to project/file_processors/editor.hpp diff --git a/project/filesystem/gen.scanner.hpp b/project/file_processors/scanner.hpp similarity index 100% rename from project/filesystem/gen.scanner.hpp rename to project/file_processors/scanner.hpp diff --git a/project/gen.bootstrap.cpp b/project/gen.bootstrap.cpp index fac2c4c..0a706ed 100644 --- a/project/gen.bootstrap.cpp +++ b/project/gen.bootstrap.cpp @@ -2,8 +2,8 @@ #define GEN_ENFORCE_STRONG_CODE_TYPES #define GEN_EXPOSE_BACKEND #include "gen.cpp" -#include "filesystem/gen.scanner.hpp" -#include "helpers/gen.helper.hpp" +#include "filesystem/scanner.hpp" +#include "helpers/helper.hpp" using namespace gen; @@ -33,25 +33,27 @@ int gen_main() { gen::init(); - Code push_ignores = scan_file( "helpers/gen.push_ignores.inline.hpp" ); - Code pop_ignores = scan_file( "helpers/gen.pop_ignores.inline.hpp" ); + Code push_ignores = scan_file( "helpers/push_ignores.inline.hpp" ); + Code pop_ignores = scan_file( "helpers/pop_ignores.inline.hpp" ); // gen_dep.hpp { - Code header_start = scan_file( "dependencies/gen.header_start.hpp" ); + Code header_start = scan_file( "dependencies/header_start.hpp" ); Code nspace_macro = untyped_str( namespace_by_default ? nspace_default : nspace_non_default ); - Code macros = scan_file( "dependencies/gen.macros.hpp" ); - Code basic_types = scan_file( "dependencies/gen.basic_types.hpp" ); - Code debug = scan_file( "dependencies/gen.debug.hpp" ); - Code memory = scan_file( "dependencies/gen.memory.hpp" ); - Code string_ops = scan_file( "dependencies/gen.string_ops.hpp" ); - Code printing = scan_file( "dependencies/gen.printing.hpp" ); - Code containers = scan_file( "dependencies/gen.containers.hpp" ); - Code hashing = scan_file( "dependencies/gen.hashing.hpp" ); - Code string = scan_file( "dependencies/gen.string.hpp" ); - Code file_handling = scan_file( "dependencies/gen.file_handling.hpp" ); - Code parsing = scan_file( "dependencies/gen.parsing.hpp" ); - Code timing = scan_file( "dependencies/gen.timing.hpp" ); + Code macros = scan_file( "dependencies/macros.hpp" ); + Code basic_types = scan_file( "dependencies/basic_types.hpp" ); + Code debug = scan_file( "dependencies/debug.hpp" ); + Code memory = scan_file( "dependencies/memory.hpp" ); + Code string_ops = scan_file( "dependencies/string_ops.hpp" ); + Code printing = scan_file( "dependencies/printing.hpp" ); + Code containers = scan_file( "dependencies/containers.hpp" ); + Code hashing = scan_file( "dependencies/hashing.hpp" ); + Code string = scan_file( "dependencies/string.hpp" ); + Code parsing = scan_file( "dependencies/parsing.hpp" ); + Code timing = scan_file( "dependencies/timing.hpp" ); + + // TOOD : Make this optional + Code file_handling = scan_file( "dependencies/file_handling.hpp" ); Builder deps_header; @@ -82,15 +84,15 @@ int gen_main() // gen_dep.cpp { CodeInclude header = def_include( txt_StrC("gen_dep.hpp") ); - Code impl_start = scan_file( "dependencies/gen.impl_start.cpp" ); - Code debug = scan_file( "dependencies/gen.debug.cpp" ); - Code string_ops = scan_file( "dependencies/gen.string_ops.cpp" ); - Code printing = scan_file( "dependencies/gen.printing.cpp" ); - Code memory = scan_file( "dependencies/gen.memory.cpp" ); - Code parsing = scan_file( "dependencies/gen.parsing.cpp" ); - Code hashing = scan_file( "dependencies/gen.hashing.cpp" ); - Code string = scan_file( "dependencies/gen.string.cpp" ); - Code timing = scan_file( "dependencies/gen.timing.cpp" ); + Code impl_start = scan_file( "dependencies/impl_start.cpp" ); + Code debug = scan_file( "dependencies/debug.cpp" ); + Code string_ops = scan_file( "dependencies/string_ops.cpp" ); + Code printing = scan_file( "dependencies/printing.cpp" ); + Code memory = scan_file( "dependencies/memory.cpp" ); + Code parsing = scan_file( "dependencies/parsing.cpp" ); + Code hashing = scan_file( "dependencies/hashing.cpp" ); + Code string = scan_file( "dependencies/string.cpp" ); + Code timing = scan_file( "dependencies/timing.cpp" ); Builder deps_impl; @@ -115,18 +117,19 @@ int gen_main() // gen.hpp { - Code header_start = scan_file( "components/gen.header_start.hpp" ); + Code header_start = scan_file( "components/header_start.hpp" ); Code nspace_macro = untyped_str( namespace_by_default ? nspace_default : nspace_non_default ); - Code types = scan_file( "components/gen.types.hpp" ); - Code data_structs = scan_file( "components/gen.data_structures.hpp" ); - Code interface = scan_file( "components/gen.interface.hpp" ); - Code header_end = scan_file( "components/gen.header_end.hpp" ); + Code types = scan_file( "components/types.hpp" ); + Code data_structs = scan_file( "components/data_structures.hpp" ); + Code interface = scan_file( "components/interface.hpp" ); + Code header_end = scan_file( "components/header_end.hpp" ); - CodeBody ecode = gen_ecode( "./components/ECode.csv" ); - CodeBody eoperator = gen_eoperator( "./components/EOperator.csv" ); - CodeBody especifier = gen_especifier( "./components/ESpecifier.csv" ); + CodeBody ecode = gen_ecode( "enums/ECode.csv" ); + CodeBody eoperator = gen_eoperator( "enums/EOperator.csv" ); + CodeBody especifier = gen_especifier( "enums/ESpecifier.csv" ); - Code builder = scan_file( "filesystem/gen.builder.hpp" ); + // TODO : Make this optional to include + Code builder = scan_file( "file_proecessors/builder.hpp" ); Builder header; @@ -159,18 +162,19 @@ int gen_main() { Code impl_start = scan_file( "components/gen.impl_start.cpp" ); CodeInclude header = def_include( txt_StrC("gen.hpp") ); - Code data = scan_file( "components/gen.data.cpp" ); - Code ast_case_macros = scan_file( "components/gen.ast_case_macros.cpp" ); - Code ast = scan_file( "components/gen.ast.cpp" ); - Code interface = scan_file( "components/gen.interface.cpp" ); - Code upfront = scan_file( "components/gen.interface.upfront.cpp" ); - Code parsing = scan_file( "components/gen.interface.parsing.cpp" ); - Code untyped = scan_file( "components/gen.untyped.cpp" ); + Code data = scan_file( "components/static_data.cpp" ); + Code ast_case_macros = scan_file( "components/ast_case_macros.cpp" ); + Code ast = scan_file( "components/ast.cpp" ); + Code interface = scan_file( "components/interface.cpp" ); + Code upfront = scan_file( "components/interface.upfront.cpp" ); + Code parsing = scan_file( "components/interface.parsing.cpp" ); + Code untyped = scan_file( "components/untyped.cpp" ); - CodeBody etoktype = gen_etoktype( "components/ETokType.csv", "components/AttributeTokens.csv" ); + CodeBody etoktype = gen_etoktype( "enums/ETokType.csv", "enums/AttributeTokens.csv" ); CodeNamespace parser_nspace = def_namespace( name(Parser), def_namespace_body( args(etoktype)) ); - Code builder = scan_file( "filesystem/gen.builder.cpp" ); + // TODO : Make this optional to include + Code builder = scan_file( "file_proecessors/builder.cpp" ); Builder impl; diff --git a/project/gen.cpp b/project/gen.cpp index a90f524..27668b5 100644 --- a/project/gen.cpp +++ b/project/gen.cpp @@ -1,4 +1,4 @@ -#include "helpers/gen.push_ignores.inline.hpp" +#include "helpers/push_ignores.inline.hpp" // ReSharper disable CppClangTidyClangDiagnosticSwitchEnum @@ -16,19 +16,19 @@ GEN_NS_BEGIN -#include "components/gen.data.cpp" +#include "components/static_data.cpp" -#include "components/gen.ast_case_macros.cpp" -#include "components/gen.ast.cpp" +#include "components/ast_case_macros.cpp" +#include "components/ast.cpp" -#include "components/gen.interface.cpp" -#include "components/gen.interface.upfront.cpp" -#include "components/gen.etoktype.cpp" -#include "components/gen.interface.parsing.cpp" -#include "components/gen.untyped.cpp" +#include "components/interface.cpp" +#include "components/interface.upfront.cpp" +#include "components/etoktype.cpp" +#include "components/interface.parsing.cpp" +#include "components/untyped.cpp" -#include "filesystem/gen.builder.cpp" +#include "file_proecessors/builder.cpp" GEN_NS_END -#include "helpers/gen.pop_ignores.inline.hpp" +#include "helpers/pop_ignores.inline.hpp" diff --git a/project/gen.dep.cpp b/project/gen.dep.cpp index a7589b7..e7b4f92 100644 --- a/project/gen.dep.cpp +++ b/project/gen.dep.cpp @@ -1,19 +1,19 @@ // This file is intended to be included within gen.cpp (There is no pragma diagnostic ignores) #include "gen.dep.hpp" -#include "dependencies/gen.impl_start.cpp" +#include "dependencies/impl_start.cpp" GEN_NS_BEGIN -#include "dependencies/gen.debug.cpp" -#include "dependencies/gen.string_ops.cpp" -#include "dependencies/gen.printing.cpp" -#include "dependencies/gen.memory.cpp" -#include "dependencies/gen.parsing.cpp" -#include "dependencies/gen.hashing.cpp" -#include "dependencies/gen.string.cpp" -#include "dependencies/gen.timing.cpp" +#include "dependencies/debug.cpp" +#include "dependencies/string_ops.cpp" +#include "dependencies/printing.cpp" +#include "dependencies/memory.cpp" +#include "dependencies/parsing.cpp" +#include "dependencies/hashing.cpp" +#include "dependencies/string.cpp" +#include "dependencies/timing.cpp" -#include "dependencies/gen.file_handling.cpp" +#include "dependencies/file_handling.cpp" GEN_NS_END diff --git a/project/gen.dep.hpp b/project/gen.dep.hpp index b6d6303..f315397 100644 --- a/project/gen.dep.hpp +++ b/project/gen.dep.hpp @@ -1,7 +1,7 @@ // This file is intended to be included within gen.hpp (There is no pragma diagnostic ignores) #pragma once -#include "dependencies/gen.header_start.hpp" +#include "dependencies/header_start.hpp" #ifdef GEN_DONT_USE_NAMESPACE # define GEN_NS_BEGIN @@ -13,16 +13,17 @@ GEN_NS_BEGIN -#include "dependencies/gen.macros.hpp" -#include "dependencies/gen.basic_types.hpp" -#include "dependencies/gen.debug.hpp" -#include "dependencies/gen.memory.hpp" -#include "dependencies/gen.string_ops.hpp" -#include "dependencies/gen.printing.hpp" -#include "dependencies/gen.containers.hpp" -#include "dependencies/gen.string.hpp" -#include "dependencies/gen.file_handling.hpp" -#include "dependencies/gen.parsing.hpp" -#include "dependencies/gen.timing.hpp" +#include "dependencies/macros.hpp" +#include "dependencies/basic_types.hpp" +#include "dependencies/debug.hpp" +#include "dependencies/memory.hpp" +#include "dependencies/string_ops.hpp" +#include "dependencies/printing.hpp" +#include "dependencies/containers.hpp" +#include "dependencies/string.hpp" +#include "dependencies/parsing.hpp" +#include "dependencies/timing.hpp" + +#include "dependencies/file_handling.hpp" GEN_NS_END diff --git a/project/gen.hpp b/project/gen.hpp index de4836d..e3abc5b 100644 --- a/project/gen.hpp +++ b/project/gen.hpp @@ -8,8 +8,8 @@ */ #pragma once -#include "helpers/gen.push_ignores.inline.hpp" -#include "components/gen.header_start.hpp" +#include "helpers/push_ignores.inline.hpp" +#include "components/header_start.hpp" #ifdef GEN_DONT_USE_NAMESPACE # define GEN_NS_BEGIN @@ -21,16 +21,16 @@ GEN_NS_BEGIN -#include "components/gen.types.hpp" -#include "components/gen.ecode.hpp" -#include "components/gen.eoperator.hpp" -#include "components/gen.especifier.hpp" -#include "components/gen.data_structures.hpp" -#include "components/gen.interface.hpp" -#include "components/gen.header_end.hpp" +#include "components/types.hpp" +#include "components/ecode.hpp" +#include "components/eoperator.hpp" +#include "components/especifier.hpp" +#include "components/data_structures.hpp" +#include "components/interface.hpp" +#include "components/header_end.hpp" -#include "filesystem/gen.builder.hpp" +#include "file_processors/builder.hpp" GEN_NS_END -#include "helpers/gen.pop_ignores.inline.hpp" +#include "helpers/pop_ignores.inline.hpp" diff --git a/project/helpers/gen.helper.hpp b/project/helpers/helper.hpp similarity index 100% rename from project/helpers/gen.helper.hpp rename to project/helpers/helper.hpp diff --git a/project/helpers/gen.pop_ignores.inline.hpp b/project/helpers/pop_ignores.inline.hpp similarity index 100% rename from project/helpers/gen.pop_ignores.inline.hpp rename to project/helpers/pop_ignores.inline.hpp diff --git a/project/helpers/gen.push_ignores.inline.hpp b/project/helpers/push_ignores.inline.hpp similarity index 100% rename from project/helpers/gen.push_ignores.inline.hpp rename to project/helpers/push_ignores.inline.hpp diff --git a/project/helpers/gen.undef.macros.hpp b/project/helpers/undef.macros.hpp similarity index 100% rename from project/helpers/gen.undef.macros.hpp rename to project/helpers/undef.macros.hpp diff --git a/scripts/Readme.md b/scripts/Readme.md index b4060d6..ec5b2de 100644 --- a/scripts/Readme.md +++ b/scripts/Readme.md @@ -1,12 +1,34 @@ # Scripts -Build and cleanup scripts for the test directory are found here along with `natvis` and `natstepfilter` files for debugging. +Generation, testing, and cleanup scripts for the test directory are found here along with `natvis` and `natstepfilter` files for debugging. -The build works as follows: +## Refactoring -* Compile and run the meta-program, it will dump files to the `test/gen` directory. -* Format the files using clang-format -* Build a program that uses some the generated definitions. (Have not done yet) +`refactor.ps1` Provides a way to run the [refactor](github.com/Ed94/refactor) program. It uses the `gencpp.refactor` script to complete a mass refactor of all content within the files of the specified within the script. -The `test/gen` directory has the meson.build config for the meta-program -The `test` directory has the one for the dependent-program. +Currently `refactor` only supports naive sort of *find and replace* feature set and will not be able to rename identifiers excluisvely to a specific context (such as only renaming member names of a specific struct, etc). + +## Build & Run Scripts + +**`clean.ps1`** +Remove any generated content from the repository. + +**`bootstrap.ps1`** +Generate a version of gencpp where components are inlined directly to `gen.` and `gen. ` +Any heavily preprocessed code is not inlined and are instead generated using the code in the `helpers` directory. + +**`singlheader.build.ps1`** +Generate a single-header version of the library where all code that would normally good in the usual four files (see bootstrap) are inlined into a single `gen.hpp` file. +As with the bootstrap, any heavily preprocessed code is not inlined and instead generated with helper code. + +**`test.gen.build.ps1`** +Build the metaprogram for generating the test code. + +**`test.gen.ps1`** +Build (if not already) the metaprogram for generating test code, then run it to generate code. + +**`test.build.ps1`** +Build and run metaprogram, build test program. + +**`test.run.ps1`** +Build and run metaprogram, build and run test program. diff --git a/scripts/get_sources.ps1 b/scripts/get_sources.ps1 deleted file mode 100644 index e5a2681..0000000 --- a/scripts/get_sources.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -[string[]] $include = 'gen.cpp' #'*.c', '*.cc', '*.cpp' -# [string[]] $exclude = - -$path_root = git rev-parse --show-toplevel -$path_proj = Join-Path $path_root project - -$files = Get-ChildItem -Recurse -Path $path_proj -Include $include -Exclude $exclude - -$sources = $files | Select-Object -ExpandProperty FullName | Resolve-Path -Relative -$sources = $sources.Replace( '\', '/' ) - -return $sources diff --git a/scripts/build.ci.ps1 b/scripts/test.gen_build.ci.ps1 similarity index 100% rename from scripts/build.ci.ps1 rename to scripts/test.gen_build.ci.ps1 diff --git a/scripts/build.ps1 b/scripts/test.gen_build.ps1 similarity index 100% rename from scripts/build.ps1 rename to scripts/test.gen_build.ps1 diff --git a/scripts/gen.ps1 b/scripts/test.gen_run.ps1 similarity index 100% rename from scripts/gen.ps1 rename to scripts/test.gen_run.ps1 From 689646c3934108ef0dfd07cc7340867907843533 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 06:32:16 -0400 Subject: [PATCH 03/10] Finished iniital refactor pass. Comples, but has runtime issues. --- project/components/interface.parsing.cpp | 206 +++++++++++------------ project/dependencies/printing.hpp | 2 +- project/gen.bootstrap.cpp | 2 +- project/gen.cpp | 2 +- 4 files changed, 104 insertions(+), 108 deletions(-) diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 85dea41..291ffd5 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -97,9 +97,9 @@ namespace Parser result.append_fmt("\tContext:\n"); - char* current = Tokens.current().Text; - sptr length = Tokens.current().Length; - while ( current != Tokens.back().Text && current != '\n' ) + char const* current = Tokens.current().Text; + sptr length = Tokens.current().Length; + while ( current != Tokens.Arr.back().Text && *current != '\n' ) { current++; length--; @@ -109,21 +109,19 @@ namespace Parser result.append_fmt("\t(%d, %d): %s", Tokens.current().Line, Tokens.current().Column, line ); line.free(); - StackNode* current = Scope; + StackNode* curr_scope = Scope; do { - if ( current->Name ) + if ( curr_scope->Name ) { - result.append_fmt("\tProcedure: %s, AST Name: %s\n", current->ProcName, (StrC)current->Name ); + result.append_fmt("\tProcedure: %s, AST Name: %s\n", curr_scope->ProcName, (StrC)curr_scope->Name ); } else { - result.append_fmt("\tProcedure: %s\n", current->ProcName ); + result.append_fmt("\tProcedure: %s\n", curr_scope->ProcName ); } - current = current->Prev; - - name.free(); + curr_scope = curr_scope->Prev; } while ( current ); return result; @@ -137,7 +135,7 @@ namespace Parser void pop() { - Context.Scope = Context.Scope->Prev; + Scope = Scope->Prev; } }; @@ -709,12 +707,12 @@ namespace Parser # define check_parse_args( def ) \ if ( def.Len <= 0 ) \ { \ - log_failure( "gen::" stringize( __func__ ) ": length must greater than 0" ); \ + log_failure( "gen::" stringize(__func__) ": length must greater than 0" ); \ return CodeInvalid; \ } \ if ( def.Ptr == nullptr ) \ { \ - log_failure( "gen::" stringize( __func__ ) ": def was null" ); \ + log_failure( "gen::" stringize(__func__) ": def was null" ); \ return CodeInvalid; \ } @@ -871,7 +869,7 @@ CodeAttributes parse_attributes() return def_attributes( attribute_txt ); } - pop_context(); + Context.pop(); return { nullptr }; } @@ -931,7 +929,7 @@ Parser::Token parse_identifier() name.Length = ( (sptr)prevtok.Text + (sptr)prevtok.Length ) - (sptr)name.Text; } - pop_context(); + Context.pop(); return name; } @@ -996,7 +994,7 @@ CodeParam parse_params( bool use_template_capture = false ) eat( currtok.Type ); } - value = parse_type( toks, context ); + value = parse_type(); } } @@ -1031,7 +1029,7 @@ CodeParam parse_params( bool use_template_capture = false ) continue; } - type = parse_type( toks, context ); + type = parse_type(); if ( type == Code::Invalid ) return CodeInvalid; @@ -1060,7 +1058,7 @@ CodeParam parse_params( bool use_template_capture = false ) eat( currtok.Type ); } - value = parse_type( toks, context ); + value = parse_type(); } } @@ -1110,7 +1108,7 @@ CodeFn parse_function_after_name( using namespace Parser; push_scope(); - CodeParam params = parse_params( toks, stringize(parse_function) ); + CodeParam params = parse_params(); while ( left && currtok.is_specifier() ) { @@ -1121,7 +1119,7 @@ CodeFn parse_function_after_name( CodeBody body = { nullptr }; if ( check( TokType::BraceCurly_Open ) ) { - body = parse_function_body( toks, stringize(parse_function) ); + body = parse_function_body(); if ( body == Code::Invalid ) return CodeInvalid; } @@ -1382,7 +1380,7 @@ CodeOperator parse_operator_after_ret_type( eat( TokType::Operator ); // Parse Params - CodeParam params = parse_params( toks, stringize(parse_operator) ); + CodeParam params = parse_params(); while ( left && currtok.is_specifier() ) { @@ -1394,7 +1392,7 @@ CodeOperator parse_operator_after_ret_type( CodeBody body = { nullptr }; if ( check( TokType::BraceCurly_Open ) ) { - body = parse_function_body( toks, stringize(parse_function) ); + body = parse_function_body(); if ( body == Code::Invalid ) return CodeInvalid; } @@ -1515,7 +1513,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes Code result = Code::Invalid; - CodeType type = parse_type( toks, stringize(parse_variable) ); + CodeType type = parse_type(); if ( type == Code::Invalid ) return CodeInvalid; @@ -1605,39 +1603,39 @@ CodeBody parse_class_struct_body( Parser::TokType which ) break; case TokType::Decl_Class: - member = parse_class( toks, context ); + member = parse_class(); break; case TokType::Decl_Enum: - member = parse_enum( toks, context ); + member = parse_enum(); break; case TokType::Decl_Friend: - member = parse_friend( toks, context ); + member = parse_friend(); break; case TokType::Decl_Operator: - member = parse_operator_cast( toks, context ); + member = parse_operator_cast(); break; case TokType::Decl_Struct: - member = parse_struct( toks, context ); + member = parse_struct(); break; case TokType::Decl_Template: - member = parse_template( toks, context ); + member = parse_template(); break; case TokType::Decl_Typedef: - member = parse_typedef( toks, context ); + member = parse_typedef(); break; case TokType::Decl_Union: - member = parse_variable( toks, context ); + member = parse_variable(); break; case TokType::Decl_Using: - member = parse_using( toks, context ); + member = parse_using(); break; case TokType::Attribute_Open: @@ -1769,7 +1767,7 @@ Code parse_class_struct( Parser::TokType which ) attributes = parse_attributes(); if ( check( TokType::Identifier ) ) - name = parse_identifier( toks, context ); + name = parse_identifier(); local_persist char interface_arr_mem[ kilobytes(4) ] {0}; @@ -1784,7 +1782,7 @@ Code parse_class_struct( Parser::TokType which ) access = currtok.to_access_specifier(); } - Token parent_tok = parse_identifier( toks, context ); + Token parent_tok = parse_identifier(); parent = def_type( parent_tok ); while ( check(TokType::Comma) ) @@ -1796,7 +1794,7 @@ Code parse_class_struct( Parser::TokType which ) eat(currtok.Type); } - Token interface_tok = parse_identifier( toks, context ); + Token interface_tok = parse_identifier(); interfaces.append( def_type( interface_tok ) ); } @@ -1804,22 +1802,16 @@ Code parse_class_struct( Parser::TokType which ) if ( check( TokType::BraceCurly_Open ) ) { - body = parse_class_struct_body( which, toks, context ); + body = parse_class_struct_body( which ); } eat( TokType::Statement_End ); if ( which == TokType::Decl_Class ) - result = def_class( name, body, parent, access - , attributes - , mflags - ); + result = def_class( name, body, parent, access, attributes, mflags ); else - result = def_struct( name, body, (CodeType)parent, access - , attributes - , mflags - ); + result = def_struct( name, body, (CodeType)parent, access, attributes, mflags ); interfaces.free(); Context.pop(); @@ -1901,49 +1893,49 @@ CodeBody parse_global_nspace( CodeT which ) break; case TokType::Decl_Enum: - member = parse_enum( toks, context); + member = parse_enum(); break; case TokType::Decl_Class: - member = parse_class( toks, context ); + member = parse_class(); break; case TokType::Decl_Extern_Linkage: if ( which == Extern_Linkage_Body ) log_failure( "Nested extern linkage\n%s", Context.to_string() ); - member = parse_extern_link_body( toks, context ); + member = parse_extern_link_body(); break; case TokType::Decl_Namespace: - member = parse_namespace( toks, context ); + member = parse_namespace(); break; case TokType::Decl_Struct: - member = parse_struct( toks, context ); + member = parse_struct(); break; case TokType::Decl_Template: - member = parse_template( toks, context ); + member = parse_template(); break; case TokType::Decl_Typedef: - member = parse_typedef( toks, context ); + member = parse_typedef(); break; case TokType::Decl_Union: - member = parse_union( toks, context ); + member = parse_union(); break; case TokType::Decl_Using: - member = parse_using( toks, context ); + member = parse_using(); break; case TokType::Module_Export: if ( which == Export_Body ) log_failure( "Nested export declaration\n%s", Context.to_string() ); - member = parse_export_body( toks, context ); + member = parse_export_body(); break; case TokType::Module_Import: @@ -2016,7 +2008,7 @@ CodeBody parse_global_nspace( CodeT which ) case TokType::Type_double: case TokType::Type_int: { - member = parse_operator_function_or_variable( expects_function, attributes, specifiers, toks, context ); + member = parse_operator_function_or_variable( expects_function, attributes, specifiers ); } } @@ -2039,15 +2031,16 @@ CodeBody parse_global_nspace( CodeT which ) internal CodeClass parse_class() { + using namespace Parser; push_scope(); - CodeClass result = (CodeClass) parse_class_struct( Parser::TokType::Decl_Class, toks, context ); + CodeClass result = (CodeClass) parse_class_struct( Parser::TokType::Decl_Class ); Context.pop(); return result; } CodeClass parse_class( StrC def ) { - check_parse_args( parse_class, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2103,7 +2096,7 @@ CodeEnum parse_enum() { eat( TokType::Assign_Classifer ); - type = parse_type( toks, stringize(parse_enum) ); + type = parse_type(); if ( type == Code::Invalid ) return CodeInvalid; } @@ -2171,7 +2164,7 @@ CodeEnum parse_enum() CodeEnum parse_enum( StrC def ) { - check_parse_args( parse_enum, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2184,6 +2177,7 @@ CodeEnum parse_enum( StrC def ) internal inline CodeBody parse_export_body() { + using namespace Parser; push_scope(); CodeBody result = parse_global_nspace( ECode::Export_Body ); Context.pop(); @@ -2192,7 +2186,7 @@ CodeBody parse_export_body() CodeBody parse_export_body( StrC def ) { - check_parse_args( parse_export_body, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2205,6 +2199,7 @@ CodeBody parse_export_body( StrC def ) internal inline CodeBody parse_extern_link_body() { + using namespace Parser; push_scope(); CodeBody result = parse_global_nspace( ECode::Extern_Linkage_Body ); Context.pop(); @@ -2230,7 +2225,7 @@ CodeExtern parse_extern_link() result->Type = ECode::Extern_Linkage; result->Name = get_cached_string( name ); - Code entry = parse_extern_link_body( toks, context ); + Code entry = parse_extern_link_body(); if ( entry == Code::Invalid ) { log_failure( "Failed to parse body\n%s", Context.to_string() ); @@ -2245,7 +2240,7 @@ CodeExtern parse_extern_link() CodeExtern parse_extern_link( StrC def ) { - check_parse_args( parse_extern_link, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2267,7 +2262,7 @@ CodeFriend parse_friend() CodeFn function = { nullptr }; // Type declaration or return type - CodeType type = parse_type( toks, stringize(parse_friend) ); + CodeType type = parse_type(); if ( type == Code::Invalid ) return CodeInvalid; @@ -2275,10 +2270,10 @@ CodeFriend parse_friend() if ( currtok.Type == TokType::Identifier ) { // Name - Token name = parse_identifier( toks, stringize(parse_friend) ); + Token name = parse_identifier(); // Parameter list - CodeParam params = parse_params( toks, stringize(parse_friend) ); + CodeParam params = parse_params(); function = make_code(); function->Type = Function_Fwd; @@ -2307,7 +2302,7 @@ CodeFriend parse_friend() CodeFriend parse_friend( StrC def ) { - check_parse_args( parse_friend, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2370,11 +2365,11 @@ CodeFn parse_functon() specifiers = def_specifiers( NumSpecifiers, specs_found ); } - CodeType ret_type = parse_type( toks, stringize(parse_function) ); + CodeType ret_type = parse_type(); if ( ret_type == Code::Invalid ) return CodeInvalid; - Token name = parse_identifier( toks, stringize(parse_function) ); + Token name = parse_identifier(); if ( ! name ) return CodeInvalid; @@ -2386,7 +2381,7 @@ CodeFn parse_functon() CodeFn parse_function( StrC def ) { - check_parse_args( parse_function, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2398,7 +2393,7 @@ CodeFn parse_function( StrC def ) CodeBody parse_global_body( StrC def ) { - check_parse_args( parse_global_body, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2412,16 +2407,16 @@ CodeBody parse_global_body( StrC def ) } internal -CodeNamespace parse_namespace( Parser::TokArray& toks, char const* context ) +CodeNamespace parse_namespace() { using namespace Parser; push_scope(); eat( TokType::Decl_Namespace ); - Token name = parse_identifier( toks, stringize(parse_namespace) ); + Token name = parse_identifier(); - CodeBody body = parse_global_nspace( ECode::Namespace_Body, toks, stringize(parse_namespace) ); + CodeBody body = parse_global_nspace( ECode::Namespace_Body ); if ( body == Code::Invalid ) return CodeInvalid; @@ -2438,7 +2433,7 @@ CodeNamespace parse_namespace( Parser::TokArray& toks, char const* context ) CodeNamespace parse_namespace( StrC def ) { - check_parse_args( parse_namespace, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2500,7 +2495,7 @@ CodeOperator parse_operator() } // Parse Return Type - CodeType ret_type = parse_type( toks, stringize(parse_operator) ); + CodeType ret_type = parse_type(); CodeOperator result = parse_operator_after_ret_type( mflags, attributes, specifiers, ret_type ); @@ -2510,7 +2505,7 @@ CodeOperator parse_operator() CodeOperator parse_operator( StrC def ) { - check_parse_args( parse_operator, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2527,7 +2522,7 @@ CodeOpCast parse_operator_cast() eat( TokType::Decl_Operator ); - Code type = parse_type( toks, stringize(parse_operator_cast) ); + Code type = parse_type(); eat( TokType::Capture_Start ); eat( TokType::Capture_End ); @@ -2587,7 +2582,7 @@ CodeOpCast parse_operator_cast() CodeOpCast parse_operator_cast( StrC def ) { - check_parse_args( parse_operator_cast, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2600,15 +2595,16 @@ CodeOpCast parse_operator_cast( StrC def ) internal inline CodeStruct parse_struct() { + using namespace Parser; push_scope(); - CodeStruct result = (CodeStruct) parse_class_struct(); + CodeStruct result = (CodeStruct) parse_class_struct( TokType::Decl_Struct ); Context.pop(); return result; } CodeStruct parse_struct( StrC def ) { - check_parse_args( parse_struct, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2636,7 +2632,7 @@ CodeTemplate parse_template() eat( TokType::Decl_Template ); - Code params = parse_params( toks, stringize(parse_template), UseTemplateCapture ); + Code params = parse_params( UseTemplateCapture ); if ( params == Code::Invalid ) return CodeInvalid; @@ -2646,19 +2642,19 @@ CodeTemplate parse_template() { if ( check( TokType::Decl_Class ) ) { - definition = parse_class( toks, stringize(parse_template) ); + definition = parse_class(); break; } if ( check( TokType::Decl_Struct ) ) { - definition = parse_enum( toks, stringize(parse_template) ); + definition = parse_enum(); break; } if ( check( TokType::Decl_Using )) { - definition = parse_using( toks, stringize(parse_template) ); + definition = parse_using(); break; } @@ -2717,7 +2713,7 @@ CodeTemplate parse_template() specifiers = def_specifiers( NumSpecifiers, specs_found ); } - definition = parse_operator_function_or_variable( expects_function, attributes, specifiers, toks, stringize(parse_template) ); + definition = parse_operator_function_or_variable( expects_function, attributes, specifiers ); break; } @@ -2735,7 +2731,7 @@ CodeTemplate parse_template() CodeTemplate parse_template( StrC def ) { - check_parse_args( parse_template, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2806,7 +2802,7 @@ CodeType parse_type() } else { - name = parse_identifier( toks, context ); + name = parse_identifier(); if ( ! name ) return CodeInvalid; @@ -2924,7 +2920,7 @@ CodeType parse_type() CodeType parse_type( StrC def ) { - check_parse_args( parse_type, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -2955,19 +2951,19 @@ CodeTypedef parse_typedef() eat( TokType::Decl_Typedef ); if ( check( TokType::Decl_Enum ) ) - type = parse_enum( toks, context ); + type = parse_enum(); else if ( check(TokType::Decl_Class ) ) - type = parse_class( toks, context ); + type = parse_class(); else if ( check(TokType::Decl_Struct ) ) - type = parse_struct( toks, context ); + type = parse_struct(); else if ( check(TokType::Decl_Union) ) - type = parse_union( toks, context ); + type = parse_union(); else - type = parse_type( toks, context ); + type = parse_type(); if ( ! check( TokType::Identifier ) ) { @@ -2978,7 +2974,7 @@ CodeTypedef parse_typedef() name = currtok; eat( TokType::Identifier ); - array_expr = parse_array_decl( toks, context ); + array_expr = parse_array_decl(); eat( TokType::Statement_End ); @@ -3001,7 +2997,7 @@ CodeTypedef parse_typedef() CodeTypedef parse_typedef( StrC def ) { - check_parse_args( parse_typedef, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -3046,7 +3042,7 @@ CodeUnion parse_union() while ( ! check( TokType::BraceCurly_Close ) ) { - Code entry = parse_variable( toks, context ); + Code entry = parse_variable(); if ( entry ) body.append( entry ); @@ -3075,7 +3071,7 @@ CodeUnion parse_union() CodeUnion parse_union( StrC def ) { - check_parse_args( parse_union, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -3122,14 +3118,14 @@ CodeUsing parse_using() if ( currtok.IsAssign ) { - attributes = parse_attributes( toks, context ); + attributes = parse_attributes(); eat( TokType::Operator ); - type = parse_type( toks, context ); + type = parse_type(); } - array_expr = parse_array_decl( toks, context ); + array_expr = parse_array_decl(); eat( TokType::Statement_End ); @@ -3164,7 +3160,7 @@ CodeUsing parse_using() CodeUsing parse_using( StrC def ) { - check_parse_args( parse_using, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); @@ -3195,7 +3191,7 @@ CodeVar parse_variable() attributes = parse_attributes(); - while ( left && tok_is_specifier( currtok ) ) + while ( left && currtok.is_specifier() ) { SpecifierT spec = ESpecifier::to_type( currtok ); @@ -3233,12 +3229,12 @@ CodeVar parse_variable() specifiers = def_specifiers( NumSpecifiers, specs_found ); } - CodeType type = parse_type( toks, context ); + CodeType type = parse_type(); if ( type == Code::Invalid ) return CodeInvalid; - Context.Scope->Name = current; + Context.Scope->Name = currtok; eat( TokType::Identifier ); CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, Context.Scope->Name ); @@ -3249,7 +3245,7 @@ CodeVar parse_variable() CodeVar parse_variable( StrC def ) { - check_parse_args( parse_variable, def ); + check_parse_args( def ); using namespace Parser; TokArray toks = lex( def ); diff --git a/project/dependencies/printing.hpp b/project/dependencies/printing.hpp index 567bee9..7fa3262 100644 --- a/project/dependencies/printing.hpp +++ b/project/dependencies/printing.hpp @@ -17,7 +17,7 @@ sw str_fmt_out_err_va( char const* fmt, va_list va ); // TODO : Move these to file handling. sw str_fmt_file ( FileInfo* f, char const* fmt, ... ); -sw str_fmt_file_va ( FileInfo* f, char const* fmt, va_list va ) +sw str_fmt_file_va ( FileInfo* f, char const* fmt, va_list va ); constexpr char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED"; diff --git a/project/gen.bootstrap.cpp b/project/gen.bootstrap.cpp index 0a706ed..5fc2b22 100644 --- a/project/gen.bootstrap.cpp +++ b/project/gen.bootstrap.cpp @@ -2,7 +2,7 @@ #define GEN_ENFORCE_STRONG_CODE_TYPES #define GEN_EXPOSE_BACKEND #include "gen.cpp" -#include "filesystem/scanner.hpp" +#include "file_processors/scanner.hpp" #include "helpers/helper.hpp" using namespace gen; diff --git a/project/gen.cpp b/project/gen.cpp index 27668b5..a71f597 100644 --- a/project/gen.cpp +++ b/project/gen.cpp @@ -27,7 +27,7 @@ GEN_NS_BEGIN #include "components/interface.parsing.cpp" #include "components/untyped.cpp" -#include "file_proecessors/builder.cpp" +#include "file_processors/builder.cpp" GEN_NS_END From 108b16739f91b2bf5a927314298a70c916ea2669 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 12:25:38 -0400 Subject: [PATCH 04/10] bootstrap and singleheader compile and generate. --- project/components/interface.parsing.cpp | 21 ++++- project/gen.bootstrap.cpp | 6 +- scripts/.clang-format | 8 +- ...{gen.header_start.hpp => header_start.hpp} | 0 singleheader/gen.singleheader.cpp | 90 +++++++++---------- 5 files changed, 69 insertions(+), 56 deletions(-) rename singleheader/components/{gen.header_start.hpp => header_start.hpp} (100%) diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 291ffd5..b3b1c7b 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -1738,7 +1738,6 @@ internal Code parse_class_struct( Parser::TokType which ) { using namespace Parser; - push_scope(); if ( which != TokType::Decl_Class && which != TokType::Decl_Struct ) { @@ -1814,7 +1813,6 @@ Code parse_class_struct( Parser::TokType which ) result = def_struct( name, body, (CodeType)parent, access, attributes, mflags ); interfaces.free(); - Context.pop(); return result; } @@ -1865,7 +1863,6 @@ CodeBody parse_global_nspace( CodeT which ) { using namespace Parser; using namespace ECode; - push_scope(); if ( which != Namespace_Body && which != Global_Body && which != Export_Body && which != Extern_Linkage_Body ) return CodeInvalid; @@ -2024,7 +2021,6 @@ CodeBody parse_global_nspace( CodeT which ) if ( which != Global_Body ) eat( TokType::BraceCurly_Close ); - Context.pop(); return result; } @@ -2047,6 +2043,7 @@ CodeClass parse_class( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; push_scope(); CodeClass result = (CodeClass) parse_class_struct( TokType::Decl_Class ); Context.pop(); @@ -2171,6 +2168,7 @@ CodeEnum parse_enum( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_enum(); } @@ -2193,6 +2191,7 @@ CodeBody parse_export_body( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_export_body(); } @@ -2247,6 +2246,7 @@ CodeExtern parse_extern_link( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_extern_link(); } @@ -2309,6 +2309,7 @@ CodeFriend parse_friend( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_friend(); } @@ -2388,6 +2389,7 @@ CodeFn parse_function( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return (CodeFn) parse_functon(); } @@ -2400,6 +2402,7 @@ CodeBody parse_global_body( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; push_scope(); CodeBody result = parse_global_nspace( ECode::Global_Body ); Context.pop(); @@ -2440,6 +2443,7 @@ CodeNamespace parse_namespace( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_namespace(); } @@ -2512,6 +2516,7 @@ CodeOperator parse_operator( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return (CodeOperator) parse_operator(); } @@ -2589,6 +2594,7 @@ CodeOpCast parse_operator_cast( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_operator_cast(); } @@ -2611,6 +2617,7 @@ CodeStruct parse_struct( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return (CodeStruct) parse_class_struct( TokType::Decl_Struct ); } @@ -2738,6 +2745,7 @@ CodeTemplate parse_template( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_template(); } @@ -2927,6 +2935,7 @@ CodeType parse_type( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_type(); } @@ -3004,6 +3013,7 @@ CodeTypedef parse_typedef( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_typedef(); } @@ -3078,6 +3088,7 @@ CodeUnion parse_union( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_union(); } @@ -3167,6 +3178,7 @@ CodeUsing parse_using( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_using(); } @@ -3252,6 +3264,7 @@ CodeVar parse_variable( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; + Context.Tokens = toks; return parse_variable(); } diff --git a/project/gen.bootstrap.cpp b/project/gen.bootstrap.cpp index 5fc2b22..202dc71 100644 --- a/project/gen.bootstrap.cpp +++ b/project/gen.bootstrap.cpp @@ -129,7 +129,7 @@ int gen_main() CodeBody especifier = gen_especifier( "enums/ESpecifier.csv" ); // TODO : Make this optional to include - Code builder = scan_file( "file_proecessors/builder.hpp" ); + Code builder = scan_file( "file_processors/builder.hpp" ); Builder header; @@ -160,7 +160,7 @@ int gen_main() // gen.cpp { - Code impl_start = scan_file( "components/gen.impl_start.cpp" ); + Code impl_start = scan_file( "components/impl_start.cpp" ); CodeInclude header = def_include( txt_StrC("gen.hpp") ); Code data = scan_file( "components/static_data.cpp" ); Code ast_case_macros = scan_file( "components/ast_case_macros.cpp" ); @@ -174,7 +174,7 @@ int gen_main() CodeNamespace parser_nspace = def_namespace( name(Parser), def_namespace_body( args(etoktype)) ); // TODO : Make this optional to include - Code builder = scan_file( "file_proecessors/builder.cpp" ); + Code builder = scan_file( "file_processors/builder.cpp" ); Builder impl; diff --git a/scripts/.clang-format b/scripts/.clang-format index b092188..26f9cc8 100644 --- a/scripts/.clang-format +++ b/scripts/.clang-format @@ -3,7 +3,7 @@ AccessModifierOffset: -4 AlignAfterOpenBracket: BlockIndent -AlignArrayOfStructures: Right +AlignArrayOfStructures: Left AlignConsecutiveAssignments: Enabled: true AcrossEmptyLines: false @@ -61,7 +61,7 @@ BraceWrapping: BeforeLambdaBody: false BeforeWhile: false -# BreakAfterAttributes: Always +BreakAfterAttributes: Always # BreakArrays: false # BreakBeforeInlineASMColon: OnlyMultiline BreakBeforeBinaryOperators: NonAssignment @@ -96,7 +96,7 @@ IncludeBlocks: Preserve IndentCaseBlocks: false IndentCaseLabels: false IndentExternBlock: AfterExternBlock -IndentGotoLabels: false +IndentGotoLabels: true IndentPPDirectives: AfterHash IndentRequires: true IndentWidth: 4 @@ -127,7 +127,7 @@ SeparateDefinitionBlocks: Always ShortNamespaceLines: 40 -SortIncludes: true +SortIncludes: false SortUsingDeclarations: true SpaceAfterCStyleCast: false diff --git a/singleheader/components/gen.header_start.hpp b/singleheader/components/header_start.hpp similarity index 100% rename from singleheader/components/gen.header_start.hpp rename to singleheader/components/header_start.hpp diff --git a/singleheader/gen.singleheader.cpp b/singleheader/gen.singleheader.cpp index dcebe1b..aff3cef 100644 --- a/singleheader/gen.singleheader.cpp +++ b/singleheader/gen.singleheader.cpp @@ -2,8 +2,8 @@ #define GEN_ENFORCE_STRONG_CODE_TYPES #define GEN_EXPOSE_BACKEND #include "gen.cpp" -#include "filesystem/gen.scanner.hpp" -#include "helpers/gen.helper.hpp" +#include "file_processors/scanner.hpp" +#include "helpers/helper.hpp" using namespace gen; @@ -59,10 +59,10 @@ int gen_main() #define project_dir "../project/" - Code push_ignores = scan_file( project_dir "helpers/gen.push_ignores.inline.hpp" ); - Code pop_ignores = scan_file( project_dir "helpers/gen.pop_ignores.inline.hpp" ); + Code push_ignores = scan_file( project_dir "helpers/push_ignores.inline.hpp" ); + Code pop_ignores = scan_file( project_dir "helpers/pop_ignores.inline.hpp" ); - Code header_start = scan_file( "components/gen.header_start.hpp" ); + Code header_start = scan_file( "components/header_start.hpp" ); Code nspace_macro = untyped_str( namespace_by_default ? nspace_default : nspace_non_default ); Builder @@ -80,19 +80,19 @@ int gen_main() { header.print_fmt( roll_own_dependencies_guard_start ); - Code header_start = scan_file( project_dir "dependencies/gen.header_start.hpp" ); - Code macros = scan_file( project_dir "dependencies/gen.macros.hpp" ); - Code basic_types = scan_file( project_dir "dependencies/gen.basic_types.hpp" ); - Code debug = scan_file( project_dir "dependencies/gen.debug.hpp" ); - Code memory = scan_file( project_dir "dependencies/gen.memory.hpp" ); - Code string_ops = scan_file( project_dir "dependencies/gen.string_ops.hpp" ); - Code printing = scan_file( project_dir "dependencies/gen.printing.hpp" ); - Code containers = scan_file( project_dir "dependencies/gen.containers.hpp" ); - Code hashing = scan_file( project_dir "dependencies/gen.hashing.hpp" ); - Code string = scan_file( project_dir "dependencies/gen.string.hpp" ); - Code file_handling = scan_file( project_dir "dependencies/gen.file_handling.hpp" ); - Code parsing = scan_file( project_dir "dependencies/gen.parsing.hpp" ); - Code timing = scan_file( project_dir "dependencies/gen.timing.hpp" ); + Code header_start = scan_file( project_dir "dependencies/header_start.hpp" ); + Code macros = scan_file( project_dir "dependencies/macros.hpp" ); + Code basic_types = scan_file( project_dir "dependencies/basic_types.hpp" ); + Code debug = scan_file( project_dir "dependencies/debug.hpp" ); + Code memory = scan_file( project_dir "dependencies/memory.hpp" ); + Code string_ops = scan_file( project_dir "dependencies/string_ops.hpp" ); + Code printing = scan_file( project_dir "dependencies/printing.hpp" ); + Code containers = scan_file( project_dir "dependencies/containers.hpp" ); + Code hashing = scan_file( project_dir "dependencies/hashing.hpp" ); + Code string = scan_file( project_dir "dependencies/string.hpp" ); + Code file_handling = scan_file( project_dir "dependencies/file_handling.hpp" ); + Code parsing = scan_file( project_dir "dependencies/parsing.hpp" ); + Code timing = scan_file( project_dir "dependencies/timing.hpp" ); header.print( header_start ); header.print_fmt( "GEN_NS_BEGIN\n\n" ); @@ -113,16 +113,16 @@ int gen_main() header.print_fmt( roll_own_dependencies_guard_end ); } - Code types = scan_file( project_dir "components/gen.types.hpp" ); - Code data_structs = scan_file( project_dir "components/gen.data_structures.hpp" ); - Code interface = scan_file( project_dir "components/gen.interface.hpp" ); - Code header_end = scan_file( project_dir "components/gen.header_end.hpp" ); + Code types = scan_file( project_dir "components/types.hpp" ); + Code data_structs = scan_file( project_dir "components/data_structures.hpp" ); + Code interface = scan_file( project_dir "components/interface.hpp" ); + Code header_end = scan_file( project_dir "components/header_end.hpp" ); - CodeBody ecode = gen_ecode( project_dir "components/ECode.csv" ); - CodeBody eoperator = gen_eoperator( project_dir "components/EOperator.csv" ); - CodeBody especifier = gen_especifier( project_dir "components/ESpecifier.csv" ); + CodeBody ecode = gen_ecode( project_dir "enums/ECode.csv" ); + CodeBody eoperator = gen_eoperator( project_dir "enums/EOperator.csv" ); + CodeBody especifier = gen_especifier( project_dir "enums/ESpecifier.csv" ); - Code builder = scan_file( project_dir "filesystem/gen.builder.hpp" ); + Code builder = scan_file( project_dir "file_processors/builder.hpp" ); header.print_fmt( "GEN_NS_BEGIN\n\n" ); @@ -146,17 +146,17 @@ int gen_main() { header.print_fmt( roll_own_dependencies_guard_start ); - Code impl_start = scan_file( project_dir "dependencies/gen.impl_start.cpp" ); - Code debug = scan_file( project_dir "dependencies/gen.debug.cpp" ); - Code string_ops = scan_file( project_dir "dependencies/gen.string_ops.cpp" ); - Code printing = scan_file( project_dir "dependencies/gen.printing.cpp" ); - Code memory = scan_file( project_dir "dependencies/gen.memory.cpp" ); - Code parsing = scan_file( project_dir "dependencies/gen.parsing.cpp" ); - Code hashing = scan_file( project_dir "dependencies/gen.hashing.cpp" ); - Code string = scan_file( project_dir "dependencies/gen.string.cpp" ); - Code timing = scan_file( project_dir "dependencies/gen.timing.cpp" ); + Code impl_start = scan_file( project_dir "dependencies/impl_start.cpp" ); + Code debug = scan_file( project_dir "dependencies/debug.cpp" ); + Code string_ops = scan_file( project_dir "dependencies/string_ops.cpp" ); + Code printing = scan_file( project_dir "dependencies/printing.cpp" ); + Code memory = scan_file( project_dir "dependencies/memory.cpp" ); + Code parsing = scan_file( project_dir "dependencies/parsing.cpp" ); + Code hashing = scan_file( project_dir "dependencies/hashing.cpp" ); + Code string = scan_file( project_dir "dependencies/string.cpp" ); + Code timing = scan_file( project_dir "dependencies/timing.cpp" ); - Code file_handling = scan_file( project_dir "dependencies/gen.file_handling.cpp" ); + Code file_handling = scan_file( project_dir "dependencies/file_handling.cpp" ); header.print_fmt( "GEN_NS_BEGIN\n\n"); header.print( impl_start ); @@ -176,18 +176,18 @@ int gen_main() header.print_fmt( roll_own_dependencies_guard_end ); } - Code data = scan_file( project_dir "components/gen.data.cpp" ); - Code ast_case_macros = scan_file( project_dir "components/gen.ast_case_macros.cpp" ); - Code ast = scan_file( project_dir "components/gen.ast.cpp" ); - Code interface = scan_file( project_dir "components/gen.interface.cpp" ); - Code upfront = scan_file( project_dir "components/gen.interface.upfront.cpp" ); - Code parsing = scan_file( project_dir "components/gen.interface.parsing.cpp" ); - Code untyped = scan_file( project_dir "components/gen.untyped.cpp" ); + Code data = scan_file( project_dir "components/static_data.cpp" ); + Code ast_case_macros = scan_file( project_dir "components/ast_case_macros.cpp" ); + Code ast = scan_file( project_dir "components/ast.cpp" ); + Code interface = scan_file( project_dir "components/interface.cpp" ); + Code upfront = scan_file( project_dir "components/interface.upfront.cpp" ); + Code parsing = scan_file( project_dir "components/interface.parsing.cpp" ); + Code untyped = scan_file( project_dir "components/untyped.cpp" ); - CodeBody etoktype = gen_etoktype( project_dir "components/ETokType.csv", project_dir "components/AttributeTokens.csv" ); + CodeBody etoktype = gen_etoktype( project_dir "enums/ETokType.csv", project_dir "enums/AttributeTokens.csv" ); CodeNamespace parser_nspace = def_namespace( name(Parser), def_namespace_body( args(etoktype)) ); - Code builder = scan_file( project_dir "filesystem/gen.builder.cpp" ); + Code builder = scan_file( project_dir "file_processors/builder.cpp" ); header.print_fmt( "GEN_NS_BEGIN\n\n"); header.print( data ); From f09bb99fdf7370d97cd5cb750eb1080a0fda771e Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 13:15:53 -0400 Subject: [PATCH 05/10] Fixes for test generation (sanity, soa). --- project/components/interface.parsing.cpp | 49 +++++++++++++++++++----- scripts/test.gen_build.ps1 | 2 +- test/test.cpp | 2 +- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index b3b1c7b..915a288 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -97,16 +97,18 @@ namespace Parser result.append_fmt("\tContext:\n"); - char const* current = Tokens.current().Text; - sptr length = Tokens.current().Length; - while ( current != Tokens.Arr.back().Text && *current != '\n' ) + Token last_valid = Tokens.Idx >= Tokens.Arr.num() ? Tokens.Arr[Tokens.Arr.num() -1] : Tokens.current(); + + char const* current = last_valid.Text; + sptr length = last_valid.Length; + while ( current <= Tokens.Arr.back().Text && *current != '\n' ) { current++; - length--; + length++; } - String line = String::make( GlobalAllocator, { length, Tokens.current().Text } ); - result.append_fmt("\t(%d, %d): %s", Tokens.current().Line, Tokens.current().Column, line ); + String line = String::make( GlobalAllocator, { length, last_valid.Text } ); + result.append_fmt("\t(%d, %d): %s\n", last_valid.Line, last_valid.Column, line ); line.free(); StackNode* curr_scope = Scope; @@ -114,16 +116,16 @@ namespace Parser { if ( curr_scope->Name ) { - result.append_fmt("\tProcedure: %s, AST Name: %s\n", curr_scope->ProcName, (StrC)curr_scope->Name ); + result.append_fmt("\tProcedure: %s, AST Name: %s\n", curr_scope->ProcName.Ptr, (StrC)curr_scope->Name ); } else { - result.append_fmt("\tProcedure: %s\n", curr_scope->ProcName ); + result.append_fmt("\tProcedure: %s\n", curr_scope->ProcName.Ptr ); } curr_scope = curr_scope->Prev; } - while ( current ); + while ( curr_scope ); return result; } @@ -901,10 +903,39 @@ Parser::Token parse_identifier() name.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)name.Text; eat( TokType::Identifier ); + + if ( check( TokType::Operator ) && currtok.Text[0] == '<' ) + { + eat( TokType::Operator ); + + // Template arguments can be complex so were not validating if they are correct. + s32 level = 0; + while ( left && (currtok.Text[0] != '>' || level > 0 ) ) + { + if ( currtok.Text[0] == '<' ) + level++; + + else if ( currtok.Text[0] == '>' && level > 0 ) + level--; + + eat( currtok.Type ); + } + + if ( left == 0 ) + { + log_failure( "Error, unexpected end of template arguments\n%s", Context.to_string() ); + return { nullptr, 0, TokType::Invalid }; + } + + eat( TokType::Operator ); + name.Length = ( (sptr)prevtok.Text + (sptr)prevtok.Length ) - (sptr)name.Text; + } } if ( check( TokType::Operator ) && currtok.Text[0] == '<' ) { + eat( TokType::Operator ); + // Template arguments can be complex so were not validating if they are correct. s32 level = 0; while ( left && (currtok.Text[0] != '>' || level > 0 ) ) diff --git a/scripts/test.gen_build.ps1 b/scripts/test.gen_build.ps1 index 4bbfb15..39814df 100644 --- a/scripts/test.gen_build.ps1 +++ b/scripts/test.gen_build.ps1 @@ -1,2 +1,2 @@ cls -Invoke-Expression "& $(Join-Path $PSScriptRoot 'build.ci.ps1') $args" +Invoke-Expression "& $(Join-Path $PSScriptRoot 'test.gen_build.ci.ps1') $args" diff --git a/test/test.cpp b/test/test.cpp index a91dbfa..77114a6 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -12,7 +12,7 @@ int gen_main() using namespace gen; log_fmt("\ngen_time:"); - check_sanity(); + // check_sanity(); check_SOA(); From 0a5885495f3f6438add994425759710513b66425 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 16:00:06 -0400 Subject: [PATCH 06/10] got old tests working (test.parsing.cpp and test.upfront.cpp) --- project/gen.dep.hpp | 1 + test/Parsed/Array.Parsed.hpp | 2 +- test/Parsed/Buffer.Parsed.hpp | 2 +- test/Parsed/HashTable.Parsed.hpp | 2 +- test/Parsed/Ring.Parsed.hpp | 2 +- test/Upfront/Array.Upfront.hpp | 4 +++- test/Upfront/Buffer.Upfront.hpp | 3 ++- test/Upfront/HashTable.Upfront.hpp | 2 +- test/Upfront/Ring.Upfront.hpp | 2 +- test/Upfront/Sanity.Upfront.hpp | 4 ++-- test/test.cpp | 2 +- test/{test.Upfront.cpp => test.upfront.cpp} | 0 12 files changed, 15 insertions(+), 11 deletions(-) rename test/{test.Upfront.cpp => test.upfront.cpp} (100%) diff --git a/project/gen.dep.hpp b/project/gen.dep.hpp index f315397..0e0ae16 100644 --- a/project/gen.dep.hpp +++ b/project/gen.dep.hpp @@ -20,6 +20,7 @@ GEN_NS_BEGIN #include "dependencies/string_ops.hpp" #include "dependencies/printing.hpp" #include "dependencies/containers.hpp" +#include "dependencies/hashing.hpp" #include "dependencies/string.hpp" #include "dependencies/parsing.hpp" #include "dependencies/timing.hpp" diff --git a/test/Parsed/Array.Parsed.hpp b/test/Parsed/Array.Parsed.hpp index 61b19cb..9b400dc 100644 --- a/test/Parsed/Array.Parsed.hpp +++ b/test/Parsed/Array.Parsed.hpp @@ -229,7 +229,7 @@ Array GenArrayRequests; void gen__array_request( StrC type, StrC dep = {} ) { do_once_start - GenArrayRequests = Array::init( Memory::GlobalAllocator ); + GenArrayRequests = Array::init( GlobalAllocator ); do_once_end // Make sure we don't already have a request for the type. diff --git a/test/Parsed/Buffer.Parsed.hpp b/test/Parsed/Buffer.Parsed.hpp index 9c48679..77ca4e4 100644 --- a/test/Parsed/Buffer.Parsed.hpp +++ b/test/Parsed/Buffer.Parsed.hpp @@ -143,7 +143,7 @@ Array GenBufferRequests; void gen__buffer_request( StrC type, StrC dep = {} ) { do_once_start - GenBufferRequests = Array::init( Memory::GlobalAllocator ); + GenBufferRequests = Array::init( GlobalAllocator ); do_once_end // Make sure we don't already have a request for the type. diff --git a/test/Parsed/HashTable.Parsed.hpp b/test/Parsed/HashTable.Parsed.hpp index ffe15ef..439c8df 100644 --- a/test/Parsed/HashTable.Parsed.hpp +++ b/test/Parsed/HashTable.Parsed.hpp @@ -292,7 +292,7 @@ Array GenHashTableRequests; void gen__hashtable_request( StrC type, StrC dep = {} ) { do_once_start - GenHashTableRequests = Array::init( Memory::GlobalAllocator ); + GenHashTableRequests = Array::init( GlobalAllocator ); gen_array( sw ); do_once_end diff --git a/test/Parsed/Ring.Parsed.hpp b/test/Parsed/Ring.Parsed.hpp index cf2b1d3..12a3b36 100644 --- a/test/Parsed/Ring.Parsed.hpp +++ b/test/Parsed/Ring.Parsed.hpp @@ -109,7 +109,7 @@ Array GenRingRequests; void gen__ring_request( StrC type, StrC dep = {} ) { do_once_start - GenRingRequests = Array::init( Memory::GlobalAllocator ); + GenRingRequests = Array::init( GlobalAllocator ); do_once_end // Make sure we don't already have a request for the type. diff --git a/test/Upfront/Array.Upfront.hpp b/test/Upfront/Array.Upfront.hpp index 6a378e3..ee561c8 100644 --- a/test/Upfront/Array.Upfront.hpp +++ b/test/Upfront/Array.Upfront.hpp @@ -1,5 +1,7 @@ #pragma once + + #if GEN_TIME #include "gen.hpp" @@ -308,7 +310,7 @@ Array GenArrayRequests; void gen__array_request( StrC type, StrC dep = {} ) { do_once_start - GenArrayRequests = Array::init( Memory::GlobalAllocator ); + GenArrayRequests = Array::init( GlobalAllocator ); do_once_end // Make sure we don't already have a request for the type. diff --git a/test/Upfront/Buffer.Upfront.hpp b/test/Upfront/Buffer.Upfront.hpp index 1a78ded..9ced0d3 100644 --- a/test/Upfront/Buffer.Upfront.hpp +++ b/test/Upfront/Buffer.Upfront.hpp @@ -190,6 +190,7 @@ Code gen__buffer( StrC type, sw type_size ) , free , get_header , num + , pop , wipe , op_type_ptr @@ -212,7 +213,7 @@ Array GenBufferRequests; void gen__buffer_request( StrC type, StrC dep = {} ) { do_once_start - GenBufferRequests = Array::init( Memory::GlobalAllocator ); + GenBufferRequests = Array::init( GlobalAllocator ); do_once_end // Make sure we don't already have a request for the type. diff --git a/test/Upfront/HashTable.Upfront.hpp b/test/Upfront/HashTable.Upfront.hpp index af514ff..2a8d03a 100644 --- a/test/Upfront/HashTable.Upfront.hpp +++ b/test/Upfront/HashTable.Upfront.hpp @@ -419,7 +419,7 @@ Array GenHashTableRequests; void gen__hashtable_request( StrC type, StrC dep = {} ) { do_once_start - GenHashTableRequests = Array::init( Memory::GlobalAllocator ); + GenHashTableRequests = Array::init( GlobalAllocator ); gen_array( sw ); do_once_end diff --git a/test/Upfront/Ring.Upfront.hpp b/test/Upfront/Ring.Upfront.hpp index e7446b8..ade8614 100644 --- a/test/Upfront/Ring.Upfront.hpp +++ b/test/Upfront/Ring.Upfront.hpp @@ -163,7 +163,7 @@ Array GenRingRequests; void gen__ring_request( StrC type, StrC dep = {} ) { do_once_start - GenRingRequests = Array::init( Memory::GlobalAllocator ); + GenRingRequests = Array::init( GlobalAllocator ); do_once_end // Make sure we don't already have a request for the type. diff --git a/test/Upfront/Sanity.Upfront.hpp b/test/Upfront/Sanity.Upfront.hpp index 8508b99..6fb1c07 100644 --- a/test/Upfront/Sanity.Upfront.hpp +++ b/test/Upfront/Sanity.Upfront.hpp @@ -286,8 +286,8 @@ u32 gen_sanity_upfront() // Using { - CodeUsing reg = def_using( name(TestUsing), t_u8 ); - CodeUsingNamespace nspace = def_using_namespace( name(TestNamespace) ); + CodeUsing reg = def_using( name(TestUsing), t_u8 ); + CodeUsing nspace = def_using_namespace( name(TestNamespace) ); gen_sanity_file.print(reg); gen_sanity_file.print(nspace); diff --git a/test/test.cpp b/test/test.cpp index 77114a6..7e96636 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -14,7 +14,7 @@ int gen_main() // check_sanity(); - check_SOA(); + checkSOA(); return 0; } diff --git a/test/test.Upfront.cpp b/test/test.upfront.cpp similarity index 100% rename from test/test.Upfront.cpp rename to test/test.upfront.cpp From 50a10c901f7a98df3d037fd3a336ae08a2705214 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 16:24:07 -0400 Subject: [PATCH 07/10] renamed Docs to lowercase --- {Docs => docs}/Parsing.md | 0 {Docs => docs}/Readme.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {Docs => docs}/Parsing.md (100%) rename {Docs => docs}/Readme.md (100%) diff --git a/Docs/Parsing.md b/docs/Parsing.md similarity index 100% rename from Docs/Parsing.md rename to docs/Parsing.md diff --git a/Docs/Readme.md b/docs/Readme.md similarity index 100% rename from Docs/Readme.md rename to docs/Readme.md From d704f11c81e4452ccce00ed47d9c161f00c98e52 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 16:24:07 -0400 Subject: [PATCH 08/10] renamed Docs to lowercase --- {Docs => docs}/Parsing.md | 0 {Docs => docs}/Readme.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {Docs => docs}/Parsing.md (100%) rename {Docs => docs}/Readme.md (100%) diff --git a/Docs/Parsing.md b/docs/Parsing.md similarity index 100% rename from Docs/Parsing.md rename to docs/Parsing.md diff --git a/Docs/Readme.md b/docs/Readme.md similarity index 100% rename from Docs/Readme.md rename to docs/Readme.md From b9064fba9d27ac0675e6c5d2650f863602f5987f Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 16:27:36 -0400 Subject: [PATCH 09/10] renamed parsed and upfront dirs to lowercase --- test/{Parsed => parsed}/Array.Parsed.hpp | 0 test/{Parsed => parsed}/Buffer.Parsed.hpp | 0 test/{Parsed => parsed}/HashTable.Parsed.hpp | 0 test/{Parsed => parsed}/Ring.Parsed.hpp | 0 test/{Parsed => parsed}/Sanity.Parsed.hpp | 0 test/{Upfront => upfront}/Array.Upfront.hpp | 0 test/{Upfront => upfront}/Buffer.Upfront.hpp | 0 test/{Upfront => upfront}/HashTable.Upfront.hpp | 0 test/{Upfront => upfront}/Ring.Upfront.hpp | 0 test/{Upfront => upfront}/Sanity.Upfront.hpp | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename test/{Parsed => parsed}/Array.Parsed.hpp (100%) rename test/{Parsed => parsed}/Buffer.Parsed.hpp (100%) rename test/{Parsed => parsed}/HashTable.Parsed.hpp (100%) rename test/{Parsed => parsed}/Ring.Parsed.hpp (100%) rename test/{Parsed => parsed}/Sanity.Parsed.hpp (100%) rename test/{Upfront => upfront}/Array.Upfront.hpp (100%) rename test/{Upfront => upfront}/Buffer.Upfront.hpp (100%) rename test/{Upfront => upfront}/HashTable.Upfront.hpp (100%) rename test/{Upfront => upfront}/Ring.Upfront.hpp (100%) rename test/{Upfront => upfront}/Sanity.Upfront.hpp (100%) diff --git a/test/Parsed/Array.Parsed.hpp b/test/parsed/Array.Parsed.hpp similarity index 100% rename from test/Parsed/Array.Parsed.hpp rename to test/parsed/Array.Parsed.hpp diff --git a/test/Parsed/Buffer.Parsed.hpp b/test/parsed/Buffer.Parsed.hpp similarity index 100% rename from test/Parsed/Buffer.Parsed.hpp rename to test/parsed/Buffer.Parsed.hpp diff --git a/test/Parsed/HashTable.Parsed.hpp b/test/parsed/HashTable.Parsed.hpp similarity index 100% rename from test/Parsed/HashTable.Parsed.hpp rename to test/parsed/HashTable.Parsed.hpp diff --git a/test/Parsed/Ring.Parsed.hpp b/test/parsed/Ring.Parsed.hpp similarity index 100% rename from test/Parsed/Ring.Parsed.hpp rename to test/parsed/Ring.Parsed.hpp diff --git a/test/Parsed/Sanity.Parsed.hpp b/test/parsed/Sanity.Parsed.hpp similarity index 100% rename from test/Parsed/Sanity.Parsed.hpp rename to test/parsed/Sanity.Parsed.hpp diff --git a/test/Upfront/Array.Upfront.hpp b/test/upfront/Array.Upfront.hpp similarity index 100% rename from test/Upfront/Array.Upfront.hpp rename to test/upfront/Array.Upfront.hpp diff --git a/test/Upfront/Buffer.Upfront.hpp b/test/upfront/Buffer.Upfront.hpp similarity index 100% rename from test/Upfront/Buffer.Upfront.hpp rename to test/upfront/Buffer.Upfront.hpp diff --git a/test/Upfront/HashTable.Upfront.hpp b/test/upfront/HashTable.Upfront.hpp similarity index 100% rename from test/Upfront/HashTable.Upfront.hpp rename to test/upfront/HashTable.Upfront.hpp diff --git a/test/Upfront/Ring.Upfront.hpp b/test/upfront/Ring.Upfront.hpp similarity index 100% rename from test/Upfront/Ring.Upfront.hpp rename to test/upfront/Ring.Upfront.hpp diff --git a/test/Upfront/Sanity.Upfront.hpp b/test/upfront/Sanity.Upfront.hpp similarity index 100% rename from test/Upfront/Sanity.Upfront.hpp rename to test/upfront/Sanity.Upfront.hpp From 03df940085113e0ce1e67da3a58384ae8fa9dc7e Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 17:14:02 -0400 Subject: [PATCH 10/10] Improved parser scope errors. Added the caret for indicating where the error is. --- project/components/etoktype.cpp | 8 +- project/components/interface.parsing.cpp | 96 +++++++++++++----------- project/dependencies/debug.cpp | 2 +- project/enums/ETokType.csv | 8 +- test/test.cpp | 4 +- 5 files changed, 64 insertions(+), 54 deletions(-) diff --git a/project/components/etoktype.cpp b/project/components/etoktype.cpp index 5516221..adaea66 100644 --- a/project/components/etoktype.cpp +++ b/project/components/etoktype.cpp @@ -33,8 +33,8 @@ namespace Parser Entry( BraceSquare_Close, "]" ) \ Entry( Capture_Start, "(" ) \ Entry( Capture_End, ")" ) \ - Entry( Comment, "__comment__" ) \ - Entry( Char, "__char__" ) \ + Entry( Comment, "comment" ) \ + Entry( Char, "character" ) \ Entry( Comma, "," ) \ Entry( Decl_Class, "class" ) \ Entry( Decl_GNU_Attribute, "__attribute__" ) \ @@ -50,7 +50,7 @@ namespace Parser Entry( Decl_Typedef, "typedef" ) \ Entry( Decl_Using, "using" ) \ Entry( Decl_Union, "union" ) \ - Entry( Identifier, "__identifier__" ) \ + Entry( Identifier, "identifier" ) \ Entry( Module_Import, "import" ) \ Entry( Module_Export, "export" ) \ Entry( Number, "number" ) \ @@ -80,7 +80,7 @@ namespace Parser Entry( Spec_Volatile, "volatile") \ Entry( Star, "*" ) \ Entry( Statement_End, ";" ) \ - Entry( String, "__string__" ) \ + Entry( String, "string" ) \ Entry( Type_Unsigned, "unsigned" ) \ Entry( Type_Signed, "signed" ) \ Entry( Type_Short, "short" ) \ diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index 915a288..22b46cd 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -82,6 +82,7 @@ namespace Parser { StackNode* Prev; + Token Start; Token Name; // The name of the AST node (if parsed) StrC ProcName; // The name of the procedure }; @@ -91,44 +92,6 @@ namespace Parser TokArray Tokens; StackNode* Scope; - String to_string() - { - String result = String::make_reserve( GlobalAllocator, kilobytes(4) ); - - result.append_fmt("\tContext:\n"); - - Token last_valid = Tokens.Idx >= Tokens.Arr.num() ? Tokens.Arr[Tokens.Arr.num() -1] : Tokens.current(); - - char const* current = last_valid.Text; - sptr length = last_valid.Length; - while ( current <= Tokens.Arr.back().Text && *current != '\n' ) - { - current++; - length++; - } - - String line = String::make( GlobalAllocator, { length, last_valid.Text } ); - result.append_fmt("\t(%d, %d): %s\n", last_valid.Line, last_valid.Column, line ); - line.free(); - - StackNode* curr_scope = Scope; - do - { - if ( curr_scope->Name ) - { - result.append_fmt("\tProcedure: %s, AST Name: %s\n", curr_scope->ProcName.Ptr, (StrC)curr_scope->Name ); - } - else - { - result.append_fmt("\tProcedure: %s\n", curr_scope->ProcName.Ptr ); - } - - curr_scope = curr_scope->Prev; - } - while ( curr_scope ); - return result; - } - void push( StackNode* node ) { node->Prev = Scope; @@ -139,6 +102,51 @@ namespace Parser { Scope = Scope->Prev; } + + String to_string() + { + String result = String::make_reserve( GlobalAllocator, kilobytes(4) ); + + Token scope_start = Scope->Start; + Token last_valid = Tokens.Idx >= Tokens.Arr.num() ? Tokens.Arr[Tokens.Arr.num() -1] : Tokens.current(); + + sptr length = scope_start.Length; + char const* current = scope_start.Text + length; + while ( current <= Tokens.Arr.back().Text && *current != '\n' && length < 74 ) + { + current++; + length++; + } + + String line = String::make( GlobalAllocator, { length, scope_start.Text } ); + result.append_fmt("\tScope: %s\n", line ); + line.free(); + + sptr dist = (sptr)last_valid.Text - (sptr)scope_start.Text; + sptr length_from_err = dist; + String line_from_err = String::make( GlobalAllocator, { length_from_err, last_valid.Text } ); + + result.append_fmt("\t(%d, %d):%*c\n", last_valid.Line, last_valid.Column, length_from_err, '^' ); + + StackNode* curr_scope = Scope; + s32 level = 0; + do + { + if ( curr_scope->Name ) + { + result.append_fmt("\t%d: %s, AST Name: %s\n", level, curr_scope->ProcName.Ptr, (StrC)curr_scope->Name ); + } + else + { + result.append_fmt("\t%d: %s\n", level, curr_scope->ProcName.Ptr ); + } + + curr_scope = curr_scope->Prev; + level++; + } + while ( curr_scope ); + return result; + } }; global ParseContext Context; @@ -155,7 +163,7 @@ namespace Parser { String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } ); - log_failure( "Parse Error, TokArray::eat, Expected: %s, not %s (%d, %d)`\n%s", ETokType::to_str(type), token_str, Context.to_string() ); + log_failure( "Parse Error, TokArray::eat, Expected: %s, not '%s' (%d, %d)`\n%s", ETokType::to_str(type), token_str, current().Line, current().Column, Context.to_string() ); return false; } @@ -727,7 +735,7 @@ if ( def.Ptr == nullptr ) # define check( Type_ ) ( left && currtok.Type == Type_ ) # define push_scope() \ - StackNode scope { nullptr, NullToken, txt_StrC( __func__ ) }; \ + StackNode scope { nullptr, currtok, NullToken, txt_StrC( __func__ ) }; \ Context.push( & scope ) #pragma endregion Helper Macros @@ -1796,8 +1804,7 @@ Code parse_class_struct( Parser::TokType which ) attributes = parse_attributes(); - if ( check( TokType::Identifier ) ) - name = parse_identifier(); + name = parse_identifier(); local_persist char interface_arr_mem[ kilobytes(4) ] {0}; @@ -2649,7 +2656,10 @@ CodeStruct parse_struct( StrC def ) return CodeInvalid; Context.Tokens = toks; - return (CodeStruct) parse_class_struct( TokType::Decl_Struct ); + push_scope(); + CodeStruct result = (CodeStruct) parse_class_struct( TokType::Decl_Struct ); + Context.pop(); + return result; } internal diff --git a/project/dependencies/debug.cpp b/project/dependencies/debug.cpp index 357f198..25b6e1f 100644 --- a/project/dependencies/debug.cpp +++ b/project/dependencies/debug.cpp @@ -5,7 +5,7 @@ void assert_handler( char const* condition, char const* file, s32 line, char con _printf_err( "%s:(%d): Assert Failure: ", file, line ); if ( condition ) - _printf_err( "`%s` ", condition ); + _printf_err( "`%s` \n", condition ); if ( msg ) { diff --git a/project/enums/ETokType.csv b/project/enums/ETokType.csv index 2056739..e0ff1f5 100644 --- a/project/enums/ETokType.csv +++ b/project/enums/ETokType.csv @@ -15,8 +15,8 @@ BraceSquare_Open, "[" BraceSquare_Close, "]" Capture_Start, "(" Capture_End, ")" -Comment, "__comment__" -Char, "__char__" +Comment, "comemnt" +Char, "character" Comma, "," Decl_Class, "class" Decl_GNU_Attribute, "__attribute__" @@ -32,7 +32,7 @@ Decl_Template, "template" Decl_Typedef, "typedef" Decl_Using, "using" Decl_Union, "union" -Identifier, "__identifier__" +Identifier, "identifier" Module_Import, "import" Module_Export, "export" Number, "number" @@ -62,7 +62,7 @@ Spec_ThreadLocal, "thread_local" Spec_Volatile, "volatile" Star, "*" Statement_End, ";" -String, "__string__" +String, "string" Type_Unsigned, "unsigned" Type_Signed, "signed" Type_Short, "short" diff --git a/test/test.cpp b/test/test.cpp index 7e96636..a91dbfa 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -12,9 +12,9 @@ int gen_main() using namespace gen; log_fmt("\ngen_time:"); - // check_sanity(); + check_sanity(); - checkSOA(); + check_SOA(); return 0; }