From 3c249d2fae1d52133063d66a3fc287d5562c558e Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 14 Dec 2024 11:24:21 -0500 Subject: [PATCH] pregress on fixing regressions with unreal parsing https://github.com/Ed94/gencpp/issues/64 is required. Resolving it next --- base/components/ast.cpp | 14 +-- base/components/ast_case_macros.cpp | 16 +-- base/components/interface.upfront.cpp | 28 ++--- base/components/parser.cpp | 100 +++------------- base/components/parser_case_macros.cpp | 105 +++++++++++++++++ base/components/parser_types.hpp | 18 +++ base/gen.cpp | 1 + base/helpers/base_codegen.hpp | 42 +++---- base/helpers/undef.macros.h | 12 +- gen_c_library/c_library.cpp | 3 + gen_segmented/segmented.cpp | 2 + gen_singleheader/singleheader.cpp | 22 ++-- .../components/parser_case_macros.cpp | 108 ++++++++++++++++++ gen_unreal_engine/enums/ESpecifier.csv | 27 +++++ gen_unreal_engine/enums/ETokType.csv | 96 ++++++++++++++++ gen_unreal_engine/unreal.cpp | 6 +- 16 files changed, 439 insertions(+), 161 deletions(-) create mode 100644 base/components/parser_case_macros.cpp create mode 100644 gen_unreal_engine/components/parser_case_macros.cpp create mode 100644 gen_unreal_engine/enums/ESpecifier.csv create mode 100644 gen_unreal_engine/enums/ETokType.csv diff --git a/base/components/ast.cpp b/base/components/ast.cpp index f9a4d8c..c538a4c 100644 --- a/base/components/ast.cpp +++ b/base/components/ast.cpp @@ -1165,7 +1165,7 @@ bool code_validate_body(Code self) CodeBody body = cast(CodeBody, self); for (Code code_entry = begin_CodeBody(body); code_entry != end_CodeBody(body); next_CodeBody(body, code_entry)) switch (code_entry->Type) { - GEN_AST_BODY_CLASS_UNALLOWED_TYPES: + GEN_AST_BODY_CLASS_UNALLOWED_TYPES_CASES: log_failure("AST::validate_body: Invalid entry in body %S", code_debug_str(code_entry)); return false; @@ -1192,7 +1192,7 @@ bool code_validate_body(Code self) CodeBody body = cast(CodeBody, self); for (Code code_entry = begin_CodeBody(body); code_entry != end_CodeBody(body); next_CodeBody(body, code_entry)) switch (code_entry->Type) { - GEN_AST_BODY_EXPORT_UNALLOWED_TYPES: + GEN_AST_BODY_EXPORT_UNALLOWED_TYPES_CASES: log_failure("AST::validate_body: Invalid entry in body %S", code_debug_str(code_entry)); return false; @@ -1206,7 +1206,7 @@ bool code_validate_body(Code self) CodeBody body = cast(CodeBody, self); for (Code code_entry = begin_CodeBody(body); code_entry != end_CodeBody(body); next_CodeBody(body, code_entry)) switch (code_entry->Type) { - GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES: + GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES_CASES: log_failure("AST::validate_body: Invalid entry in body %S", code_debug_str(code_entry)); return false; @@ -1220,7 +1220,7 @@ bool code_validate_body(Code self) CodeBody body = cast(CodeBody, self); for (Code code_entry = begin_CodeBody(body); code_entry != end_CodeBody(body); next_CodeBody(body, code_entry)) switch (code_entry->Type) { - GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES: + GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES_CASES: log_failure("AST::validate_body: Invalid entry in body %S", code_debug_str(code_entry)); return false; @@ -1234,7 +1234,7 @@ bool code_validate_body(Code self) CodeBody body = cast(CodeBody, self); for ( Code entry = begin_CodeBody(body); entry != end_CodeBody(body); next_CodeBody(body, entry) )switch (entry->Type) { - GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES: + GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES_CASES: log_failure("AST::validate_body: Invalid entry in body %S", code_debug_str(entry)); return false; } @@ -1245,7 +1245,7 @@ bool code_validate_body(Code self) CodeBody body = cast(CodeBody, self); for ( Code entry = begin_CodeBody(body); entry != end_CodeBody(body); next_CodeBody(body, entry) ) switch (entry->Type) { - GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES: + GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES_CASES: log_failure("AST::validate_body: Invalid entry in body %S", code_debug_str(entry)); return false; } @@ -1256,7 +1256,7 @@ bool code_validate_body(Code self) CodeBody body = cast(CodeBody, self); for ( Code entry = begin_CodeBody(body); entry != end_CodeBody(body); next_CodeBody(body, entry) ) switch (entry->Type) { - GEN_AST_BODY_STRUCT_UNALLOWED_TYPES: + GEN_AST_BODY_STRUCT_UNALLOWED_TYPES_CASES: log_failure("AST::validate_body: Invalid entry in body %S", code_debug_str(entry)); return false; } diff --git a/base/components/ast_case_macros.cpp b/base/components/ast_case_macros.cpp index 4959914..f7f66c2 100644 --- a/base/components/ast_case_macros.cpp +++ b/base/components/ast_case_macros.cpp @@ -1,6 +1,6 @@ -// These macros are used in the swtich cases are used within ast.cpp, inteface.upfront.cpp, parser.cpp +// These macros are used in the swtich cases within ast.cpp, inteface.upfront.cpp, parser.cpp -# define GEN_AST_BODY_CLASS_UNALLOWED_TYPES \ +# define GEN_AST_BODY_CLASS_UNALLOWED_TYPES_CASES \ case CT_PlatformAttributes: \ case CT_Class_Body: \ case CT_Enum_Body: \ @@ -16,9 +16,9 @@ case CT_Specifiers: \ case CT_Struct_Body: \ case CT_Typename -# define GEN_AST_BODY_STRUCT_UNALLOWED_TYPES GEN_AST_BODY_CLASS_UNALLOWED_TYPES +# define GEN_AST_BODY_STRUCT_UNALLOWED_TYPES_CASES GEN_AST_BODY_CLASS_UNALLOWED_TYPES_CASES -# define GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES \ +# define GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES_CASES \ case CT_Access_Public: \ case CT_Access_Protected: \ case CT_Access_Private: \ @@ -41,7 +41,7 @@ case CT_Struct_Body: \ case CT_Typename -# define GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES \ +# define GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES_CASES \ case CT_Access_Public: \ case CT_Access_Protected: \ case CT_Access_Private: \ @@ -58,10 +58,10 @@ case CT_Specifiers: \ case CT_Struct_Body: \ case CT_Typename -# define GEN_AST_BODY_EXPORT_UNALLOWED_TYPES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES -# define GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES +# define GEN_AST_BODY_EXPORT_UNALLOWED_TYPES_CASES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES_CASES +# define GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES_CASES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES_CASES -# define GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES \ +# define GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES_CASES \ case CT_Access_Public: \ case CT_Access_Protected: \ case CT_Access_Private: \ diff --git a/base/components/interface.upfront.cpp b/base/components/interface.upfront.cpp index 88f713a..0faec68 100644 --- a/base/components/interface.upfront.cpp +++ b/base/components/interface.upfront.cpp @@ -1364,7 +1364,7 @@ CodeBody def_class_body( s32 num, ... ) } switch (entry->Type) { - GEN_AST_BODY_CLASS_UNALLOWED_TYPES: + GEN_AST_BODY_CLASS_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_class_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1396,7 +1396,7 @@ CodeBody def_class_body( s32 num, Code* codes ) } switch (entry->Type) { - GEN_AST_BODY_CLASS_UNALLOWED_TYPES: + GEN_AST_BODY_CLASS_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_class_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1486,7 +1486,7 @@ CodeBody def_export_body( s32 num, ... ) } switch (entry->Type) { - GEN_AST_BODY_EXPORT_UNALLOWED_TYPES: + GEN_AST_BODY_EXPORT_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_export_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1518,7 +1518,7 @@ CodeBody def_export_body( s32 num, Code* codes ) } switch (entry->Type) { - GEN_AST_BODY_EXPORT_UNALLOWED_TYPES: + GEN_AST_BODY_EXPORT_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_export_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1552,7 +1552,7 @@ CodeBody def_extern_link_body( s32 num, ... ) } switch (entry->Type) { - GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES: + GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_extern_linkage_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1585,7 +1585,7 @@ CodeBody def_extern_link_body( s32 num, Code* codes ) } switch (entry->Type) { - GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES: + GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_extern_linkage_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1619,7 +1619,7 @@ CodeBody def_function_body( s32 num, ... ) } switch (entry->Type) { - GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES: + GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES_CASES: log_failure("gen::" stringize(def_function_body) ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1651,7 +1651,7 @@ CodeBody def_function_body( s32 num, Code* codes ) } switch (entry->Type) { - GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES: + GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_function_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1690,7 +1690,7 @@ CodeBody def_global_body( s32 num, ... ) body_append_body( result, cast(CodeBody, entry) ); continue; - GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES: + GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_global_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1726,7 +1726,7 @@ CodeBody def_global_body( s32 num, Code* codes ) body_append_body(result, cast(CodeBody, entry) ); continue; - GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES: + GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_global_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1761,7 +1761,7 @@ CodeBody def_namespace_body( s32 num, ... ) } switch (entry->Type) { - GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES: + GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_namespace_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1793,7 +1793,7 @@ CodeBody def_namespace_body( s32 num, Code* codes ) } switch (entry->Type) { - GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES: + GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_namespace_body" ": Entry type is not allowed: %s", code_debug_str(entry) ); return InvalidCode; @@ -1938,7 +1938,7 @@ CodeBody def_struct_body( s32 num, ... ) } switch (entry->Type) { - GEN_AST_BODY_STRUCT_UNALLOWED_TYPES: + GEN_AST_BODY_STRUCT_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_struct_body" ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1970,7 +1970,7 @@ CodeBody def_struct_body( s32 num, Code* codes ) } switch (entry->Type) { - GEN_AST_BODY_STRUCT_UNALLOWED_TYPES: + GEN_AST_BODY_STRUCT_UNALLOWED_TYPES_CASES: log_failure("gen::" "def_struct_body" ": Entry type is not allowed: %s", code_debug_str(entry) ); return InvalidCode; diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 61329b5..3c850f8 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -1,6 +1,7 @@ #ifdef GEN_INTELLISENSE_DIRECTIVES #pragma once #include "gen/etoktype.cpp" +#include "parser_case_macros.cpp" #include "interface.upfront.cpp" #include "lexer.cpp" #endif @@ -979,17 +980,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) // } //! Fallthrough intended - case Tok_Spec_Consteval: - case Tok_Spec_Constexpr: - case Tok_Spec_Constinit: - case Tok_Spec_Explicit: - case Tok_Spec_ForceInline: - case Tok_Spec_Inline: - case Tok_Spec_Mutable: - case Tok_Spec_NeverInline: - case Tok_Spec_Static: - case Tok_Spec_Volatile: - case Tok_Spec_Virtual: + GEN_PARSER_CLASS_STRUCT_BODY_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES: { Specifier specs_found[16] = { Spec_NumSpecifiers }; s32 NumSpecifiers = 0; @@ -1002,16 +993,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) switch ( spec ) { - case Spec_Constexpr: - case Spec_Constinit: - case Spec_Explicit: - case Spec_Inline: - case Spec_ForceInline: - case Spec_Mutable: - case Spec_NeverInline: - case Spec_Static: - case Spec_Volatile: - case Spec_Virtual: + GEN_PARSER_CLASS_STRUCT_BODY_ALLOWED_MEMBER_SPECIFIERS_CASES: break; case Spec_Consteval: @@ -1023,7 +1005,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; default: - log_failure( "Invalid specifier %S for variable\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); + log_failure( "Invalid specifier %S for class/struct member\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); parser_pop(& _ctx->parser); return InvalidCode; } @@ -1792,16 +1774,7 @@ CodeBody parse_global_nspace( CodeType which ) // } //! Fallthrough intentional - case Tok_Spec_Consteval: - case Tok_Spec_Constexpr: - case Tok_Spec_Constinit: - case Tok_Spec_Extern: - case Tok_Spec_ForceInline: - case Tok_Spec_Global: - case Tok_Spec_Inline: - case Tok_Spec_Internal_Linkage: - case Tok_Spec_NeverInline: - case Tok_Spec_Static: + GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES: { Specifier specs_found[16] = { Spec_NumSpecifiers }; s32 NumSpecifiers = 0; @@ -1814,17 +1787,7 @@ CodeBody parse_global_nspace( CodeType which ) switch ( spec ) { - case Spec_Constexpr: - case Spec_Constinit: - case Spec_ForceInline: - case Spec_Global: - case Spec_External_Linkage: - case Spec_Internal_Linkage: - case Spec_Inline: - case Spec_Mutable: - case Spec_NeverInline: - case Spec_Static: - case Spec_Volatile: + GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIERS_CASES: break; case Spec_Consteval: @@ -2560,7 +2523,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes #ifndef GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES b32 lone_macro = false; - if ( currtok.Type == Tok_Preprocess_Macro && nexttok.Type == Tok_Statement_End ) + if ( currtok.Type == Tok_Preprocess_Macro && ( nexttok.Type == Tok_Statement_End || nexttok.Type == Tok_Comment ) ) { // Were dealing with a lone macro after attributes/specifiers, there was a end statement ';' after. result = parse_simple_preprocess( Tok_Preprocess_Macro, parser_consume_braces ); @@ -3971,10 +3934,8 @@ CodeFriend parser_parse_friend() switch ( spec ) { - case Spec_Const : - case Spec_Inline : - case Spec_ForceInline : - break; + GEN_PARSER_FRIEND_ALLOWED_SPECIFIERS_CASES: + break; default : log_failure( "Invalid specifier %S for friend definition\n%S", spec_to_str( spec ), strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); @@ -4095,14 +4056,7 @@ CodeFn parser_parse_function() switch ( spec ) { - case Spec_Const: - case Spec_Consteval: - case Spec_Constexpr: - case Spec_External_Linkage: - case Spec_ForceInline: - case Spec_Inline: - case Spec_NeverInline: - case Spec_Static: + GEN_PARSER_FUNCTION_ALLOWED_SPECIFIERS_CASES: break; default: @@ -4208,12 +4162,7 @@ CodeOperator parser_parse_operator() switch ( spec ) { - case Spec_Const: - case Spec_Constexpr: - case Spec_ForceInline: - case Spec_Inline: - case Spec_NeverInline: - case Spec_Static: + GEN_PARSER_OPERATOR_ALLOWED_SPECIFIERS_CASES: break; default: @@ -4451,19 +4400,8 @@ CodeTemplate parser_parse_template() switch ( spec ) { - case Spec_Const : - case Spec_Constexpr : - case Spec_Constinit : - case Spec_External_Linkage : - case Spec_Global : - case Spec_Inline : - case Spec_ForceInline : - case Spec_Local_Persist : - case Spec_Mutable : - case Spec_Static : - case Spec_Thread_Local : - case Spec_Volatile : - break; + GEN_PARSER_TEMPLATE_ALLOWED_SPECIFIERS_CASES: + break; case Spec_Consteval : expects_function = true; @@ -5509,17 +5447,7 @@ CodeVar parser_parse_variable() Specifier spec = str_to_specifier( tok_to_str(currtok) ); switch ( spec ) { - case Spec_Const: - case Spec_Constexpr: - case Spec_Constinit: - case Spec_External_Linkage: - case Spec_Global: - case Spec_Inline: - case Spec_Local_Persist: - case Spec_Mutable: - case Spec_Static: - case Spec_Thread_Local: - case Spec_Volatile: + GEN_PARSER_VARIABLE_ALLOWED_SPECIFIER_CASES: break; default: diff --git a/base/components/parser_case_macros.cpp b/base/components/parser_case_macros.cpp new file mode 100644 index 0000000..c2497c4 --- /dev/null +++ b/base/components/parser_case_macros.cpp @@ -0,0 +1,105 @@ +// These macros are used in the swtich cases within parser.cpp + +#define GEN_PARSER_CLASS_STRUCT_BODY_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES \ +case Tok_Spec_Consteval: \ +case Tok_Spec_Constexpr: \ +case Tok_Spec_Constinit: \ +case Tok_Spec_Explicit: \ +case Tok_Spec_ForceInline: \ +case Tok_Spec_Inline: \ +case Tok_Spec_Mutable: \ +case Tok_Spec_NeverInline: \ +case Tok_Spec_Static: \ +case Tok_Spec_Volatile: \ +case Tok_Spec_Virtual + +#define GEN_PARSER_CLASS_STRUCT_BODY_ALLOWED_MEMBER_SPECIFIERS_CASES \ +case Spec_Constexpr: \ +case Spec_Constinit: \ +case Spec_Explicit: \ +case Spec_Inline: \ +case Spec_ForceInline: \ +case Spec_Mutable: \ +case Spec_NeverInline: \ +case Spec_Static: \ +case Spec_Volatile: \ +case Spec_Virtual + +#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES \ +case Tok_Spec_Consteval: \ +case Tok_Spec_Constexpr: \ +case Tok_Spec_Constinit: \ +case Tok_Spec_Extern: \ +case Tok_Spec_ForceInline: \ +case Tok_Spec_Global: \ +case Tok_Spec_Inline: \ +case Tok_Spec_Internal_Linkage: \ +case Tok_Spec_NeverInline: \ +case Tok_Spec_Static + +#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIERS_CASES \ +case Spec_Constexpr: \ +case Spec_Constinit: \ +case Spec_ForceInline: \ +case Spec_Global: \ +case Spec_External_Linkage: \ +case Spec_Internal_Linkage: \ +case Spec_Inline: \ +case Spec_Mutable: \ +case Spec_NeverInline: \ +case Spec_Static: \ +case Spec_Volatile + +#define GEN_PARSER_FRIEND_ALLOWED_SPECIFIERS_CASES \ +case Spec_Const: \ +case Spec_Inline: \ +case Spec_ForceInline + +#define GEN_PARSER_FUNCTION_ALLOWED_SPECIFIERS_CASES \ +case Spec_Const: \ +case Spec_Consteval: \ +case Spec_Constexpr: \ +case Spec_External_Linkage: \ +case Spec_Internal_Linkage: \ +case Spec_ForceInline: \ +case Spec_Inline: \ +case Spec_NeverInline: \ +case Spec_Static + +#define GEN_PARSER_OPERATOR_ALLOWED_SPECIFIERS_CASES \ +case Spec_Const: \ +case Spec_Constexpr: \ +case Spec_ForceInline: \ +case Spec_Inline: \ +case Spec_NeverInline: \ +case Spec_Static + +#define GEN_PARSER_TEMPLATE_ALLOWED_SPECIFIERS_CASES \ +case Spec_Const: \ +case Spec_Constexpr: \ +case Spec_Constinit: \ +case Spec_External_Linkage: \ +case Spec_Global: \ +case Spec_Inline: \ +case Spec_ForceInline: \ +case Spec_Local_Persist: \ +case Spec_Mutable: \ +case Spec_Static: \ +case Spec_Thread_Local: \ +case Spec_Volatile + +#define GEN_PARSER_VARIABLE_ALLOWED_SPECIFIER_CASES \ +case Spec_Const: \ +case Spec_Constexpr: \ +case Spec_Constinit: \ +case Spec_External_Linkage: \ +case Spec_Global: \ +case Spec_Inline: \ +case Spec_Local_Persist: \ +case Spec_Mutable: \ +case Spec_Static: \ +case Spec_Thread_Local: \ +case Spec_Volatile + + + diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index 0cb1ac5..cb91e9a 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -6,6 +6,24 @@ #include "gen/especifier.hpp" #endif +enum MacroFlags : u32 +{ + // Can only be one of these at a time (required) + MF_Block_Start = bit(0), // Start of a "block" scope + MF_Block_End = bit(1), // End of a "block" scope + MF_Case_Statement = bit(2), // Used as a case statement (not utilized by the parser yet) + MF_Expression = bit(3), // Used as an expresssion (not utilized by the parser yet) + MF_Statement = bit(4), // Used a statement (will expect to be a lone macro) + MF_Expects_Body = bit(5), // Expects to consume a braced scope + MF_Typename = bit(6), // Behaves as a typename + + // Optional + MF_Functional = bit(7), + + MF_Null = 0, + MF_UnderlyingType = GEN_U32_MAX, +}; + enum TokFlags : u32 { TF_Operator = bit(0), diff --git a/base/gen.cpp b/base/gen.cpp index 3983dc3..0f629cf 100644 --- a/base/gen.cpp +++ b/base/gen.cpp @@ -32,6 +32,7 @@ GEN_NS_BEGIN #include "components/interface.cpp" #include "components/interface.upfront.cpp" #include "components/lexer.cpp" +#include "components/parser_case_macros.cpp" #include "components/parser.cpp" #include "components/interface.parsing.cpp" #include "components/interface.untyped.cpp" diff --git a/base/helpers/base_codegen.hpp b/base/helpers/base_codegen.hpp index c208f33..8b69c89 100644 --- a/base/helpers/base_codegen.hpp +++ b/base/helpers/base_codegen.hpp @@ -9,10 +9,7 @@ using namespace gen; CodeBody gen_ecode( char const* path, bool use_c_definition = false ) { - FixedArena_32KB scratch; fixed_arena_init(& scratch); - AllocatorInfo scratch_info = fixed_arena_allocator_info(& scratch); - - CSV_Columns2 csv_enum = parse_csv_two_columns( scratch_info, path ); + CSV_Columns2 csv_enum = parse_csv_two_columns( _ctx->Allocator_Temp, path ); StrBuilder enum_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(1) ); StrBuilder to_c_str_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(1) ); StrBuilder to_keyword_c_str_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(1) ); @@ -93,10 +90,7 @@ CodeBody gen_ecode( char const* path, bool use_c_definition = false ) CodeBody gen_eoperator( char const* path, bool use_c_definition = false ) { - FixedArena_16KB scratch; fixed_arena_init(& scratch); - AllocatorInfo scratch_info = fixed_arena_allocator_info(& scratch); - - CSV_Columns2 csv_enum = parse_csv_two_columns( scratch_info, path ); + CSV_Columns2 csv_enum = parse_csv_two_columns( _ctx->Allocator_Temp, path ); StrBuilder enum_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, 32 ); StrBuilder to_c_str_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, 32 ); @@ -178,12 +172,9 @@ CodeBody gen_eoperator( char const* path, bool use_c_definition = false ) CodeBody gen_especifier( char const* path, bool use_c_definition = false ) { - FixedArena_16KB scratch; fixed_arena_init(& scratch); - AllocatorInfo scratch_info = fixed_arena_allocator_info(& scratch); - - CSV_Columns2 csv_enum = parse_csv_two_columns( scratch_info, path ); - StrBuilder enum_entries = strbuilder_make_reserve( scratch_info, kilobytes(1) ); - StrBuilder to_c_str_entries = strbuilder_make_reserve( scratch_info, kilobytes(1) ); + CSV_Columns2 csv_enum = parse_csv_two_columns( _ctx->Allocator_Temp, path ); + StrBuilder enum_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(1) ); + StrBuilder to_c_str_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(1) ); for (usize idx = 0; idx < array_num(csv_enum.Col_1); idx++) { @@ -317,29 +308,24 @@ CodeBody gen_especifier( char const* path, bool use_c_definition = false ) CodeBody gen_etoktype( char const* etok_path, char const* attr_path, bool use_c_definition = false ) { - FixedArena_64KB scratch; fixed_arena_init(& scratch); - AllocatorInfo scratch_info = fixed_arena_allocator_info(& scratch); - - FileContents enum_content = file_read_contents( scratch_info, file_zero_terminate, etok_path ); - + FileContents enum_content = file_read_contents( _ctx->Allocator_Temp, file_zero_terminate, etok_path ); CSV_Object csv_enum_nodes; - csv_parse( &csv_enum_nodes, rcast(char*, enum_content.data), scratch_info, false ); - - FileContents attrib_content = file_read_contents( scratch_info, file_zero_terminate, attr_path ); + csv_parse( &csv_enum_nodes, rcast(char*, enum_content.data), _ctx->Allocator_Temp, false ); + FileContents attrib_content = file_read_contents( _ctx->Allocator_Temp, file_zero_terminate, attr_path ); CSV_Object csv_attr_nodes; - csv_parse( &csv_attr_nodes, rcast(char*, attrib_content.data), scratch_info, false ); + csv_parse( &csv_attr_nodes, rcast(char*, attrib_content.data), _ctx->Allocator_Temp, false ); Array enum_strs = csv_enum_nodes.nodes[0].nodes; Array enum_c_str_strs = csv_enum_nodes.nodes[1].nodes; Array attribute_strs = csv_attr_nodes.nodes[0].nodes; Array attribute_c_str_strs = csv_attr_nodes.nodes[1].nodes; - StrBuilder enum_entries = strbuilder_make_reserve( scratch_info, kilobytes(2) ); - StrBuilder to_c_str_entries = strbuilder_make_reserve( scratch_info, kilobytes(4) ); - StrBuilder attribute_entries = strbuilder_make_reserve( scratch_info, kilobytes(2) ); - StrBuilder to_c_str_attributes = strbuilder_make_reserve( scratch_info, kilobytes(4) ); - StrBuilder attribute_define_entries = strbuilder_make_reserve( scratch_info, kilobytes(4) ); + StrBuilder enum_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(2) ); + StrBuilder to_c_str_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(4) ); + StrBuilder attribute_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(2) ); + StrBuilder to_c_str_attributes = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(4) ); + StrBuilder attribute_define_entries = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(4) ); for (usize idx = 0; idx < array_num(enum_strs); idx++) { diff --git a/base/helpers/undef.macros.h b/base/helpers/undef.macros.h index 4e12938..e31d308 100644 --- a/base/helpers/undef.macros.h +++ b/base/helpers/undef.macros.h @@ -173,12 +173,12 @@ #undef GEN_DEFINE_ATTRIBUTE_TOKENS -#undef GEN_AST_BODY_CLASS_UNALLOWED_TYPES -#undef GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES -#undef GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES -#undef GEN_AST_BODY_EXPORT_UNALLOWED_TYPES -#undef GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES -#undef GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES +#undef GEN_AST_BODY_CLASS_UNALLOWED_TYPES_CASES +#undef GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES_CASES +#undef GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES_CASES +#undef GEN_AST_BODY_EXPORT_UNALLOWED_TYPES_CASES +#undef GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES_CASES +#undef GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES_CASES #undef GEN_GLOBAL_BUCKET_SIZE #undef GEN_CODEPOOL_NUM_BLOCKS diff --git a/gen_c_library/c_library.cpp b/gen_c_library/c_library.cpp index 5ece289..8c7e2b3 100644 --- a/gen_c_library/c_library.cpp +++ b/gen_c_library/c_library.cpp @@ -1247,6 +1247,7 @@ R"(#define ( code ) _Generic( (code), \ Code src_interface = scan_file( path_base "components/interface.cpp" ); Code src_parsing_interface = scan_file( path_base "components/interface.parsing.cpp" ); Code src_untyped = scan_file( path_base "components/interface.untyped.cpp" ); + Code src_parser_case_macros = scan_file( path_base "components/parser_case_macros.cpp" ); CodeBody parsed_src_ast = parse_file( path_base "components/ast.cpp" ); CodeBody src_ast = def_body(CT_Global_Body); @@ -1500,6 +1501,7 @@ R"(#define ( code ) _Generic( (code), \ Code r_src_ast_case_macros = refactor(src_ast_case_macros); Code r_src_ast = refactor(src_ast); Code r_src_code_serialization = refactor(src_code_serialization); + Code r_src_parser_case_macros = refactor(src_parser_case_macros); Code r_src_interface = refactor(src_interface); Code r_src_upfront = refactor_and_format(src_upfront); @@ -1649,6 +1651,7 @@ R"(#define ( code ) _Generic( (code), \ header.print( fmt_newline); header.print( rf_array_code_typename ); header.print( fmt_newline); + header.print( r_src_parser_case_macros ); header.print( rf_src_parser ); header.print( r_src_parsing ); header.print_fmt( "\n#pragma endregion Parsing\n" ); diff --git a/gen_segmented/segmented.cpp b/gen_segmented/segmented.cpp index 387e263..a613767 100644 --- a/gen_segmented/segmented.cpp +++ b/gen_segmented/segmented.cpp @@ -193,6 +193,7 @@ int gen_main() Code interface = scan_file( path_base "components/interface.cpp" ); Code upfront = scan_file( path_base "components/interface.upfront.cpp" ); Code lexer = scan_file( path_base "components/lexer.cpp" ); + Code parser_case_macros = scan_file( path_base "components/parser_case_macros.cpp" ); Code parser = scan_file( path_base "components/parser.cpp" ); Code parsing_interface = scan_file( path_base "components/interface.parsing.cpp" ); Code untyped = scan_file( path_base "components/interface.untyped.cpp" ); @@ -217,6 +218,7 @@ int gen_main() builder_print( src, upfront ); builder_print_fmt( src, "\n#pragma region Parsing\n\n" ); builder_print( src, lexer ); + builder_print( src, parser_case_macros ); builder_print( src, parser ); builder_print( src, parsing_interface ); builder_print_fmt( src, "\n#pragma endregion Parsing\n\n" ); diff --git a/gen_singleheader/singleheader.cpp b/gen_singleheader/singleheader.cpp index b331a84..f3b4327 100644 --- a/gen_singleheader/singleheader.cpp +++ b/gen_singleheader/singleheader.cpp @@ -210,16 +210,17 @@ int gen_main() header.print_fmt( roll_own_dependencies_guard_end ); } - Code static_data = scan_file( path_base "components/static_data.cpp" ); - Code ast_case_macros = scan_file( path_base "components/ast_case_macros.cpp" ); - Code ast = scan_file( path_base "components/ast.cpp" ); - Code code = scan_file( path_base "components/code_serialization.cpp" ); - Code interface = scan_file( path_base "components/interface.cpp" ); - Code upfront = scan_file( path_base "components/interface.upfront.cpp" ); - Code lexer = scan_file( path_base "components/lexer.cpp" ); - Code parser = scan_file( path_base "components/parser.cpp" ); - Code parsing_interface = scan_file( path_base "components/interface.parsing.cpp" ); - Code untyped = scan_file( path_base "components/interface.untyped.cpp" ); + Code static_data = scan_file( path_base "components/static_data.cpp" ); + Code ast_case_macros = scan_file( path_base "components/ast_case_macros.cpp" ); + Code ast = scan_file( path_base "components/ast.cpp" ); + Code code = scan_file( path_base "components/code_serialization.cpp" ); + Code interface = scan_file( path_base "components/interface.cpp" ); + Code upfront = scan_file( path_base "components/interface.upfront.cpp" ); + Code lexer = scan_file( path_base "components/lexer.cpp" ); + Code parser_case_macros = scan_file( path_base "components/parser_case_macros.cpp" ); + Code parser = scan_file( path_base "components/parser.cpp" ); + Code parsing_interface = scan_file( path_base "components/interface.parsing.cpp" ); + Code untyped = scan_file( path_base "components/interface.untyped.cpp" ); header.print_fmt( "\nGEN_NS_BEGIN\n"); header.print( static_data ); @@ -236,6 +237,7 @@ int gen_main() header.print( upfront ); header.print_fmt( "\n#pragma region Parsing\n\n" ); header.print( lexer ); + header.print( parser_case_macros ); header.print( parser ); header.print( parsing_interface ); header.print_fmt( "\n#pragma endregion Parsing\n" ); diff --git a/gen_unreal_engine/components/parser_case_macros.cpp b/gen_unreal_engine/components/parser_case_macros.cpp new file mode 100644 index 0000000..e0b48af --- /dev/null +++ b/gen_unreal_engine/components/parser_case_macros.cpp @@ -0,0 +1,108 @@ +// These macros are used in the swtich cases within parser.cpp + +#define GEN_PARSER_CLASS_STRUCT_BODY_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES \ +case Tok_Spec_Consteval: \ +case Tok_Spec_Constexpr: \ +case Tok_Spec_Constinit: \ +case Tok_Spec_Explicit: \ +case Tok_Spec_ForceInline: \ +case Tok_Spec_ForceInline_Debuggable: \ +case Tok_Spec_Inline: \ +case Tok_Spec_Mutable: \ +case Tok_Spec_NeverInline: \ +case Tok_Spec_Static: \ +case Tok_Spec_Volatile: \ +case Tok_Spec_Virtual + +#define GEN_PARSER_CLASS_STRUCT_BODY_ALLOWED_MEMBER_SPECIFIERS_CASES \ +case Spec_Constexpr: \ +case Spec_Constinit: \ +case Spec_Explicit: \ +case Spec_Inline: \ +case Spec_ForceInline: \ +case Spec_ForceInline_Debuggable: \ +case Spec_Mutable: \ +case Spec_NeverInline: \ +case Spec_Static: \ +case Spec_Volatile: \ +case Spec_Virtual + +#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES \ +case Tok_Spec_Consteval: \ +case Tok_Spec_Constexpr: \ +case Tok_Spec_Constinit: \ +case Tok_Spec_Extern: \ +case Tok_Spec_ForceInline: \ +case Tok_Spec_ForceInline_Debuggable: \ +case Tok_Spec_Global: \ +case Tok_Spec_Inline: \ +case Tok_Spec_Internal_Linkage: \ +case Tok_Spec_NeverInline: \ +case Tok_Spec_Static + +#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIERS_CASES \ +case Spec_Constexpr: \ +case Spec_Constinit: \ +case Spec_ForceInline: \ +case Spec_ForceInline_Debuggable: \ +case Spec_Global: \ +case Spec_External_Linkage: \ +case Spec_Internal_Linkage: \ +case Spec_Inline: \ +case Spec_Mutable: \ +case Spec_NeverInline: \ +case Spec_Static: \ +case Spec_Volatile + +#define GEN_PARSER_FRIEND_ALLOWED_SPECIFIERS_CASES \ +case Spec_Const: \ +case Spec_Inline: \ +case Spec_ForceInline + +#define GEN_PARSER_FUNCTION_ALLOWED_SPECIFIERS_CASES \ +case Spec_Const: \ +case Spec_Consteval: \ +case Spec_Constexpr: \ +case Spec_External_Linkage: \ +case Spec_Internal_Linkage: \ +case Spec_ForceInline: \ +case Spec_ForceInline_Debuggable: \ +case Spec_Inline: \ +case Spec_NeverInline: \ +case Spec_Static + +#define GEN_PARSER_OPERATOR_ALLOWED_SPECIFIERS_CASES \ +case Spec_Const: \ +case Spec_Constexpr: \ +case Spec_ForceInline: \ +case Spec_Inline: \ +case Spec_NeverInline: \ +case Spec_Static + +#define GEN_PARSER_TEMPLATE_ALLOWED_SPECIFIERS_CASES \ +case Spec_Const: \ +case Spec_Constexpr: \ +case Spec_Constinit: \ +case Spec_External_Linkage: \ +case Spec_Global: \ +case Spec_Inline: \ +case Spec_ForceInline: \ +case Spec_ForceInline_Debuggable: \ +case Spec_Local_Persist: \ +case Spec_Mutable: \ +case Spec_Static: \ +case Spec_Thread_Local: \ +case Spec_Volatile + +#define GEN_PARSER_VARIABLE_ALLOWED_SPECIFIER_CASES \ +case Spec_Const: \ +case Spec_Constexpr: \ +case Spec_Constinit: \ +case Spec_External_Linkage: \ +case Spec_Global: \ +case Spec_Inline: \ +case Spec_Local_Persist: \ +case Spec_Mutable: \ +case Spec_Static: \ +case Spec_Thread_Local: \ +case Spec_Volatile diff --git a/gen_unreal_engine/enums/ESpecifier.csv b/gen_unreal_engine/enums/ESpecifier.csv new file mode 100644 index 0000000..aa3d550 --- /dev/null +++ b/gen_unreal_engine/enums/ESpecifier.csv @@ -0,0 +1,27 @@ +Invalid, INVALID +Consteval, consteval +Constexpr, constexpr +Constinit, constinit +Explicit, explicit +External_Linkage, extern +ForceInline, FORCEINLINE +ForceInline_Debuggable, FORCEINLINE_DEBUGGABLE +Global, global +Inline, inline +Internal_Linkage, internal +Local_Persist, local_persist +Mutable, mutable +NeverInline, neverinline +Ptr, * +Ref, & +Register, register +RValue, && +Static, static +Thread_Local, thread_local +Virtual, virtual +Const, const +Final, final +NoExceptions, noexcept +Override, override +Pure, = 0 +Volatile, volatile diff --git a/gen_unreal_engine/enums/ETokType.csv b/gen_unreal_engine/enums/ETokType.csv new file mode 100644 index 0000000..d57ad95 --- /dev/null +++ b/gen_unreal_engine/enums/ETokType.csv @@ -0,0 +1,96 @@ +Invalid, "__invalid__" +Access_Private, "private" +Access_Protected, "protected" +Access_Public, "public" +Access_MemberSymbol, "." +Access_StaticSymbol, "::" +Ampersand, "&" +Ampersand_DBL, "&&" +Assign_Classifer, ":" +Attribute_Open, "[[" +Attribute_Close, "]]" +BraceCurly_Open, "{" +BraceCurly_Close, "}" +BraceSquare_Open, "[" +BraceSquare_Close, "]" +Capture_Start, "(" +Capture_End, ")" +Comment, "__comment__" +Comment_End, "__comment_end__" +Comment_Start, "__comment_start__" +Char, "__character__" +Comma, "," +Decl_Class, "class" +Decl_GNU_Attribute, "__attribute__" +Decl_MSVC_Attribute, "__declspec" +Decl_Enum, "enum" +Decl_Extern_Linkage, "extern" +Decl_Friend, "friend" +Decl_Module, "module" +Decl_Namespace, "namespace" +Decl_Operator, "operator" +Decl_Struct, "struct" +Decl_Template, "template" +Decl_Typedef, "typedef" +Decl_Using, "using" +Decl_Union, "union" +Identifier, "__identifier__" +Module_Import, "import" +Module_Export, "export" +NewLine, "__new_line__" +Number, "__number__" +Operator, "__operator__" +Preprocess_Hash, "#" +Preprocess_Define, "define" +Preprocess_If, "if" +Preprocess_IfDef, "ifdef" +Preprocess_IfNotDef, "ifndef" +Preprocess_ElIf, "elif" +Preprocess_Else, "else" +Preprocess_EndIf, "endif" +Preprocess_Include, "include" +Preprocess_Pragma, "pragma" +Preprocess_Content, "__macro_content__" +Preprocess_Macro, "__macro__" +Preprocess_Unsupported, "__unsupported__" +Spec_Alignas, "alignas" +Spec_Const, "const" +Spec_Consteval, "consteval" +Spec_Constexpr, "constexpr" +Spec_Constinit, "constinit" +Spec_Explicit, "explicit" +Spec_Extern, "extern" +Spec_Final, "final" +Spec_ForceInline, "FORCEINLINE" +Spec_ForceInline_Debuggable, "FORCEINLINE_DEBUGGABLE" +Spec_Global, "global" +Spec_Inline, "inline" +Spec_Internal_Linkage, "internal" +Spec_LocalPersist, "local_persist" +Spec_Mutable, "mutable" +Spec_NeverInline, "neverinline" +Spec_Override, "override" +Spec_Static, "static" +Spec_ThreadLocal, "thread_local" +Spec_Volatile, "volatile" +Spec_Virtual, "virtual" +Star, "*" +Statement_End, ";" +StaticAssert, "static_assert" +String, "__string__" +Type_Typename, "typename" +Type_Unsigned, "unsigned" +Type_Signed, "signed" +Type_Short, "short" +Type_Long, "long" +Type_bool, "bool" +Type_char, "char" +Type_int, "int" +Type_double, "double" +Type_MS_int8, "__int8" +Type_MS_int16, "__int16" +Type_MS_int32, "__int32" +Type_MS_int64, "__int64" +Type_MS_W64, "_W64" +Varadic_Argument, "..." +__Attributes_Start, "__attrib_start__" diff --git a/gen_unreal_engine/unreal.cpp b/gen_unreal_engine/unreal.cpp index e9b676a..9640340 100644 --- a/gen_unreal_engine/unreal.cpp +++ b/gen_unreal_engine/unreal.cpp @@ -186,13 +186,13 @@ int gen_main() CodeBody ecode = gen_ecode ( path_base "enums/ECodeTypes.csv" ); CodeBody eoperator = gen_eoperator ( path_base "enums/EOperator.csv" ); - CodeBody especifier = gen_especifier( path_base "enums/ESpecifier.csv" ); + CodeBody especifier = gen_especifier( "enums/ESpecifier.csv" ); CodeBody ast_inlines = gen_ast_inlines(); // Note(Ed): The Attribute tokens need to be expanded and regenerated on a per-project/installation of this library for a specific codebase of Unreal. // We can support an arbitrary set of modules or plugin apis for parsing // but its up to the user to define them all (This will just provide whats I've used up till now). - CodeBody etoktype = gen_etoktype( path_base "enums/ETokType.csv", "enums/AttributeTokens.csv" ); + CodeBody etoktype = gen_etoktype( "enums/ETokType.csv", "enums/AttributeTokens.csv" ); Builder header = Builder::open( "gen/gen.hpp" ); @@ -247,6 +247,7 @@ int gen_main() Code interface = scan_file( path_base "components/interface.cpp" ); Code upfront = scan_file( path_base "components/interface.upfront.cpp" ); Code lexer = scan_file( path_base "components/lexer.cpp" ); + Code parser_case_macros = scan_file( "components/parser_case_macros.cpp" ); Code parser = scan_file( path_base "components/parser.cpp" ); Code parsing_interface = scan_file( path_base "components/interface.parsing.cpp" ); Code untyped = scan_file( path_base "components/interface.untyped.cpp" ); @@ -273,6 +274,7 @@ int gen_main() src.print( upfront ); src.print_fmt( "\n#pragma region Parsing\n\n" ); src.print( lexer ); + src.print( parser_case_macros ); src.print( parser ); src.print( parsing_interface ); src.print( untyped );