From 956ab731309e540740f5c7c6ed950d22678945c1 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 14 Dec 2024 23:10:23 -0500 Subject: [PATCH] Unreal variant generates, doing c_library corrections next --- base/components/code_types.hpp | 14 +++--- base/components/inlines.hpp | 9 ++++ base/components/interface.cpp | 38 ++++++++++++++- base/components/interface.hpp | 4 ++ base/components/lexer.cpp | 72 ++++++++++++++++++++++++---- base/components/parser.cpp | 16 ++++--- base/components/parser_types.hpp | 16 ++++--- base/dependencies/macros.hpp | 4 +- gen_unreal_engine/enums/ETokType.csv | 4 +- gen_unreal_engine/unreal.cpp | 39 +++++++++++++++ 10 files changed, 183 insertions(+), 33 deletions(-) diff --git a/base/components/code_types.hpp b/base/components/code_types.hpp index d1885ff..26d5316 100644 --- a/base/components/code_types.hpp +++ b/base/components/code_types.hpp @@ -202,7 +202,7 @@ struct CodeParams { #if ! GEN_C_LIKE_CPP Using_Code( CodeParams ); - forceinline void append( CodeParams other ) { return params_append(* this, other) } + forceinline void append( CodeParams other ) { return params_append(* this, other); } forceinline CodeParams get( s32 idx ) { return params_get( * this, idx); } forceinline bool has_entries() { return params_has_entries(* this); } forceinline StrBuilder to_strbuilder() { return params_to_strbuilder(* this); } @@ -225,8 +225,8 @@ struct CodeDefineParams { #if ! GEN_C_LIKE_CPP Using_Code( CodeDefineParams ); - forceinline void append( CodeDefineParams other ) { return params_append( cast(CodeParams, * this), other) } - forceinline CodeDefineParams get( s32 idx ) { return params_get( cast(CodeParams, * this), idx); } + forceinline void append( CodeDefineParams other ) { return params_append( cast(CodeParams, * this), cast(CodeParams, other)); } + forceinline CodeDefineParams get( s32 idx ) { return (CodeDefineParams) (Code) params_get( cast(CodeParams, * this), idx); } forceinline bool has_entries() { return params_has_entries( cast(CodeParams, * this)); } forceinline StrBuilder to_strbuilder() { return define_params_to_strbuilder(* this); } forceinline void to_strbuilder( StrBuilder& result ) { return define_params_to_strbuilder_ref(* this, & result); } @@ -240,7 +240,7 @@ struct CodeDefineParams GEN_ASSERT(ast); return ast; } - forceinline CodeDefineParams& operator++() { return (CodeDefineParams) (Code) cast(CodeParams, * this).operator ++(); }; + forceinline CodeDefineParams& operator++(); AST_DefineParams* ast; }; @@ -1058,9 +1058,9 @@ forceinline StrBuilder to_strbuilder ( CodeClass self ) forceinline void to_strbuilder_def( CodeClass self, StrBuilder& result ) { return class_to_strbuilder_def(self, & result); } forceinline void to_strbuilder_fwd( CodeClass self, StrBuilder& result ) { return class_to_strbuilder_fwd(self, & result); } -forceinline void append (CodeDefineParams appendee, CodeDefineParams other ) { return params_append(cast(CodeParam, appendee), other); } -forceinline CodeDefineParams get (CodeDefineParams params, s32 idx) { return params_get(cast(CodeParam, params), idx); } -forceinline bool has_entries (CodeDefineParams params ) { return params_has_entries(cast(CodeParam, params)); } +forceinline void append (CodeDefineParams appendee, CodeDefineParams other ) { params_append(cast(CodeParams, appendee), cast(CodeParams, other)); } +forceinline CodeDefineParams get (CodeDefineParams params, s32 idx) { return (CodeDefineParams) (Code) params_get(cast(CodeParams, params), idx); } +forceinline bool has_entries (CodeDefineParams params ) { return params_has_entries(cast(CodeParams, params)); } forceinline StrBuilder to_strbuilder(CodeDefineParams params ) { return define_params_to_strbuilder(params); } forceinline void to_strbuilder(CodeDefineParams params, StrBuilder& result ) { return define_params_to_strbuilder_ref(params, & result); } diff --git a/base/components/inlines.hpp b/base/components/inlines.hpp index e6b4b09..b557a4f 100644 --- a/base/components/inlines.hpp +++ b/base/components/inlines.hpp @@ -265,6 +265,15 @@ forceinline bool define_params_has_entries(CodeDefineParams self) CodeDefineParams begin_CodeDefineParams(CodeDefineParams params) { return (CodeDefineParams) (Code) begin_CodeParams( cast(CodeParams, (Code)params)); } CodeDefineParams end_CodeDefineParams (CodeDefineParams params) { return (CodeDefineParams) (Code) end_CodeParams ( cast(CodeParams, (Code)params)); } CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter) { return (CodeDefineParams) (Code) next_CodeParams ( cast(CodeParams, (Code)params), cast(CodeParams, (Code)entry_iter)); } + +#if GEN_COMPILER_CPP +forceinline +CodeDefineParams& CodeDefineParams::operator ++() +{ + * this = ast->Next; + return * this; +} +#endif #pragma endregion CodeDefineParams #pragma region CodeSpecifiers diff --git a/base/components/interface.cpp b/base/components/interface.cpp index 69d6482..a5a21ec 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -207,7 +207,7 @@ void define_constants() code_set_global(cast(Code, spec_local_persist)); if (enum_underlying_macro.Name.Len == 0) { - enum_underlying_macro.Name = txt("enum_underlying("); + enum_underlying_macro.Name = txt("enum_underlying"); enum_underlying_macro.Type = MT_Expression; enum_underlying_macro.Flags = MF_Functional; } @@ -470,6 +470,42 @@ PreprocessorMacro* lookup_preprocess_macro( Str name ) { } void register_preprocess_macro( PreprocessorMacro macro ) { + GEN_ASSERT_NOT_NULL(macro.Name.Ptr); + GEN_ASSERT(macro.Name.Len > 0); u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); hashtable_set( _ctx->PreprocessorMacros, key, macro ); } + +void register_preprocess_macros( s32 num, ... ) +{ + GEN_ASSERT(num > 0); + va_list va; + va_start(va, num); + do + { + PreprocessorMacro macro = va_arg(va, PreprocessorMacro); + GEN_ASSERT_NOT_NULL(macro.Name.Ptr); + GEN_ASSERT(macro.Name.Len > 0); + + u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); + hashtable_set( _ctx->PreprocessorMacros, key, macro ); + } + while (num--, num > 0); + va_end(va); +} + +void register_preprocess_macros( s32 num, PreprocessorMacro* macros ) +{ + GEN_ASSERT(num > 0); + do + { + PreprocessorMacro macro = * macros; + GEN_ASSERT_NOT_NULL(macro.Name.Ptr); + GEN_ASSERT(macro.Name.Len > 0); + + u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); + hashtable_set( _ctx->PreprocessorMacros, key, macro ); + ++ macros; + } + while (num--, num > 0); +} diff --git a/base/components/interface.hpp b/base/components/interface.hpp index f3e13fd..d18937e 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -114,6 +114,10 @@ GEN_API PreprocessorMacro* lookup_preprocess_macro( Str Name ); // Macros are tracked by name so if the name already exists the entry will be overwritten. GEN_API void register_preprocess_macro( PreprocessorMacro macro ); +// Ease of use batch registration +GEN_API void register_preprocess_macros( s32 num, ... ); +GEN_API void register_preprocess_macros( s32 num, PreprocessorMacro* macros ); + // Used internally to retrive or make string allocations. // Strings are stored in a series of string arenas of fixed size (SizePer_StringArena) GEN_API StrCached cache_str( Str str ); diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 97f3c42..5a0a807 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -118,25 +118,34 @@ s32 lex_preprocessor_define( LexContext* ctx ) Token name = { { ctx->scanner, 1 }, Tok_Identifier, ctx->line, ctx->column, TF_Preprocess }; move_forward(); - PreprocessorMacro macro = { name.Text, MT_Statement, (MacroFlags)0 }; - PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); - if ( registered_macro == nullptr ) { - log_fmt("Warning: %S is was not registered before the lexer processed its #define directive, it will be registered as a statement macro" - , name.Text - ); - } while ( ctx->left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) { move_forward(); name.Text.Len++; } + + Specifier spec = str_to_specifier( name.Text ); + TokType attrib = str_to_toktype( name.Text ); + b32 not_specifier = spec == Spec_Invalid; + b32 not_attribute = attrib <= Tok___Attributes_Start; + + PreprocessorMacro macro = { name.Text, MT_Statement, (MacroFlags)0 }; + PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); + + if ( registered_macro == nullptr && not_specifier && not_attribute ) { + log_fmt("Warning: '%S' was not registered before the lexer processed its #define directive, it will be registered as a statement macro\n" + , name.Text + ); + GEN_DEBUG_TRAP(); + } array_append( _ctx->Lexer_Tokens, name ); if ( ctx->left && (* ctx->scanner) == '(' ) { if (registered_macro && ! macro_is_functional(* registered_macro)) { - log_fmt("Warning: %S registered macro is not flagged as functional yet the definition detects opening parenthesis '(' for arguments" + log_fmt("Warning: %S registered macro is not flagged as functional yet the definition detects opening parenthesis '(' for arguments\n" , name.Text ); + GEN_DEBUG_TRAP(); } else { macro.Flags |= MF_Functional; @@ -151,7 +160,39 @@ s32 lex_preprocessor_define( LexContext* ctx ) while( ctx->left && * ctx->scanner != ')') { skip_whitespace(); - if ( char_is_alpha( (* ctx->scanner) ) || (* ctx->scanner) == '_' ) + + Str possible_varadic = { ctx->scanner, 3 }; + if ( ctx->left > 3 && str_are_equal( txt("..."), possible_varadic ) ) { + Token parameter = { { ctx->scanner, 3 }, Tok_Preprocess_Define_Param, ctx->line, ctx->column, TF_Preprocess }; + move_forward(); + move_forward(); + move_forward(); + + array_append(_ctx->Lexer_Tokens, parameter); + skip_whitespace(); + last_parameter = parameter; + + while ( (* ctx->scanner) == '\\' ) { + move_forward(); + skip_whitespace(); + } + if (* ctx->scanner != ')' ) + { + log_failure("lex_preprocessor_define(%d, %d): Expected a ')' after '...' (varaidc macro param) %S\n" + , ctx->line + , ctx->column + , name.Text + ); + return Lex_ReturnNull; + } + break; + } + else if ( (* ctx->scanner) == '\\' ) { + move_forward(); + skip_whitespace(); + continue; + } + else if ( char_is_alpha( (* ctx->scanner) ) || (* ctx->scanner) == '_' ) { Token parameter = { { ctx->scanner, 1 }, Tok_Preprocess_Define_Param, ctx->line, ctx->column, TF_Preprocess }; move_forward(); @@ -173,6 +214,10 @@ s32 lex_preprocessor_define( LexContext* ctx ) ); return Lex_ReturnNull; } + + if (* ctx->scanner == ')' ) + break; + // There should be a comma if ( * ctx->scanner != ',' ) { log_failure("lex_preprocessor_define(%d, %d): Expected a comma after parameter %S for %S\n" @@ -201,6 +246,15 @@ s32 lex_preprocessor_define( LexContext* ctx ) array_append(_ctx->Lexer_Tokens, closing_paren); move_forward(); } + else if ( registered_macro && macro_is_functional( * registered_macro) ) { + if (registered_macro && ! macro_is_functional(* registered_macro)) { + log_fmt("Warning: %S registered macro is flagged as functional yet the definition detects no opening parenthesis '(' for arguments\n" + , name.Text + ); + GEN_DEBUG_TRAP(); + } + } + if ( registered_macro == nullptr ) { register_preprocess_macro(macro); } diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 46c71c2..749661c 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -1318,10 +1318,13 @@ CodeDefine parse_define() define->Params = params; eat( Tok_Preprocess_Define_Param ); - // #define ( ) + // #define ( } while( left && currtok.Type != Tok_Capture_End ) { + eat( Tok_Comma ); + // #define ( , + CodeDefineParams next_param = (CodeDefineParams) make_code(); next_param->Type = CT_Parameters_Define; next_param->Name = currtok.Text; @@ -1344,7 +1347,7 @@ CodeDefine parse_define() if ( currtok.Text.Len == 0 ) { - define->Body = untyped_str( tok_to_str(currtok) ); + define->Body = untyped_str( txt("\n") ); eat( Tok_Preprocess_Content ); // #define ( ) @@ -1882,7 +1885,6 @@ CodeBody parse_global_nspace( CodeType which ) } } - Member_Resolved_To_Lone_Macro: if ( member == Code_Invalid ) { log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(currtok_noskip), parser_to_strbuilder(_ctx->parser) ); @@ -2930,9 +2932,9 @@ Code parse_simple_preprocess( TokType which ) eat( which ); // - PreprocessorMacro macro = * lookup_preprocess_macro( full_macro.Text ); + PreprocessorMacro* macro = lookup_preprocess_macro( full_macro.Text ); - if ( macro_expects_body(macro) && peektok.Type == Tok_BraceCurly_Open ) + if ( macro && macro_expects_body(* macro) && peektok.Type == Tok_BraceCurly_Open ) { // Eat the block scope right after the macro. Were assuming the macro defines a function definition's signature eat( Tok_BraceCurly_Open ); @@ -2955,7 +2957,7 @@ Code parse_simple_preprocess( TokType which ) // TODO(Ed): Review this? Str prev_proc = _ctx->parser.Scope->Prev->ProcName; - if ( macro.Type == MT_Typename && c_str_compare_len( prev_proc.Ptr, "parser_parse_typedef", prev_proc.Len ) != 0 ) + if ( macro->Type == MT_Typename && c_str_compare_len( prev_proc.Ptr, "parser_parse_typedef", prev_proc.Len ) != 0 ) { if ( check( Tok_Statement_End )) { @@ -2982,7 +2984,7 @@ Code parse_simple_preprocess( TokType which ) // Do nothing goto Leave_Scope_Early; } - else if (macro.Type == MT_Typename && str_contains(_ctx->parser.Scope->Prev->ProcName, txt("parser_parse_typedef"))) + else if (macro && macro->Type == MT_Typename && str_contains(_ctx->parser.Scope->Prev->ProcName, txt("parser_parse_typedef"))) { if ( peektok.Type == Tok_Statement_End ) { diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index f198aa9..ad15bf4 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -132,6 +132,8 @@ enum MacroType : u16 MT_Statement, // A macro is assumed to be a statement if not resolved. MT_Expression, MT_Typename, + MT_Attribute, // More of a note to the parser than anythign else (attributes should be defined in the user attribues def). + MT_Specifier, // More of a note to the parser than anythign else (specifiers should be defined in the user attribues def). MT_Block_Start, // Not Supported yet MT_Block_End, // Not Supported yet MT_Case_Statement, // Not Supported yet @@ -154,12 +156,14 @@ Str macrotype_to_str( MacroType type ) { local_persist Str lookup[] = { - { "Statement", sizeof("Statement") - 1 }, - { "Expression", sizeof("Expression") - 1 }, - { "Typename", sizeof("Typename") - 1 }, - { "Block_Start", sizeof("Block_Start") - 1 }, - { "Block_End", sizeof("Block_End") - 1 }, - { "Case_Statement", sizeof("Case_Statement") - 1 }, + { "Statement", sizeof("Statement") - 1 }, + { "Expression", sizeof("Expression") - 1 }, + { "Typename", sizeof("Typename") - 1 }, + { "Attribute(Macro)", sizeof("Attribute(Macro)") - 1 }, + { "Specifier(Macro)", sizeof("Specifier(Macro)") - 1 }, + { "Block_Start", sizeof("Block_Start") - 1 }, + { "Block_End", sizeof("Block_End") - 1 }, + { "Case_Statement", sizeof("Case_Statement") - 1 }, }; local_persist Str invalid = { "Invalid", sizeof("Invalid") }; diff --git a/base/dependencies/macros.hpp b/base/dependencies/macros.hpp index cbcf0f7..a8636fa 100644 --- a/base/dependencies/macros.hpp +++ b/base/dependencies/macros.hpp @@ -248,9 +248,9 @@ # if ! GEN_COMPILER_C # define typeof decltype # elif defined(_MSC_VER) -# define typeof(x) __typeof__(x) +# define typeof __typeof__ # elif defined(__GNUC__) || defined(__clang__) -# define typeof(x) __typeof__(x) +# define typeof __typeof__ # else # error "Compiler not supported" # endif diff --git a/gen_unreal_engine/enums/ETokType.csv b/gen_unreal_engine/enums/ETokType.csv index d57ad95..739deee 100644 --- a/gen_unreal_engine/enums/ETokType.csv +++ b/gen_unreal_engine/enums/ETokType.csv @@ -51,7 +51,9 @@ Preprocess_EndIf, "endif" Preprocess_Include, "include" Preprocess_Pragma, "pragma" Preprocess_Content, "__macro_content__" -Preprocess_Macro, "__macro__" +Preprocess_Macro_Expr, "__macro_expression__" +Preprocess_Macro_Stmt, "__macro_statment__" +Preprocess_Macro_Typename, "__macro_typename__" Preprocess_Unsupported, "__unsupported__" Spec_Alignas, "alignas" Spec_Const, "const" diff --git a/gen_unreal_engine/unreal.cpp b/gen_unreal_engine/unreal.cpp index 9640340..47d0314 100644 --- a/gen_unreal_engine/unreal.cpp +++ b/gen_unreal_engine/unreal.cpp @@ -64,6 +64,45 @@ int gen_main() Code ue_forceinline = code_str(FORCEINLINE); // Code + register_preprocess_macros( args( + (PreprocessorMacro { txt("bit"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("bitfield_is_set"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_C_LIKE_CPP"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("cast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("ccast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("rcast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("pcast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("scast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("stringize_va"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("stringize"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once_defer"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once_start"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("do_once_end"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("labeled_scope_start"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("labeled_scope_end"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("num_args_impl"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("num_args"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("count_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("clamp"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("is_between"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("size_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("min"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("max"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("offset_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("static_assert"), MT_Statement, MF_Functional }), + (PreprocessorMacro { txt("typeof"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN_API_C_BEGIN"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("GEN_API_C_END"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("nullptr"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN_REMOVE_PTR"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_PARAM_DEFAULT"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("struct_init"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("GEN_OPITMIZE_MAPPINGS_END"), MT_Statement, MF_Null }) + )); + // gen_dep.hpp { CodeBody macros = def_body( CT_Global_Body );