From 76257123da6ed59a8b7c7278bd1a0284da76c635 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 14 Dec 2024 14:02:16 -0500 Subject: [PATCH 01/15] WIP (Not compiling prob): Started to overhaul macro handling --- base/components/ast.hpp | 11 +- base/components/ast_types.hpp | 29 ++- base/components/code_types.hpp | 66 ++++-- base/components/inlines.hpp | 10 + base/components/interface.cpp | 16 +- base/components/interface.hpp | 18 +- base/components/lexer.cpp | 38 ++-- base/components/parser.cpp | 6 +- base/components/parser_case_macros.cpp | 4 +- base/components/parser_types.hpp | 60 +++--- base/enums/ECodeTypes.csv | 1 + base/enums/ETokType.csv | 192 +++++++++--------- gen_c_library/components/misc.hpp | 3 +- .../components/parser_case_macros.cpp | 4 +- 14 files changed, 282 insertions(+), 176 deletions(-) diff --git a/base/components/ast.hpp b/base/components/ast.hpp index ac39d08..007c52d 100644 --- a/base/components/ast.hpp +++ b/base/components/ast.hpp @@ -26,6 +26,7 @@ struct AST_Constructor; // struct AST_BaseClass; struct AST_Class; struct AST_Define; +struct AST_DefineParams; struct AST_Destructor; struct AST_Enum; struct AST_Exec; @@ -98,6 +99,7 @@ typedef AST_Comment* CodeComment; typedef AST_Class* CodeClass; typedef AST_Constructor* CodeConstructor; typedef AST_Define* CodeDefine; +typedef AST_DefineParams* CodeDefineParams; typedef AST_Destructor* CodeDestructor; typedef AST_Enum* CodeEnum; typedef AST_Exec* CodeExec; @@ -120,6 +122,7 @@ struct CodeComment; struct CodeClass; struct CodeConstructor; struct CodeDefine; +struct CodeDefineParams; struct CodeDestructor; struct CodeEnum; struct CodeExec; @@ -305,6 +308,7 @@ struct Code operator CodeClass() const; operator CodeConstructor() const; operator CodeDefine() const; + operator CodeDefineParams() const; operator CodeDestructor() const; operator CodeExec() const; operator CodeEnum() const; @@ -365,6 +369,7 @@ int AST_ArrSpecs_Cap = /* Simple AST POD with functionality to seralize into C++ syntax. + TODO(Ed): Eventually haven't a transparent AST like this will longer be viable once statements & expressions are in (most likely....) */ struct AST { @@ -372,7 +377,7 @@ struct AST struct { Code InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable - Code Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable + Code Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable // TODO(Ed): Parameters can have attributes Code Specs; // Destructor, Function, Operator, Typename, Variable union { Code InitializerList; // Constructor @@ -384,12 +389,12 @@ struct AST union { Code Macro; // Parameter Code BitfieldSize; // Variable (Class/Struct Data Member) - Code Params; // Constructor, Function, Operator, Template, Typename + Code Params; // Constructor, Define, Function, Operator, Template, Typename Code UnderlyingTypeMacro; // Enum }; union { Code ArrExpr; // Typename - Code Body; // Class, Constructor, Destructor, Enum, Friend, Function, Namespace, Struct, Union + Code Body; // Class, Constructor, Define, Destructor, Enum, Friend, Function, Namespace, Struct, Union Code Declaration; // Friend, Template Code Value; // Parameter, Variable }; diff --git a/base/components/ast_types.hpp b/base/components/ast_types.hpp index f11e72e..8899174 100644 --- a/base/components/ast_types.hpp +++ b/base/components/ast_types.hpp @@ -146,9 +146,15 @@ struct AST_Define { union { char _PAD_[ sizeof(Specifier) * AST_ArrSpecs_Cap + sizeof(AST*) ]; - StrCached Content; + struct + { + char _PAD_PROPERTIES_ [ sizeof(AST*) * 4 ]; + CodeDefineParams Params; + Code Body; // Should be completely serialized for now to a: StrCached Content. + char _PAD_PROPERTIES_ [ sizeof(AST*) * 1 ]; + }; }; - StrCached Name; + StrCached Name; Code Prev; Code Next; Token* Tok; @@ -158,6 +164,22 @@ struct AST_Define }; static_assert( sizeof(AST_Define) == sizeof(AST), "ERROR: AST_Define is not the same size as AST"); +struct AST_DefineParams +{ + union { + char _PAD_[ sizeof(Specifier) * AST_ArrSpecs_Cap + sizeof(AST*) ]; + }; + StrCached Name; + CodeDefineParams Last; + CodeDefineParams Next; + Token* Tok; + Code Parent; + CodeType Type; + char _PAD_UNUSED_[ sizeof(ModuleFlag) ]; + s32 NumEntries; +}; +static_assert( sizeof(AST_DefineParams) == sizeof(AST), "ERROR: AST_DefineParams is not the same size as AST"); + struct AST_Destructor { union { @@ -660,6 +682,7 @@ struct AST_Params char _PAD_[ sizeof(Specifier) * AST_ArrSpecs_Cap + sizeof(AST*) ]; struct { + // TODO(Ed): Support attributes for parameters (Some prefix macros can be converted to that...) char _PAD_PROPERTIES_2_[ sizeof(AST*) * 3 ]; CodeTypename ValueType; Code Macro; @@ -668,7 +691,7 @@ struct AST_Params // char _PAD_PROPERTIES_3_[sizeof( AST* )]; }; }; - StrCached Name; + StrCached Name; CodeParams Last; CodeParams Next; Token* Tok; diff --git a/base/components/code_types.hpp b/base/components/code_types.hpp index 414d8c7..2ab1941 100644 --- a/base/components/code_types.hpp +++ b/base/components/code_types.hpp @@ -32,7 +32,17 @@ GEN_API StrBuilder class_to_strbuilder ( CodeClass self ); GEN_API void class_to_strbuilder_def( CodeClass self, StrBuilder* result ); GEN_API void class_to_strbuilder_fwd( CodeClass self, StrBuilder* result ); -GEN_API void params_append (CodeParams params, CodeParams param ); +GEN_API void define_params_append (CodeDefineParams appendee, CodeDefineParams other ); +GEN_API CodeDefineParams define_params_get (CodeDefineParams params, s32 idx); +GEN_API bool define_params_has_entries (CodeDefineParams params ); +GEN_API StrBuilder define_params_to_strbuilder (CodeDefineParams params ); +GEN_API void define_params_to_strbuilder_ref(CodeDefineParams params, StrBuilder* result ); + +GEN_API CodeDefineParams begin_CodeDefineParams(CodeDefineParams params); +GEN_API CodeDefineParams end_CodeDefineParams (CodeDefineParams params); +GEN_API CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter); + +GEN_API void params_append (CodeParams appendee, CodeParams other ); GEN_API CodeParams params_get (CodeParams params, s32 idx); GEN_API bool params_has_entries (CodeParams params ); GEN_API StrBuilder params_to_strbuilder (CodeParams params ); @@ -192,12 +202,11 @@ struct CodeParams { #if ! GEN_C_LIKE_CPP Using_Code( CodeParams ); - forceinline void append( CodeParams other ); - forceinline CodeParams get( s32 idx ); - forceinline bool has_entries(); - forceinline StrBuilder to_strbuilder(); - forceinline void to_strbuilder( StrBuilder& result ); - + forceinline void append( CodeParams other ) { return params_append(* this, other) } + forceinline CodeParams get( s32 idx ) { return params_get( * this, idx); } + forceinline bool has_entries() { return params_has_entries(* this); } + forceinline StrBuilder to_strbuilder() { return params_to_strbuilder(* this); } + forceinline void to_strbuilder( StrBuilder& result ) { return params_to_strbuilder_ref(*this, & result); } #endif Using_CodeOps( CodeParams ); forceinline CodeParams begin() { return begin_CodeParams(* this); } @@ -212,6 +221,29 @@ struct CodeParams AST_Params* ast; }; +struct CodeDefineParams +{ +#if ! GEN_C_LIKE_CPP + Using_Code( CodeDefineParams ); + forceinline void append( CodeDefineParams other ) { return params_append( cast(CodeParams, * this), other) } + forceinline CodeDefineParams get( s32 idx ) { return params_get( cast(CodeParams, * this), idx); } + forceinline bool has_entries() { return params_has_entries( cast(CodeParams, * this)); } + forceinline StrBuilder to_strbuilder() { return define_params_to_strbuilder(* this); } + forceinline void to_strbuilder( StrBuilder& result ) { return define_params_to_strbuilder_ref(* this, & result); } +#endif + Using_CodeOps( CodeDefineParams ); + forceinline CodeDefineParams begin() { return begin_CodeParams( cast(CodeParams, * this)); } + forceinline CodeDefineParams end() { return end_CodeParams( cast(CodeParams, * this)); } + forceinline operator Code() { return { (AST*)ast }; } + forceinline CodeDefineParams operator *() { return * this; } // Required to support for-range iteration. + forceinline AST_DefineParams* operator->() { + GEN_ASSERT(ast); + return ast; + } + forceinline CodeDefineParams& operator++() { return cast(CodeParams, * this).operator ++() }; + AST_DefineParams* ast; +}; + struct CodeSpecifiers { #if ! GEN_C_LIKE_CPP @@ -941,6 +973,7 @@ struct InvalidCode_ImplictCaster operator CodeClass () const { return cast(CodeClass, Code_Invalid); } operator CodeConstructor () const { return cast(CodeConstructor, Code_Invalid); } operator CodeDefine () const { return cast(CodeDefine, Code_Invalid); } + operator CodeDefineParams () const { return cast(CodeDefineParams, Code_Invalid); } operator CodeDestructor () const { return cast(CodeDestructor, Code_Invalid); } operator CodeExec () const { return cast(CodeExec, Code_Invalid); } operator CodeEnum () const { return cast(CodeEnum, Code_Invalid); } @@ -974,6 +1007,7 @@ struct NullCode_ImplicitCaster operator CodeClass () const { return {nullptr}; } operator CodeConstructor () const { return {nullptr}; } operator CodeDefine () const { return {nullptr}; } + operator CodeDefineParams () const { return {nullptr}; } operator CodeDestructor () const { return {nullptr}; } operator CodeExec () const { return {nullptr}; } operator CodeEnum () const { return {nullptr}; } @@ -1024,17 +1058,23 @@ forceinline StrBuilder to_strbuilder ( CodeClass self ) forceinline void to_strbuilder_def( CodeClass self, StrBuilder& result ) { return class_to_strbuilder_def(self, & result); } forceinline void to_strbuilder_fwd( CodeClass self, StrBuilder& result ) { return class_to_strbuilder_fwd(self, & result); } -forceinline void append (CodeParams params, CodeParams param ) { return params_append(params, param); } -forceinline CodeParams get (CodeParams params, s32 idx) { return params_get(params, idx); } -forceinline bool has_entries (CodeParams params ) { return params_has_entries(params); } -forceinline StrBuilder to_strbuilder(CodeParams params ) { return params_to_strbuilder(params); } -forceinline void to_strbuilder(CodeParams params, StrBuilder& result ) { return params_to_strbuilder_ref(params, & result); } +forceinline void append (CodeDefineParams appendee, CodeDefineParams other ) { return params_append(cast(CodeParam, appendee), other); } +forceinline CodeDefineParams get (CodeDefineParams params, s32 idx) { return params_get(cast(CodeParam, params), idx); } +forceinline bool has_entries (CodeDefineParams params ) { return params_has_entries(cast(CodeParam, params)); } +forceinline StrBuilder to_strbuilder(CodeDefineParams params ) { return define_params_to_strbuilder(params); } +forceinline void to_strbuilder(CodeDefineParams params, StrBuilder& result ) { return define_params_to_strbuilder_ref(params, & result); } + +forceinline void append (CodeParams appendee, CodeParams other ) { return params_append(appendee, other); } +forceinline CodeParams get (CodeParams params, s32 idx) { return params_get(params, idx); } +forceinline bool has_entries (CodeParams params ) { return params_has_entries(params); } +forceinline StrBuilder to_strbuilder(CodeParams params ) { return params_to_strbuilder(params); } +forceinline void to_strbuilder(CodeParams params, StrBuilder& result ) { return params_to_strbuilder_ref(params, & result); } forceinline bool append (CodeSpecifiers specifiers, Specifier spec) { return specifiers_append(specifiers, spec); } forceinline s32 has (CodeSpecifiers specifiers, Specifier spec) { return specifiers_has(specifiers, spec); } forceinline s32 remove (CodeSpecifiers specifiers, Specifier to_remove ) { return specifiers_remove(specifiers, to_remove); } forceinline StrBuilder to_strbuilder(CodeSpecifiers specifiers) { return specifiers_to_strbuilder(specifiers); } -forceinline void to_strbuilder(CodeSpecifiers specifiers, StrBuilder& result) { return specifiers_to_strbuilder_ref(specifiers, & result); } +forceinline void to_strbuilder(CodeSpecifiers specifiers, StrBuilder& result) { return specifiers_to_strbuilder_ref(specifiers, & result); } forceinline void add_interface (CodeStruct self, CodeTypename interface) { return struct_add_interface(self, interface); } forceinline StrBuilder to_strbuilder (CodeStruct self) { return struct_to_strbuilder(self); } diff --git a/base/components/inlines.hpp b/base/components/inlines.hpp index 7a74511..b84ab3b 100644 --- a/base/components/inlines.hpp +++ b/base/components/inlines.hpp @@ -257,6 +257,16 @@ CodeParams next_CodeParams(CodeParams params, CodeParams param_iter) } #pragma endregion CodeParams +#pragma region CodeDefineParams +forceinline void define_params_append (CodeDefineParams appendee, CodeDefineParams other ) { params_append( cast(CodeParams, appendee), cast(CodeParams, other) ); } +forceinline CodeDefineParams define_params_get (CodeDefineParams self, s32 idx ) { return params_get( cast(CodeParams, self), idx); } +forceinline bool define_params_has_entries(CodeDefineParams self) { return params_has_entries( cast(CodeParams, self)); } + +CodeDefineParams begin_CodeDefineParams(CodeDefineParams params) { return begin_CodeParams( cast(CodeParams, params)); } +CodeDefineParams end_CodeDefineParams (CodeDefineParams params) { return end_CodeParams ( cast(CodeParams, params)); } +CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter) { return next_CodeParams ( cast(CodeParams, params), cast(CoeParams, entry_iter)); } +#pragma endregion CodeDefineParams + #pragma region CodeSpecifiers inline bool specifiers_append(CodeSpecifiers self, Specifier spec ) diff --git a/base/components/interface.cpp b/base/components/interface.cpp index e08b422..7e0f37f 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -354,7 +354,7 @@ void deinit(Context* ctx) array_free( ctx->CodePools); array_free( ctx->StringArenas); - arena_free(& ctx->LexArena); + // arena_free(& ctx->LexArena); array_free(ctx->PreprocessorDefines); @@ -463,10 +463,12 @@ Code make_code() return result; } -void set_preprocess_define( Str id, b32 is_functional ) { - StrBuilder builder = strbuilder_make_str( _ctx->Allocator_Temp, id ); - if (is_functional) { - strbuilder_append_char( & builder, '(' ); - } - array_append( _ctx->PreprocessorDefines, cache_str( strbuilder_to_str(builder)) ); +PreprocessorMacro* lookup_preprocess_macro( Str name ) { + u32 key = crc32( name.Ptr, name.Len ); + return hashtable_get( _ctx->PreprocessorMacros, key ); +} + +void register_preprocess_macro( PreprocessorMacro macro ) { + u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); + hashtable_set( _ctx->PreprocessorMacros, key, macro ); } diff --git a/base/components/interface.hpp b/base/components/interface.hpp index 99e8d1e..ec3326b 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -71,7 +71,7 @@ struct Context // Used by the lexer to persistently treat all these identifiers as preprocessor defines. // Populate with strings via gen::cache_str. // Functional defines must have format: id( ;at minimum to indicate that the define is only valid with arguments. - Array(StrCached) PreprocessorDefines; + HashTable(PreprocessorMacro) PreprocessorMacros; // Backend @@ -88,8 +88,8 @@ struct Context // TODO(Ed): This needs to be just handled by a parser context - Arena LexArena; - StringTable Lexer_defines; + // Arena LexArena; + // StringTable Lexer_defines; Array(Token) Lexer_Tokens; // TODO(Ed): Active parse context vs a parse result need to be separated conceptually @@ -109,9 +109,13 @@ GEN_API void reset(Context* ctx); GEN_API void set_context(Context* ctx); +// Mostly used to verify a macro entry exists for the given name +GEN_API PreprocessMacro* lookup_preprocess_macro( Str Name ); + // Alternative way to add a preprocess define entry for the lexer & parser to utilize // if the user doesn't want to use def_define -GEN_API void set_preprocess_define( Str id, b32 is_functional ); +// Macros are tracked by name so if the name already exists the entry will be overwritten. +GEN_API void register_preprocess_macro( PreprocessorMacro macro ); // Used internally to retrive or make string allocations. // Strings are stored in a series of string arenas of fixed size (SizePer_StringArena) @@ -150,9 +154,11 @@ struct Opts_def_constructor { GEN_API CodeConstructor def_constructor( Opts_def_constructor opts GEN_PARAM_DEFAULT ); struct Opts_def_define { - b32 dont_append_preprocess_defines; + MacroFlags flags; + CodeDefineParams params; + b32 dont_register_to_preprocess_macros; }; -GEN_API CodeDefine def_define( Str name, Str content, Opts_def_define opts GEN_PARAM_DEFAULT ); +GEN_API CodeDefine def_define( Str name, MacroType type, Opts_def_define opts GEN_PARAM_DEFAULT ); struct Opts_def_destructor { Code body; diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 206562c..54f9258 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -221,6 +221,9 @@ s32 lex_preprocessor_directive( LexContext* ctx ) name.Text.Len = 1; move_forward(); + PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); + + while ( ctx->left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) { move_forward(); @@ -477,23 +480,24 @@ TokArray lex( Str content ) return null_array; } - for ( StrCached* entry = array_begin(_ctx->PreprocessorDefines); entry != array_end(_ctx->PreprocessorDefines); entry = array_next(_ctx->PreprocessorDefines, entry)) - { - s32 length = 0; - char const* entry_scanner = (*entry).Ptr; - while ( entry->Len > length && (char_is_alphanumeric( *entry_scanner ) || *entry_scanner == '_') ) - { - entry_scanner++; - length ++; - } - if ( entry_scanner[0] == '(' ) - { - length++; - } + // TODO(ED): Remove this when preprocess defines has been converted + // for ( StrCached* entry = array_begin(_ctx->PreprocessorDefines); entry != array_end(_ctx->PreprocessorDefines); entry = array_next(_ctx->PreprocessorDefines, entry)) + // { + // s32 length = 0; + // char const* entry_scanner = (*entry).Ptr; + // while ( entry->Len > length && (char_is_alphanumeric( *entry_scanner ) || *entry_scanner == '_') ) + // { + // entry_scanner++; + // length ++; + // } + // if ( entry_scanner[0] == '(' ) + // { + // length++; + // } - u64 key = crc32( entry->Ptr, length ); - hashtable_set(c.defines, key, * entry ); - } + // u64 key = crc32( entry->Ptr, length ); + // hashtable_set(c.defines, key, * entry ); + // } array_clear(_ctx->Lexer_Tokens); @@ -1210,8 +1214,6 @@ TokArray lex( Str content ) } } - hashtable_clear(_ctx->Lexer_defines); - // defines_map_arena.free(); TokArray result = { _ctx->Lexer_Tokens, 0 }; return result; } diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 3c850f8..f0babe6 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -1774,7 +1774,7 @@ CodeBody parse_global_nspace( CodeType which ) // } //! Fallthrough intentional - GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES: + GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIER_CASES: { Specifier specs_found[16] = { Spec_NumSpecifiers }; s32 NumSpecifiers = 0; @@ -1787,7 +1787,7 @@ CodeBody parse_global_nspace( CodeType which ) switch ( spec ) { - GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIERS_CASES: + GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIER_CASES: break; case Spec_Consteval: @@ -2687,6 +2687,7 @@ CodeParams parse_params( bool use_template_capture ) #define CheckEndParams() \ (use_template_capture ? (currtok.Text.Ptr[ 0 ] != '>') : (currtok.Type != Tok_Capture_End)) + // TODO(Ed): Use expression macros or this? macro as attribute? // Ex: Unreal has this type of macro: vvvvvvvvv // COREUOBJECT_API void CallFunction( FFrame& Stack, RESULT_DECL, UFunction* Function ); // and: vvvv @@ -2713,6 +2714,7 @@ CodeParams parse_params( bool use_template_capture ) // ( } + // TODO(Ed): Use expression macro for this? // Unreal has yet another type of macro: // template::Value)> // class T ... and then ^this^ UE_REQUIRES shows up diff --git a/base/components/parser_case_macros.cpp b/base/components/parser_case_macros.cpp index c2497c4..a84e401 100644 --- a/base/components/parser_case_macros.cpp +++ b/base/components/parser_case_macros.cpp @@ -25,7 +25,7 @@ case Spec_Static: \ case Spec_Volatile: \ case Spec_Virtual -#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES \ +#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIER_CASES \ case Tok_Spec_Consteval: \ case Tok_Spec_Constexpr: \ case Tok_Spec_Constinit: \ @@ -37,7 +37,7 @@ case Tok_Spec_Internal_Linkage: \ case Tok_Spec_NeverInline: \ case Tok_Spec_Static -#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIERS_CASES \ +#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIER_CASES \ case Spec_Constexpr: \ case Spec_Constinit: \ case Spec_ForceInline: \ diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index cb91e9a..55babf4 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -6,37 +6,49 @@ #include "gen/especifier.hpp" #endif -enum MacroFlags : u32 +enum MacroTypes : u16 { - // Can only be one of these at a time (required) - MF_Block_Start = bit(0), // Start of a "block" scope - MF_Block_End = bit(1), // End of a "block" scope - MF_Case_Statement = bit(2), // Used as a case statement (not utilized by the parser yet) - MF_Expression = bit(3), // Used as an expresssion (not utilized by the parser yet) - MF_Statement = bit(4), // Used a statement (will expect to be a lone macro) - MF_Expects_Body = bit(5), // Expects to consume a braced scope - MF_Typename = bit(6), // Behaves as a typename + MT_Block_Start, // Not Supported yet + MT_Block_End, // Not Supported yet + MT_Case_Statement, // Not Supported yet + MT_Expression, + MT_Statement, + MT_Typename, - // Optional - MF_Functional = bit(7), + MF_UnderlyingType = GEN_U16_Max, +}; + +enum MacroFlags : u16 +{ + MF_Functional = bit(0), // Macro has parameters (args expected to be passed) + MF_Expects_Body = bit(1), // Expects to assign a braced scope to its body. MF_Null = 0, - MF_UnderlyingType = GEN_U32_MAX, + MF_UnderlyingType = GEN_U16_Max, +}; + +struct PreprocessorMacro +{ + StrCached Name; + MacroTypes Type; + MacroFlags Flags; }; enum TokFlags : u32 { - TF_Operator = bit(0), - TF_Assign = bit(1), - TF_Preprocess = bit(2), - TF_Preprocess_Cond = bit(3), - TF_Attribute = bit(6), - TF_AccessOperator = bit( 7 ), - TF_AccessSpecifier = bit( 8 ), - TF_Specifier = bit( 9 ), - TF_EndDefinition = bit( 10 ), // Either ; or } - TF_Formatting = bit( 11 ), - TF_Literal = bit( 12 ), + TF_Operator = bit(0), + TF_Assign = bit(1), + TF_Preprocess = bit(2), + TF_Preprocess_Cond = bit(3), + TF_Attribute = bit(6), + TF_AccessOperator = bit(7), + TF_AccessSpecifier = bit(8), + TF_Specifier = bit(9), + TF_EndDefinition = bit(10), // Either ; or } + TF_Formatting = bit(11), + TF_Literal = bit(12), + TF_Macro_Functional = bit(13), + TF_Macro_Expects_Body = bit(14), TF_Null = 0, TF_UnderlyingType = GEN_U32_MAX, @@ -123,7 +135,7 @@ struct LexContext char const* scanner; s32 line; s32 column; - StringTable defines; + // StringTable defines; Token token; }; diff --git a/base/enums/ECodeTypes.csv b/base/enums/ECodeTypes.csv index fe30838..d1ca2d6 100644 --- a/base/enums/ECodeTypes.csv +++ b/base/enums/ECodeTypes.csv @@ -37,6 +37,7 @@ Operator_Member_Fwd, "operator" Operator_Cast, "operator" Operator_Cast_Fwd, "operator" Parameters, "__NA__" +Parameters_Define, "__NA__" Preprocess_Define, "define" Preprocess_Include, "include" Preprocess_If, "if" diff --git a/base/enums/ETokType.csv b/base/enums/ETokType.csv index d3c28fe..8780b28 100644 --- a/base/enums/ETokType.csv +++ b/base/enums/ETokType.csv @@ -1,95 +1,97 @@ -Invalid, "__invalid__" -Access_Private, "private" -Access_Protected, "protected" -Access_Public, "public" -Access_MemberSymbol, "." -Access_StaticSymbol, "::" -Ampersand, "&" -Ampersand_DBL, "&&" -Assign_Classifer, ":" -Attribute_Open, "[[" -Attribute_Close, "]]" -BraceCurly_Open, "{" -BraceCurly_Close, "}" -BraceSquare_Open, "[" -BraceSquare_Close, "]" -Capture_Start, "(" -Capture_End, ")" -Comment, "__comment__" -Comment_End, "__comment_end__" -Comment_Start, "__comment_start__" -Char, "__character__" -Comma, "," -Decl_Class, "class" -Decl_GNU_Attribute, "__attribute__" -Decl_MSVC_Attribute, "__declspec" -Decl_Enum, "enum" -Decl_Extern_Linkage, "extern" -Decl_Friend, "friend" -Decl_Module, "module" -Decl_Namespace, "namespace" -Decl_Operator, "operator" -Decl_Struct, "struct" -Decl_Template, "template" -Decl_Typedef, "typedef" -Decl_Using, "using" -Decl_Union, "union" -Identifier, "__identifier__" -Module_Import, "import" -Module_Export, "export" -NewLine, "__new_line__" -Number, "__number__" -Operator, "__operator__" -Preprocess_Hash, "#" -Preprocess_Define, "define" -Preprocess_If, "if" -Preprocess_IfDef, "ifdef" -Preprocess_IfNotDef, "ifndef" -Preprocess_ElIf, "elif" -Preprocess_Else, "else" -Preprocess_EndIf, "endif" -Preprocess_Include, "include" -Preprocess_Pragma, "pragma" -Preprocess_Content, "__macro_content__" -Preprocess_Macro, "__macro__" -Preprocess_Unsupported, "__unsupported__" -Spec_Alignas, "alignas" -Spec_Const, "const" -Spec_Consteval, "consteval" -Spec_Constexpr, "constexpr" -Spec_Constinit, "constinit" -Spec_Explicit, "explicit" -Spec_Extern, "extern" -Spec_Final, "final" -Spec_ForceInline, "forceinline" -Spec_Global, "global" -Spec_Inline, "inline" -Spec_Internal_Linkage, "internal" -Spec_LocalPersist, "local_persist" -Spec_Mutable, "mutable" -Spec_NeverInline, "neverinline" -Spec_Override, "override" -Spec_Static, "static" -Spec_ThreadLocal, "thread_local" -Spec_Volatile, "volatile" -Spec_Virtual, "virtual" -Star, "*" -Statement_End, ";" -StaticAssert, "static_assert" -String, "__string__" -Type_Typename, "typename" -Type_Unsigned, "unsigned" -Type_Signed, "signed" -Type_Short, "short" -Type_Long, "long" -Type_bool, "bool" -Type_char, "char" -Type_int, "int" -Type_double, "double" -Type_MS_int8, "__int8" -Type_MS_int16, "__int16" -Type_MS_int32, "__int32" -Type_MS_int64, "__int64" -Type_MS_W64, "_W64" -Varadic_Argument, "..." -__Attributes_Start, "__attrib_start__" +Invalid, "__invalid__" +Access_Private, "private" +Access_Protected, "protected" +Access_Public, "public" +Access_MemberSymbol, "." +Access_StaticSymbol, "::" +Ampersand, "&" +Ampersand_DBL, "&&" +Assign_Classifer, ":" +Attribute_Open, "[[" +Attribute_Close, "]]" +BraceCurly_Open, "{" +BraceCurly_Close, "}" +BraceSquare_Open, "[" +BraceSquare_Close, "]" +Capture_Start, "(" +Capture_End, ")" +Comment, "__comment__" +Comment_End, "__comment_end__" +Comment_Start, "__comment_start__" +Char, "__character__" +Comma, "," +Decl_Class, "class" +Decl_GNU_Attribute, "__attribute__" +Decl_MSVC_Attribute, "__declspec" +Decl_Enum, "enum" +Decl_Extern_Linkage, "extern" +Decl_Friend, "friend" +Decl_Module, "module" +Decl_Namespace, "namespace" +Decl_Operator, "operator" +Decl_Struct, "struct" +Decl_Template, "template" +Decl_Typedef, "typedef" +Decl_Using, "using" +Decl_Union, "union" +Identifier, "__identifier__" +Module_Import, "import" +Module_Export, "export" +NewLine, "__new_line__" +Number, "__number__" +Operator, "__operator__" +Preprocess_Hash, "#" +Preprocess_Define, "define" +Preprocess_If, "if" +Preprocess_IfDef, "ifdef" +Preprocess_IfNotDef, "ifndef" +Preprocess_ElIf, "elif" +Preprocess_Else, "else" +Preprocess_EndIf, "endif" +Preprocess_Include, "include" +Preprocess_Pragma, "pragma" +Preprocess_Content, "__macro_content__" +Preprocess_Macro_Expr, "__macro_expression__" +Preprocess_Macro_Stmt, "__macro_statment__" +Preprocess_Macro_Typename, "__macro_typename__" +Preprocess_Unsupported, "__unsupported__" +Spec_Alignas, "alignas" +Spec_Const, "const" +Spec_Consteval, "consteval" +Spec_Constexpr, "constexpr" +Spec_Constinit, "constinit" +Spec_Explicit, "explicit" +Spec_Extern, "extern" +Spec_Final, "final" +Spec_ForceInline, "forceinline" +Spec_Global, "global" +Spec_Inline, "inline" +Spec_Internal_Linkage, "internal" +Spec_LocalPersist, "local_persist" +Spec_Mutable, "mutable" +Spec_NeverInline, "neverinline" +Spec_Override, "override" +Spec_Static, "static" +Spec_ThreadLocal, "thread_local" +Spec_Volatile, "volatile" +Spec_Virtual, "virtual" +Star, "*" +Statement_End, ";" +StaticAssert, "static_assert" +String, "__string__" +Type_Typename, "typename" +Type_Unsigned, "unsigned" +Type_Signed, "signed" +Type_Short, "short" +Type_Long, "long" +Type_bool, "bool" +Type_char, "char" +Type_int, "int" +Type_double, "double" +Type_MS_int8, "__int8" +Type_MS_int16, "__int16" +Type_MS_int32, "__int32" +Type_MS_int64, "__int64" +Type_MS_W64, "_W64" +Varadic_Argument, "..." +__Attributes_Start, "__attrib_start__" diff --git a/gen_c_library/components/misc.hpp b/gen_c_library/components/misc.hpp index bd06189..727d2b6 100644 --- a/gen_c_library/components/misc.hpp +++ b/gen_c_library/components/misc.hpp @@ -7,12 +7,13 @@ void convert_cpp_enum_to_c( CodeEnum to_convert, CodeBody to_append ) { #pragma push_macro("enum_underlying") #undef enum_underlying + StrCached type = to_convert->UnderlyingType ? to_convert->UnderlyingType : to_convert->Name + CodeTypedef tdef = parse_typedef(token_fmt("type", type, "name", to_convert->Name, stringize( typedef enum ; ))); if (to_convert->UnderlyingType) { to_convert->UnderlyingTypeMacro = untyped_str(token_fmt("type", to_convert->UnderlyingType->Name, stringize(enum_underlying()))); to_convert->UnderlyingType = CodeTypename{nullptr}; } - CodeTypedef tdef = parse_typedef(token_fmt("name", to_convert->Name, stringize( typedef enum ; ))); to_append.append(to_convert); to_append.append(tdef); #pragma pop_macro("enum_underlying") diff --git a/gen_unreal_engine/components/parser_case_macros.cpp b/gen_unreal_engine/components/parser_case_macros.cpp index e0b48af..4d71614 100644 --- a/gen_unreal_engine/components/parser_case_macros.cpp +++ b/gen_unreal_engine/components/parser_case_macros.cpp @@ -27,7 +27,7 @@ case Spec_Static: \ case Spec_Volatile: \ case Spec_Virtual -#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIERS_CASES \ +#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIER_CASES \ case Tok_Spec_Consteval: \ case Tok_Spec_Constexpr: \ case Tok_Spec_Constinit: \ @@ -40,7 +40,7 @@ case Tok_Spec_Internal_Linkage: \ case Tok_Spec_NeverInline: \ case Tok_Spec_Static -#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIERS_CASES \ +#define GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIER_CASES \ case Spec_Constexpr: \ case Spec_Constinit: \ case Spec_ForceInline: \ From c8cf55403b1d23304d365b2c9890967f6f1262ed Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 14 Dec 2024 18:49:41 -0500 Subject: [PATCH 02/15] WIP: more progress on new macro handling --- base/components/ast.cpp | 25 +++- base/components/code_serialization.cpp | 79 ++++++++---- base/components/gen/ecodetypes.hpp | 3 + base/components/gen/etoktype.hpp | 10 +- base/components/header_end.hpp | 2 +- base/components/interface.cpp | 23 ++-- base/components/interface.hpp | 47 +++---- base/components/interface.upfront.cpp | 99 +++++++++++---- base/components/lexer.cpp | 162 +++++++++++++++++-------- base/components/parser.cpp | 27 +++-- base/components/parser_types.hpp | 101 +++++++++------ base/components/static_data.cpp | 2 +- base/dependencies/macros.hpp | 4 +- base/enums/ETokType.csv | 1 + base/helpers/undef.macros.h | 2 +- gen_c_library/c_library.refactor | 2 +- 16 files changed, 400 insertions(+), 189 deletions(-) diff --git a/base/components/ast.cpp b/base/components/ast.cpp index c538a4c..55067bf 100644 --- a/base/components/ast.cpp +++ b/base/components/ast.cpp @@ -36,7 +36,6 @@ Str code_debug_str(Code self) case CT_Execution: case CT_Comment: case CT_PlatformAttributes: - case CT_Preprocess_Define: case CT_Preprocess_Include: case CT_Preprocess_Pragma: case CT_Preprocess_If: @@ -52,6 +51,11 @@ Str code_debug_str(Code self) strbuilder_append_fmt( result, "\n\tContent: %S", self->Content ); break; + case CT_Preprocess_Define: + // TODO(ED): Needs implementaton + log_failure("code_debug_str: NOT IMPLEMENTED for CT_Preprocess_Define"); + break; + case CT_Class: case CT_Struct: if ( self->Prev ) @@ -259,6 +263,11 @@ Str code_debug_str(Code self) strbuilder_append_fmt( result, "\n\tValue : %S", self->Value ? strbuilder_to_str( code_to_strbuilder(self->Value)) : txt("Null") ); break; + case CT_Parameters_Define: + // TODO(ED): Needs implementaton + log_failure("code_debug_str: NOT IMPLEMENTED for CT_Parameters_Define"); + break; + case CT_Specifiers: { strbuilder_append_fmt( result, "\n\tNumEntries: %d", self->NumEntries ); @@ -497,6 +506,10 @@ void code_to_strbuilder_ptr( Code self, StrBuilder* result ) params_to_strbuilder_ref(cast(CodeParams, self), result ); break; + case CT_Parameters_Define: + define_params_to_strbuilder_ref(cast(CodeDefineParams, self), result); + break; + case CT_Preprocess_Define: define_to_strbuilder_ref(cast(CodeDefine, self), result ); break; @@ -987,11 +1000,17 @@ bool code_is_equal( Code self, Code other ) return true; } + case CT_Parameters_Define: + { + // TODO(ED): Needs implementaton + log_failure("code_is_equal: NOT IMPLEMENTED for CT_Parameters_Define"); + return false; + } + case CT_Preprocess_Define: { check_member_str( Name ); - check_member_content( Content ); - + check_member_content( Body->Content ); return true; } diff --git a/base/components/code_serialization.cpp b/base/components/code_serialization.cpp index 2161c21..b219e80 100644 --- a/base/components/code_serialization.cpp +++ b/base/components/code_serialization.cpp @@ -176,7 +176,7 @@ void class_to_strbuilder_def( CodeClass self, StrBuilder* result ) { GEN_ASSERT(self); - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); strbuilder_append_str( result, txt("class ") ); @@ -221,7 +221,7 @@ void class_to_strbuilder_fwd( CodeClass self, StrBuilder* result ) { GEN_ASSERT(self); - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes ) @@ -241,12 +241,48 @@ void class_to_strbuilder_fwd( CodeClass self, StrBuilder* result ) StrBuilder define_to_strbuilder(CodeDefine define) { - return strbuilder_fmt_buf( _ctx->Allocator_Temp, "#define %S %S", define->Name, define->Content ); + StrBuilder result = strbuilder_make_reserve( _ctx->Allocator_Temp, 512 ); + define_to_strbuilder_ref(define, & result); + return result; } void define_to_strbuilder_ref(CodeDefine define, StrBuilder* result ) { - strbuilder_append_fmt( result, "#define %S %S", define->Name, define->Content ); + GEN_ASSERT(define); + GEN_ASSERT(define->Body); + GEN_ASSERT(define->Body->Content); + if (define->Params) { + StrBuilder params_builder = define_params_to_strbuilder(define->Params) + strbuilder_append_fmt( result, "#define %S(%S) %S", define->Name, strbuilder_to_str(params_builder), define->Body->Content ); + } + else { + strbuilder_append_fmt( result, "#define %S %S", define->Name, define->Body->Content ); + } +} + +StrBuilder define_params_to_strbuilder(CodeDefineParams params) +{ + GEN_ASSERT(params); + StrBuilder result = strbuilder_make_reserve( _ctx->Allocator_Temp, 128 ); + define_params_to_strbuilder_ref( params, & result ); + return result; +} + +void define_params_to_strbuilder_ref(CodeDefineParams params, StrBuilder* result) +{ + GEN_ASSERT(self); + GEN_ASSERT(result); + if ( self->Name.Ptr && self->Name.Len ) + { + strbuilder_append_fmt( result, " %S", self->Name ); + } + if ( self->NumEntries - 1 > 0 ) + { + for ( CodeParams param = begin_CodeDefineParams(self->Next); param != end_CodeDefineParams(self->Next); param = next_CodeDefineParams(self->Next, param) ) + { + strbuilder_append_fmt( result, ", %SB", params_to_strbuilder(param) ); + } + } } StrBuilder destructor_to_strbuilder(CodeDestructor self) @@ -329,7 +365,7 @@ StrBuilder enum_to_strbuilder(CodeEnum self) void enum_to_strbuilder_def(CodeEnum self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes || self->UnderlyingType || self->UnderlyingTypeMacro ) @@ -362,7 +398,7 @@ void enum_to_strbuilder_def(CodeEnum self, StrBuilder* result ) void enum_to_strbuilder_fwd(CodeEnum self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes ) @@ -389,7 +425,7 @@ void enum_to_strbuilder_fwd(CodeEnum self, StrBuilder* result ) void enum_to_strbuilder_class_def(CodeEnum self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes || self->UnderlyingType ) @@ -421,7 +457,7 @@ void enum_to_strbuilder_class_def(CodeEnum self, StrBuilder* result ) void enum_to_strbuilder_class_fwd(CodeEnum self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); strbuilder_append_str( result, txt("enum class ") ); @@ -505,7 +541,7 @@ StrBuilder fn_to_strbuilder(CodeFn self) void fn_to_strbuilder_def(CodeFn self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export") ); if ( self->Attributes ) @@ -558,7 +594,7 @@ void fn_to_strbuilder_def(CodeFn self, StrBuilder* result ) void fn_to_strbuilder_fwd(CodeFn self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes ) @@ -646,7 +682,7 @@ StrBuilder namespace_to_strbuilder(CodeNS self) void namespace_to_strbuilder_ref(CodeNS self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); strbuilder_append_fmt( result, "namespace %S\n{\n%SB\n}\n", self->Name, body_to_strbuilder(self->Body) ); @@ -671,7 +707,7 @@ StrBuilder code_op_to_strbuilder(CodeOperator self) void code_op_to_strbuilder_def(CodeOperator self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes ) @@ -725,7 +761,7 @@ void code_op_to_strbuilder_def(CodeOperator self, StrBuilder* result ) void code_op_to_strbuilder_fwd(CodeOperator self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes ) @@ -865,7 +901,6 @@ void opcast_to_strbuilder_fwd(CodeOpCast self, StrBuilder* result ) StrBuilder params_to_strbuilder(CodeParams self) { - GEN_ASSERT(self); GEN_ASSERT(self); StrBuilder result = strbuilder_make_reserve( _ctx->Allocator_Temp, 128 ); params_to_strbuilder_ref( self, & result ); @@ -1030,7 +1065,7 @@ void struct_to_strbuilder_def( CodeStruct self, StrBuilder* result ) { GEN_ASSERT(self); GEN_ASSERT(result); - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); strbuilder_append_str( result, txt("struct ") ); @@ -1076,7 +1111,7 @@ void struct_to_strbuilder_fwd( CodeStruct self, StrBuilder* result ) { GEN_ASSERT(self); GEN_ASSERT(result); - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes ) @@ -1105,7 +1140,7 @@ void template_to_strbuilder_ref(CodeTemplate self, StrBuilder* result ) { GEN_ASSERT(self); GEN_ASSERT(result); - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Params ) @@ -1123,7 +1158,7 @@ StrBuilder typedef_to_strbuilder(CodeTypedef self) void typedef_to_strbuilder_ref(CodeTypedef self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); strbuilder_append_str( result, txt("typedef ")); @@ -1236,7 +1271,7 @@ StrBuilder union_to_strbuilder(CodeUnion self) void union_to_strbuilder_def(CodeUnion self, StrBuilder* result ) { - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); strbuilder_append_str( result, txt("union ") ); @@ -1267,7 +1302,7 @@ void union_to_strbuilder_fwd(CodeUnion self, StrBuilder* result ) { GEN_ASSERT(self); GEN_ASSERT(result); - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); strbuilder_append_str( result, txt("union ") ); @@ -1304,7 +1339,7 @@ void using_to_strbuilder_ref(CodeUsing self, StrBuilder* result ) { GEN_ASSERT(self); GEN_ASSERT(result); - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes ) @@ -1401,7 +1436,7 @@ void var_to_strbuilder_ref(CodeVar self, StrBuilder* result ) return; } - if ( bitfield_is_equal( u32, self->ModuleFlags, ModuleFlag_Export )) + if ( bitfield_is_set( u32, self->ModuleFlags, ModuleFlag_Export )) strbuilder_append_str( result, txt("export ") ); if ( self->Attributes || self->Specs ) diff --git a/base/components/gen/ecodetypes.hpp b/base/components/gen/ecodetypes.hpp index aaf6d0d..c7ad1d0 100644 --- a/base/components/gen/ecodetypes.hpp +++ b/base/components/gen/ecodetypes.hpp @@ -46,6 +46,7 @@ enum CodeType : u32 CT_Operator_Cast, CT_Operator_Cast_Fwd, CT_Parameters, + CT_Parameters_Define, CT_Preprocess_Define, CT_Preprocess_Include, CT_Preprocess_If, @@ -114,6 +115,7 @@ inline Str codetype_to_str( CodeType type ) { "Operator_Cast", sizeof( "Operator_Cast" ) - 1 }, { "Operator_Cast_Fwd", sizeof( "Operator_Cast_Fwd" ) - 1 }, { "Parameters", sizeof( "Parameters" ) - 1 }, + { "Parameters_Define", sizeof( "Parameters_Define" ) - 1 }, { "Preprocess_Define", sizeof( "Preprocess_Define" ) - 1 }, { "Preprocess_Include", sizeof( "Preprocess_Include" ) - 1 }, { "Preprocess_If", sizeof( "Preprocess_If" ) - 1 }, @@ -182,6 +184,7 @@ inline Str codetype_to_keyword_str( CodeType type ) { "operator", sizeof( "operator" ) - 1 }, { "operator", sizeof( "operator" ) - 1 }, { "__NA__", sizeof( "__NA__" ) - 1 }, + { "__NA__", sizeof( "__NA__" ) - 1 }, { "define", sizeof( "define" ) - 1 }, { "include", sizeof( "include" ) - 1 }, { "if", sizeof( "if" ) - 1 }, diff --git a/base/components/gen/etoktype.hpp b/base/components/gen/etoktype.hpp index 2f5ce6a..dc4a400 100644 --- a/base/components/gen/etoktype.hpp +++ b/base/components/gen/etoktype.hpp @@ -53,6 +53,7 @@ enum TokType : u32 Tok_Operator, Tok_Preprocess_Hash, Tok_Preprocess_Define, + Tok_Preprocess_Define_Param, Tok_Preprocess_If, Tok_Preprocess_IfDef, Tok_Preprocess_IfNotDef, @@ -62,7 +63,9 @@ enum TokType : u32 Tok_Preprocess_Include, Tok_Preprocess_Pragma, Tok_Preprocess_Content, - Tok_Preprocess_Macro, + Tok_Preprocess_Macro_Expr, + Tok_Preprocess_Macro_Stmt, + Tok_Preprocess_Macro_Typename, Tok_Preprocess_Unsupported, Tok_Spec_Alignas, Tok_Spec_Const, @@ -155,6 +158,7 @@ inline Str toktype_to_str( TokType type ) { "__operator__", sizeof( "__operator__" ) - 1 }, { "#", sizeof( "#" ) - 1 }, { "define", sizeof( "define" ) - 1 }, + { "__define_param__", sizeof( "__define__param__" ) - 1 }, { "if", sizeof( "if" ) - 1 }, { "ifdef", sizeof( "ifdef" ) - 1 }, { "ifndef", sizeof( "ifndef" ) - 1 }, @@ -164,7 +168,9 @@ inline Str toktype_to_str( TokType type ) { "include", sizeof( "include" ) - 1 }, { "pragma", sizeof( "pragma" ) - 1 }, { "__macro_content__", sizeof( "__macro_content__" ) - 1 }, - { "__macro__", sizeof( "__macro__" ) - 1 }, + { "__macro_expression__", sizeof( "__macro_expression__" ) - 1 }, + { "__macro_statment__", sizeof( "__macro_statment__" ) - 1 }, + { "__macro_typename__", sizeof( "__macro_typename__" ) - 1 }, { "__unsupported__", sizeof( "__unsupported__" ) - 1 }, { "alignas", sizeof( "alignas" ) - 1 }, { "const", sizeof( "const" ) - 1 }, diff --git a/base/components/header_end.hpp b/base/components/header_end.hpp index 703c1d9..edf639d 100644 --- a/base/components/header_end.hpp +++ b/base/components/header_end.hpp @@ -6,7 +6,7 @@ #pragma region Constants -extern Str enum_underlying_sig; +extern PreprocessorMacro enum_underlying_macro; extern Code access_public; extern Code access_protected; diff --git a/base/components/interface.cpp b/base/components/interface.cpp index 7e0f37f..6a2d648 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -206,10 +206,12 @@ void define_constants() spec_local_persist = def_specifiers( 1, Spec_Local_Persist ); code_set_global(cast(Code, spec_local_persist)); - if (enum_underlying_sig.Len == 0) { - enum_underlying_sig = txt("enum_underlying("); + if (enum_underlying_macro.Name.Len == 0) { + enum_underlying_macro.Name = txt("enum_underlying("); + enum_underlying_macro.Type = MT_Expression; + enum_underlying_macro.Flags = MF_Functional; } - array_append( _ctx->PreprocessorDefines, enum_underlying_sig); + register_preprocess_macro(enum_underlying_macro); } void init(Context* ctx) @@ -301,9 +303,6 @@ void init(Context* ctx) GEN_FATAL( "gen::init: Failed to initialize the code pool" ); array_append( ctx->CodePools, code_pool ); - // TODO(Ed): This is going to be phased out most likely. - ctx->LexArena = arena_init_from_allocator( ctx->Allocator_DyanmicContainers, ctx->InitSize_LexArena ); - // TODO(Ed): Eventually the string arenas needs to be phased out for a dedicated string slab allocator Arena strbuilder_arena = arena_init_from_allocator( ctx->Allocator_StrCache, ctx->SizePer_StringArena ); if ( strbuilder_arena.PhysicalStart == nullptr ) @@ -315,9 +314,12 @@ void init(Context* ctx) ctx->StrCache = hashtable_init(StrCached, ctx->Allocator_DyanmicContainers); if ( ctx->StrCache.Entries == nullptr ) GEN_FATAL( "gen::init: Failed to initialize the StringCache"); + + ctx->PreprocessorMacros = hashtable_init(PreprocessorMacros, ctx->Allocator_DyanmicContainers); + if (ctx->PreprocessorMacros.Hashes == nullptr || ctx->PreprocessorMacros.Entries == nullptr) { + GEN_FATAL( "gen::init: Failed to initialize the PreprocessMacros table" ); + } } - // Preprocessor Defines - ctx->PreprocessorDefines = array_init_reserve(StrCached, ctx->Allocator_DyanmicContainers, kilobytes(1) ); define_constants(); parser_init(); @@ -354,9 +356,7 @@ void deinit(Context* ctx) array_free( ctx->CodePools); array_free( ctx->StringArenas); - // arena_free(& ctx->LexArena); - - array_free(ctx->PreprocessorDefines); + hashtable_destroy(ctx->PreprocessorMacros); left = array_num( ctx->Fallback_AllocatorBuckets); if (left) @@ -401,6 +401,7 @@ void reset(Context* ctx) while ( left--, left ); hashtable_clear(ctx->StrCache); + hashtable_clear(ctx->PreprocessorMacros); define_constants(); } diff --git a/base/components/interface.hpp b/base/components/interface.hpp index ec3326b..e56a87a 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -156,6 +156,7 @@ GEN_API CodeConstructor def_constructor( Opts_def_constructor opts GEN_PARAM_DEF struct Opts_def_define { MacroFlags flags; CodeDefineParams params; + Str content; b32 dont_register_to_preprocess_macros; }; GEN_API CodeDefine def_define( Str name, MacroType type, Opts_def_define opts GEN_PARAM_DEFAULT ); @@ -269,28 +270,30 @@ GEN_API CodeBody def_body( CodeType type ); // There are two options for defining a struct body, either varadically provided with the args macro to auto-deduce the arg num, /// or provide as an array of Code objects. -GEN_API CodeBody def_class_body ( s32 num, ... ); -GEN_API CodeBody def_class_body ( s32 num, Code* codes ); -GEN_API CodeBody def_enum_body ( s32 num, ... ); -GEN_API CodeBody def_enum_body ( s32 num, Code* codes ); -GEN_API CodeBody def_export_body ( s32 num, ... ); -GEN_API CodeBody def_export_body ( s32 num, Code* codes); -GEN_API CodeBody def_extern_link_body( s32 num, ... ); -GEN_API CodeBody def_extern_link_body( s32 num, Code* codes ); -GEN_API CodeBody def_function_body ( s32 num, ... ); -GEN_API CodeBody def_function_body ( s32 num, Code* codes ); -GEN_API CodeBody def_global_body ( s32 num, ... ); -GEN_API CodeBody def_global_body ( s32 num, Code* codes ); -GEN_API CodeBody def_namespace_body ( s32 num, ... ); -GEN_API CodeBody def_namespace_body ( s32 num, Code* codes ); -GEN_API CodeParams def_params ( s32 num, ... ); -GEN_API CodeParams def_params ( s32 num, CodeParams* params ); -GEN_API CodeSpecifiers def_specifiers ( s32 num, ... ); -GEN_API CodeSpecifiers def_specifiers ( s32 num, Specifier* specs ); -GEN_API CodeBody def_struct_body ( s32 num, ... ); -GEN_API CodeBody def_struct_body ( s32 num, Code* codes ); -GEN_API CodeBody def_union_body ( s32 num, ... ); -GEN_API CodeBody def_union_body ( s32 num, Code* codes ); +GEN_API CodeBody def_class_body ( s32 num, ... ); +GEN_API CodeBody def_class_body ( s32 num, Code* codes ); +GEN_API CodeDefineParams def_define_params ( s32 num, ... ); +GEN_API CodeDefineParams def_define_params ( s32 num, CodeDefineParams* codes ) +GEN_API CodeBody def_enum_body ( s32 num, ... ); +GEN_API CodeBody def_enum_body ( s32 num, Code* codes ); +GEN_API CodeBody def_export_body ( s32 num, ... ); +GEN_API CodeBody def_export_body ( s32 num, Code* codes); +GEN_API CodeBody def_extern_link_body( s32 num, ... ); +GEN_API CodeBody def_extern_link_body( s32 num, Code* codes ); +GEN_API CodeBody def_function_body ( s32 num, ... ); +GEN_API CodeBody def_function_body ( s32 num, Code* codes ); +GEN_API CodeBody def_global_body ( s32 num, ... ); +GEN_API CodeBody def_global_body ( s32 num, Code* codes ); +GEN_API CodeBody def_namespace_body ( s32 num, ... ); +GEN_API CodeBody def_namespace_body ( s32 num, Code* codes ); +GEN_API CodeParams def_params ( s32 num, ... ); +GEN_API CodeParams def_params ( s32 num, CodeParams* params ); +GEN_API CodeSpecifiers def_specifiers ( s32 num, ... ); +GEN_API CodeSpecifiers def_specifiers ( s32 num, Specifier* specs ); +GEN_API CodeBody def_struct_body ( s32 num, ... ); +GEN_API CodeBody def_struct_body ( s32 num, Code* codes ); +GEN_API CodeBody def_union_body ( s32 num, ... ); +GEN_API CodeBody def_union_body ( s32 num, Code* codes ); #pragma endregion Upfront diff --git a/base/components/interface.upfront.cpp b/base/components/interface.upfront.cpp index 0faec68..1e4b6b1 100644 --- a/base/components/interface.upfront.cpp +++ b/base/components/interface.upfront.cpp @@ -516,7 +516,6 @@ CodeClass def_class( Str name, Opts_def_struct p ) GEN_DEBUG_TRAP(); return InvalidCode; } - if ( p.attributes && p.attributes->Type != CT_PlatformAttributes ) { log_failure( "gen::def_class: attributes was not a 'PlatformAttributes' type: %s", code_debug_str(p.attributes) ); GEN_DEBUG_TRAP(); @@ -529,9 +528,12 @@ CodeClass def_class( Str name, Opts_def_struct p ) } CodeClass - result = (CodeClass) make_code(); - result->Name = cache_str( name ); - result->ModuleFlags = p.mflags; + result = (CodeClass) make_code(); + result->Name = cache_str( name ); + result->ModuleFlags = p.mflags; + result->Attributes = p.attributes; + result->ParentAccess = p.parent_access; + result->ParentType = p.parent; if ( p.body ) { switch ( p.body->Type ) @@ -552,44 +554,32 @@ CodeClass def_class( Str name, Opts_def_struct p ) else { result->Type = CT_Class_Fwd; } - - result->Attributes = p.attributes; - result->ParentAccess = p.parent_access; - result->ParentType = p.parent; - for (s32 idx = 0; idx < p.num_interfaces; idx++ ) { class_add_interface(result, p.interfaces[idx] ); } return result; } -CodeDefine def_define( Str name, Str content, Opts_def_define p ) +CodeDefine def_define( Str name, MacroType type, Opts_def_define p ) { if ( ! name_check( def_define, name ) ) { GEN_DEBUG_TRAP(); return InvalidCode; } - CodeDefine result = (CodeDefine) make_code(); result->Type = CT_Preprocess_Define; result->Name = cache_str( name ); - - if ( content.Len <= 0 || content.Ptr == nullptr ) - result->Content = cache_str( txt("") ); + result->Params = p.params; + if ( p.content.Len <= 0 || p.content.Ptr == nullptr ) + result->Content = cache_str( txt("\n") ); else result->Content = cache_str( strbuilder_to_str(strbuilder_fmt_buf(_ctx->Allocator_Temp, "%S\n", content)) ); - b32 append_preprocess_defines = ! p.dont_append_preprocess_defines; - if ( append_preprocess_defines ) { - // Add the define to PreprocessorDefines for usage in parsing - s32 lex_id_len = 0; - for (; lex_id_len < result->Name.Len; ++ lex_id_len ) { - if ( result->Name.Ptr[lex_id_len] == '(' ) - break; - } - Str lex_id = { result->Name.Ptr, lex_id_len }; - array_append(_ctx->PreprocessorDefines, cache_str(lex_id) ); + b32 register_define = ! p.dont_register_to_preprocess_macros; + if ( register_define ) { + macro_entry = { result->Name, type, p.flags }; + register_preprocess_macro(macro_entry); } return result; } @@ -1410,6 +1400,67 @@ CodeBody def_class_body( s32 num, Code* codes ) return result; } +CodeDefineParams def_define_params( s32 num, ... ) +{ + def_body_start( def_define_params ); + + va_list va; + va_start(va, num); + + Code_POD pod = va_arg(va, Code_POD); + CodeDefineParams param = pcast( CodeDefineParams, pod ); + + null_check( def_define_params, param ); + if ( param->Type != CT_Parameters_Define ) { + log_failure( "gen::def_define_params: param %d is not a parameter for a preprocessor define", num - num + 1 ); + return InvalidCode; + } + + CodeDefineParams result = (CodeDefineParams) code_duplicate(param); + while ( -- num ) + { + pod = va_arg(va, Code_POD); + param = pcast( CodeDefineParams, pod ); + if ( param->Type != CT_Parameters_Define ) { + log_failure( "gen::def_define_params: param %d is not a parameter for a preprocessor define", num - num + 1 ); + return InvalidCode; + } + define_params_append(result, param ); + } + va_end(va); + + return result; +} + +CodeDefineParams def_define_params( s32 num, CodeDefineParams* codes ) +{ + def_body_code_array_start( def_define_params ); + +# define check_current(current) \ + if ( current == nullptr ) { \ + log_failure("gen::def_define_params: Provide a null code in codes array"); \ + return InvalidCode; \ + } \ + if (current->Type != CT_Parameters_Define ) { \ + log_failure("gen::def_define_params: Code in coes array is not of paramter for preprocessor define type - %s", code_debug_str(current) ); \ + return InvalidCode; \ + } + CodeDefineParams current = (CodeDefineParams)code_duplicate(* codes); + check_current(current); + + CodeDefineParams + result = (CodeDefineParams) make_code(); + result->Name = current->Name; + result->Type = current->Type; + while( codes++, current = * codes, num--, num > 0 ) { + check_current(current); + define_params_append(result, current ); + } +# undef check_current + + return result; +} + CodeBody def_enum_body( s32 num, ... ) { def_body_start( def_enum_body ); diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 54f9258..b036706 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -112,6 +112,102 @@ void lexer_end_line( LexContext* ctx ) } #define end_line() lexer_end_line(ctx) +// TODO(Ed): We need to to attempt to recover from a lex failure? +s32 lex_preprocessor_define( LexContext* ctx ) +{ + Token name = { { ctx->scanner, 1 }, Tok_Identifier, ctx->line, ctx->column, TF_Preprocess }; + move_forward(); + + PreprocessorMacro macro = { name.Text, MT_Statement, 0 }; + PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); + if ( registered_macro == nullptr ) { + log_fmt("Warning: %S is was not registered before the lexer processed its #define directive, it will be registered as a statement macro" + , name.Text + ); + } + while ( ctx->left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) { + move_forward(); + name.Text.Len++; + } + array_append( _ctx->Lexer_Tokens, name ); + + if ( ctx->left && (* ctx->scanner) == '(' ) + { + if (registered_macro && ! macro_is_functional(* registered_macro)) { + log_fmt("Warning: %S registered macro is not flagged as functional yet the definition detects opening parenthesis '(' for arguments" + , name.Text + ); + } + else { + macro.Flags |= MF_Functional; + } + + Token opening_paren = { { ctx->scanner, 1 }, Tok_Capture_Start, ctx->line, ctx->column, TF_Preprocess }; + array_append( _ctx->Lexer_Tokens, opening_paren ); + move_forward(); + + // We need to tokenize the define's arguments now: + while( ctx->left && * ctx->scanner != ')') + { + skip_whitespace(); + if ( char_is_alpha( (* ctx->scanner) ) || (* ctx->scanner) == '_' ) + { + Token parameter = { { ctx->scanner, 1 }, Tok_Preprocess_Define_Param, ctx->line, ctx->column, TF_Preprocess }; + move_forward(); + + while ( ctx->left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) + { + move_forward(); + parameter.Text.Len++; + } + array_append(_ctx->Lexer_Tokens, parameter); + skip_whitespace(); + } + else { + log_failure("lex_preprocessor_define(%d, %d): Expected a '_' or alpha character for a parameter name for %S\n" + , ctx->line + , ctx->column + , name.Text + ); + return Lex_ReturnNull; + } + // There should be a comma + if ( * ctx->scanner != ',' ) { + log_failure("lex_preprocessor_define(%d, %d): Expected a comma after parameter %S for %S\n" + , ctx->line + , ctx->column + , parameter.Text + , name.Text + ); + return Lex_ReturnNull; + } + Token comma = { { ctx->scanner, 1 }, Tok_Comma, ctx->line, ctx->column, TF_Preprocess }; + array_append(_ctx->Lexer_Tokens, comma); + move_forward(); + } + + if ( * ctx->scanner != ')' ) { + log_failure("lex_preprocessor_define(%d, %d): Expected a ')' after last_parameter %S for %S (ran out of characters...)\n" + , ctx->line + , ctx->column + , parameter.Text + , name.Text + ); + return Lex_ReturnNull; + } + Token closing_paren = { { ctx->scanner, 1 }, Tok_Capture_End, ctx->line, ctx->column, TF_Preprocess }; + array_append(_ctx->Lexer_Tokens, closing_paren); + move_forward(); + } + if ( registered_macro == nullptr ) { + register_preprocess_macro(macro); + } + + // Define's content handled by lex_preprocessor_directive (the original caller of this) + return Lex_Continue; +} + +// TODO(Ed): We need to to attempt to recover from a lex failure? forceinline s32 lex_preprocessor_directive( LexContext* ctx ) { @@ -215,31 +311,9 @@ s32 lex_preprocessor_directive( LexContext* ctx ) if ( ctx->token.Type == Tok_Preprocess_Define ) { - Token name = { { ctx->scanner, 0 }, Tok_Identifier, ctx->line, ctx->column, TF_Preprocess }; - - name.Text.Ptr = ctx->scanner; - name.Text.Len = 1; - move_forward(); - - PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); - - - while ( ctx->left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) - { - move_forward(); - name.Text.Len++; - } - - if ( ctx->left && (* ctx->scanner) == '(' ) - { - move_forward(); - name.Text.Len++; - } - - array_append( _ctx->Lexer_Tokens, name ); - - u64 key = crc32( name.Text.Ptr, name.Text.Len ); - hashtable_set(ctx->defines, key, tok_to_str(name) ); + u32 result = lex_preprocessor_define(ctx); // handles: #define ( ) - define's content handled later on within this scope. + if (result != Lex_Continue) + return Lex_ReturnNull; } Token preprocess_content = { { ctx->scanner, 0 }, Tok_Preprocess_Content, ctx->line, ctx->column, TF_Preprocess }; @@ -289,7 +363,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) s32 within_string = false; s32 within_char = false; - // SkipWhitespace(); + // Consume preprocess content while ( ctx->left ) { if ( (* ctx->scanner) == '"' && ! within_char ) @@ -325,6 +399,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) , (* ctx->scanner), ctx->line, ctx->column , directive_str, preprocess_content.Line, preprocess_content.Column , content_str ); + return Lex_ReturnNull; break; } } @@ -404,14 +479,15 @@ void lex_found_token( LexContext* ctx ) } u64 key = 0; - if ( (* ctx->scanner) == '(') - key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len + 1 ); - else - key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len ); + // if ( (* ctx->scanner) == '(') + // key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len + 1 ); + // else + key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len ); - Str* define = hashtable_get(ctx->defines, key ); + Str* define = hashtable_get(_ctx->PreprocessMacros, key ); if ( define ) { + // TODO(Ed): Needs updating (Macros) ctx->token.Type = Tok_Preprocess_Macro; // Want to ignore any arguments the define may have as they can be execution expressions. @@ -456,6 +532,7 @@ void lex_found_token( LexContext* ctx ) array_append( _ctx->Lexer_Tokens, ctx->token ); } +// TODO(Ed): We need to to attempt to recover from a lex failure? neverinline // TokArray lex( Array tokens, Str content ) TokArray lex( Str content ) @@ -480,27 +557,10 @@ TokArray lex( Str content ) return null_array; } - // TODO(ED): Remove this when preprocess defines has been converted - // for ( StrCached* entry = array_begin(_ctx->PreprocessorDefines); entry != array_end(_ctx->PreprocessorDefines); entry = array_next(_ctx->PreprocessorDefines, entry)) - // { - // s32 length = 0; - // char const* entry_scanner = (*entry).Ptr; - // while ( entry->Len > length && (char_is_alphanumeric( *entry_scanner ) || *entry_scanner == '_') ) - // { - // entry_scanner++; - // length ++; - // } - // if ( entry_scanner[0] == '(' ) - // { - // length++; - // } - - // u64 key = crc32( entry->Ptr, length ); - // hashtable_set(c.defines, key, * entry ); - // } - array_clear(_ctx->Lexer_Tokens); + b32 preprocess_args = true; + while (c.left ) { #if 0 @@ -1182,6 +1242,7 @@ TokArray lex( Str content ) { lex_found_token( ctx ); TokType last_type = array_back(_ctx->Lexer_Tokens)->Type; + // TODO(Ed): Change this to just detect if its a MACRO THAT SHOULD ACCEPT NEWLINES if ( last_type == Tok_Preprocess_Macro ) { Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; @@ -1217,6 +1278,7 @@ TokArray lex( Str content ) TokArray result = { _ctx->Lexer_Tokens, 0 }; return result; } + #undef move_forward #undef skip_whitespace #undef end_line diff --git a/base/components/parser.cpp b/base/components/parser.cpp index f0babe6..21c2e14 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -825,22 +825,21 @@ CodeBody parse_class_struct_body( TokType which, Token name ) switch ( currtok_noskip.Type ) { - case Tok_Statement_End: - { + case Tok_Statement_End: { // TODO(Ed): Convert this to a general warning procedure log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(currtok_noskip)); eat( Tok_Statement_End ); continue; } - case Tok_NewLine: + case Tok_NewLine: { member = fmt_newline; eat( Tok_NewLine ); - break; - - case Tok_Comment: + break; + } + case Tok_Comment: { member = cast(Code, parse_comment()); - break; - + break; + } case Tok_Access_Public: member = access_public; eat( Tok_Access_Public ); @@ -1727,8 +1726,8 @@ CodeBody parse_global_nspace( CodeType which ) // macro_found = true; goto Preprocess_Macro_Bare_In_Body; + // TODO(Ed): MACRO UPDATE } - break; case Tok_Preprocess_Pragma: { member = cast(Code, parse_pragma()); @@ -2727,7 +2726,7 @@ CodeParams parse_params( bool use_template_capture ) // In template captures you can have a typename have direct assignment without a name // typename = typename ... // Which would result in a static value type from a struct expansion (traditionally) - if ( ( name.Text.Ptr || use_template_capture ) && bitfield_is_equal( u32, currtok.Flags, TF_Assign ) ) + if ( ( name.Text.Ptr || use_template_capture ) && bitfield_is_set( u32, currtok.Flags, TF_Assign ) ) { eat( Tok_Operator ); // ( = @@ -2839,7 +2838,7 @@ CodeParams parse_params( bool use_template_capture ) // In template captures you can have a typename have direct assignment without a name // typename = typename ... // Which would result in a static value type from a struct expansion (traditionally) - if ( ( name.Text.Ptr || use_template_capture ) && bitfield_is_equal( u32, currtok.Flags, TF_Assign ) ) + if ( ( name.Text.Ptr || use_template_capture ) && bitfield_is_set( u32, currtok.Flags, TF_Assign ) ) { eat( Tok_Operator ); // ( = , = @@ -3166,7 +3165,7 @@ CodeVar parse_variable_after_name( b32 using_constructor_initializer = false; - if ( bitfield_is_equal( u32, currtok.Flags, TF_Assign ) ) + if ( bitfield_is_set( u32, currtok.Flags, TF_Assign ) ) { // = expr = parse_assignment_expression(); @@ -4645,6 +4644,7 @@ else if ( currtok.Type == Tok_DeclType ) } } } + // TODO(Ed): This needs updating else if ( currtok.Type == Tok_Preprocess_Macro ) { // Typename is a macro name = currtok; @@ -4988,6 +4988,7 @@ CodeTypedef parser_parse_typedef() const bool from_typedef = true; + // TODO(Ed): UPDATE MACRO USAGE HERE #if GEN_PARSER_DISABLE_MACRO_TYPEDEF if ( false ) #else @@ -5365,7 +5366,7 @@ CodeUsing parser_parse_using() if ( ! is_namespace ) { - if ( bitfield_is_equal( u32, currtok.Flags, TF_Assign ) ) + if ( bitfield_is_set( u32, currtok.Flags, TF_Assign ) ) { attributes = parse_attributes(); // using diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index 55babf4..d829850 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -6,34 +6,6 @@ #include "gen/especifier.hpp" #endif -enum MacroTypes : u16 -{ - MT_Block_Start, // Not Supported yet - MT_Block_End, // Not Supported yet - MT_Case_Statement, // Not Supported yet - MT_Expression, - MT_Statement, - MT_Typename, - - MF_UnderlyingType = GEN_U16_Max, -}; - -enum MacroFlags : u16 -{ - MF_Functional = bit(0), // Macro has parameters (args expected to be passed) - MF_Expects_Body = bit(1), // Expects to assign a braced scope to its body. - - MF_Null = 0, - MF_UnderlyingType = GEN_U16_Max, -}; - -struct PreprocessorMacro -{ - StrCached Name; - MacroTypes Type; - MacroFlags Flags; -}; - enum TokFlags : u32 { TF_Operator = bit(0), @@ -82,42 +54,42 @@ bool tok_is_valid( Token tok ) { forceinline bool tok_is_access_operator(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_AccessOperator ); + return bitfield_is_set( u32, tok.Flags, TF_AccessOperator ); } forceinline bool tok_is_access_specifier(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_AccessSpecifier ); + return bitfield_is_set( u32, tok.Flags, TF_AccessSpecifier ); } forceinline bool tok_is_attribute(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Attribute ); + return bitfield_is_set( u32, tok.Flags, TF_Attribute ); } forceinline bool tok_is_operator(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Operator ); + return bitfield_is_set( u32, tok.Flags, TF_Operator ); } forceinline bool tok_is_preprocessor(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Preprocess ); + return bitfield_is_set( u32, tok.Flags, TF_Preprocess ); } forceinline bool tok_is_preprocess_cond(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Preprocess_Cond ); + return bitfield_is_set( u32, tok.Flags, TF_Preprocess_Cond ); } forceinline bool tok_is_specifier(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Specifier ); + return bitfield_is_set( u32, tok.Flags, TF_Specifier ); } forceinline bool tok_is_end_definition(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_EndDefinition ); + return bitfield_is_set( u32, tok.Flags, TF_EndDefinition ); } StrBuilder tok_to_strbuilder(Token tok); @@ -153,3 +125,60 @@ struct ParseContext TokArray Tokens; StackNode* Scope; }; + +enum MacroType : u16 +{ + MT_Statement, // A macro is assumed to be a statement if not resolved. + MT_Expression, + MT_Typename, + MT_Block_Start, // Not Supported yet + MT_Block_End, // Not Supported yet + MT_Case_Statement, // Not Supported yet + + MF_UnderlyingType = GEN_U16_Max, +}; + +Str macro_type_to_str( MacroType type ) +{ + local_persist + Str lookup[ (u32)Num_ModuleFlags ] = { + { "Statement", sizeof("Statement") - 1 }, + { "Expression", sizeof("Expression") - 1 }, + { "Typename", sizeof("Typename") - 1 }, + { "Block_Start", sizeof("Block_Start") - 1 }, + { "Block_End", sizeof("Block_End") - 1 }, + { "Case_Statement", sizeof("Case_Statement") - 1 }, + }; + local_persist + Str invalid_flag = { "Invalid", sizeof("Invalid") }; + if ( flag > ModuleFlag_Import ) + return invalid_flag; + + return lookup[ (u32)flag ]; +} + +enum MacroFlags : u16 +{ + MF_Functional = bit(0), // Macro has parameters (args expected to be passed) + MF_Expects_Body = bit(1), // Expects to assign a braced scope to its body. + + MF_Null = 0, + MF_UnderlyingType = GEN_U16_Max, +}; + +struct PreprocessorMacro +{ + StrCached Name; + MacroType Type; + MacroFlags Flags; +}; + +forceinine +b32 macro_is_functional( PreprocessorMacro macro ) { + return bitfield_is_set( macro->Flags, MF_Functional ); +} + +forceinline +b32 macro_expects_body( PreprocessorMacro macro ) { + return bitfield_is_set( macro->Flags, MF_Expects_Body ); +} diff --git a/base/components/static_data.cpp b/base/components/static_data.cpp index cad615c..70a6ff2 100644 --- a/base/components/static_data.cpp +++ b/base/components/static_data.cpp @@ -9,7 +9,7 @@ global Context* _ctx; #pragma region Constants global u32 context_counter; -global Str enum_underlying_sig; +global PreprocessorMacro enum_underlying_macro; global Code Code_Global; global Code Code_Invalid; diff --git a/base/dependencies/macros.hpp b/base/dependencies/macros.hpp index 78bbe30..cbcf0f7 100644 --- a/base/dependencies/macros.hpp +++ b/base/dependencies/macros.hpp @@ -34,8 +34,8 @@ #endif #ifndef bit -#define bit( Value ) ( 1 << Value ) -#define bitfield_is_equal( Type, Field, Mask ) ( (scast(Type, Mask) & scast(Type, Field)) == scast(Type, Mask) ) +#define bit( Value ) ( 1 << Value ) +#define bitfield_is_set( Type, Field, Mask ) ( (scast(Type, Mask) & scast(Type, Field)) == scast(Type, Mask) ) #endif // Mainly intended for forcing the base library to utilize only C-valid constructs or type coercion diff --git a/base/enums/ETokType.csv b/base/enums/ETokType.csv index 8780b28..bdf1d03 100644 --- a/base/enums/ETokType.csv +++ b/base/enums/ETokType.csv @@ -42,6 +42,7 @@ Number, "__number__" Operator, "__operator__" Preprocess_Hash, "#" Preprocess_Define, "define" +Preprocess_Define_Param, "__define_param__" Preprocess_If, "if" Preprocess_IfDef, "ifdef" Preprocess_IfNotDef, "ifndef" diff --git a/base/helpers/undef.macros.h b/base/helpers/undef.macros.h index e31d308..0bc7a48 100644 --- a/base/helpers/undef.macros.h +++ b/base/helpers/undef.macros.h @@ -41,7 +41,7 @@ #undef local_persist #undef bit -#undef bitfield_is_equal +#undef bitfield_is_set #undef cast diff --git a/gen_c_library/c_library.refactor b/gen_c_library/c_library.refactor index 1c98342..c743e5c 100644 --- a/gen_c_library/c_library.refactor +++ b/gen_c_library/c_library.refactor @@ -21,7 +21,7 @@ word global, gen_global word internal, gen_internal word local_persist, gen_local_persist word bit, gen_bit -word bitfield_is_equal, gen_bitfield_is_equal +word bitfield_is_set, gen_bitfield_is_set word cast, gen_cast word ccast, gen_ccast word pcast, gen_pcast From 1b4f9a2e771ddcfeac760aa431feccc2911f63ff Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 14 Dec 2024 20:33:14 -0500 Subject: [PATCH 03/15] WIP: prepped for dealing with the compiler... --- base/components/lexer.cpp | 72 ++--- base/components/parser.cpp | 524 ++++++++++++++----------------- base/components/parser_types.hpp | 15 +- base/enums/ETokType.csv | 4 +- base/helpers/base_codegen.hpp | 118 +++---- 5 files changed, 332 insertions(+), 401 deletions(-) diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index b036706..6dc5ff9 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -427,30 +427,24 @@ s32 lex_preprocessor_directive( LexContext* ctx ) forceinline void lex_found_token( LexContext* ctx ) { - if ( ctx->token.Type != Tok_Invalid ) - { + if ( ctx->token.Type != Tok_Invalid ) { array_append( _ctx->Lexer_Tokens, ctx->token ); return; } TokType type = str_to_toktype( tok_to_str(ctx->token) ); - if (type <= Tok_Access_Public && type >= Tok_Access_Private ) - { + if (type <= Tok_Access_Public && type >= Tok_Access_Private ) { ctx->token.Flags |= TF_AccessSpecifier; } - - if ( type > Tok___Attributes_Start ) - { + if ( type > Tok___Attributes_Start ) { ctx->token.Flags |= TF_Attribute; } - if ( type == Tok_Decl_Extern_Linkage ) { skip_whitespace(); - if ( (* ctx->scanner) != '"' ) - { + if ( (* ctx->scanner) != '"' ) { type = Tok_Spec_Extern; ctx->token.Flags |= TF_Specifier; } @@ -459,7 +453,6 @@ void lex_found_token( LexContext* ctx ) array_append( _ctx->Lexer_Tokens, ctx->token ); return; } - if ( ( type <= Tok_Star && type >= Tok_Spec_Alignas) || type == Tok_Ampersand || type == Tok_Ampersand_DBL ) @@ -469,8 +462,6 @@ void lex_found_token( LexContext* ctx ) array_append( _ctx->Lexer_Tokens, ctx->token ); return; } - - if ( type != Tok_Invalid ) { ctx->token.Type = type; @@ -478,21 +469,17 @@ void lex_found_token( LexContext* ctx ) return; } - u64 key = 0; - // if ( (* ctx->scanner) == '(') - // key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len + 1 ); - // else - key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len ); - - Str* define = hashtable_get(_ctx->PreprocessMacros, key ); - if ( define ) + PreprocessorMacro* macro = lookup_preprocess_macro( ctx->token.Text ); + if ( macro ) { - // TODO(Ed): Needs updating (Macros) - ctx->token.Type = Tok_Preprocess_Macro; + ctx->token.Type = macrotype_to_toktype(macro->Type); + // TODO(Ed): When we introduce a macro AST (and expression support), we'll properly lex this section. // Want to ignore any arguments the define may have as they can be execution expressions. if ( ctx->left && (* ctx->scanner) == '(' ) { + ctx->token.Flags |= TF_Macro_Functional; + move_forward(); ctx->token.Text.Len++; @@ -512,7 +499,6 @@ void lex_found_token( LexContext* ctx ) move_forward(); ctx->token.Text.Len++; } - //if ( (* ctx->scanner) == '\r' && ctx->scanner[1] == '\n' ) //{ // move_forward(); @@ -1002,7 +988,7 @@ TokArray lex( Str content ) goto FoundToken; } - // Dash is unfortunatlly a bit more complicated... + // Dash is unfortunately a bit more complicated... case '-': { Str text = { c.scanner, 1 }; @@ -1139,8 +1125,7 @@ TokArray lex( Str content ) c.token.Text = text; move_forward(); - while ( c.left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) - { + while ( c.left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) { move_forward(); c.token.Text.Len++; } @@ -1166,8 +1151,7 @@ TokArray lex( Str content ) move_forward(); c.token.Text.Len++; - while ( c.left && char_is_hex_digit((* ctx->scanner)) ) - { + while ( c.left && char_is_hex_digit((* ctx->scanner)) ) { move_forward(); c.token.Text.Len++; } @@ -1175,8 +1159,7 @@ TokArray lex( Str content ) goto FoundToken; } - while ( c.left && char_is_digit((* ctx->scanner)) ) - { + while ( c.left && char_is_digit((* ctx->scanner)) ) { move_forward(); c.token.Text.Len++; } @@ -1186,8 +1169,7 @@ TokArray lex( Str content ) move_forward(); c.token.Text.Len++; - while ( c.left && char_is_digit((* ctx->scanner)) ) - { + while ( c.left && char_is_digit((* ctx->scanner)) ) { move_forward(); c.token.Text.Len++; } @@ -1205,8 +1187,7 @@ TokArray lex( Str content ) c.token.Text.Len++; // Handle 'll'/'LL' as a special case when we just processed an 'l'/'L' - if (c.left && (prev == 'l' || prev == 'L') && ((* ctx->scanner) == 'l' || (* ctx->scanner) == 'L')) - { + if (c.left && (prev == 'l' || prev == 'L') && ((* ctx->scanner) == 'l' || (* ctx->scanner) == 'L')) { move_forward(); c.token.Text.Len++; } @@ -1232,8 +1213,7 @@ TokArray lex( Str content ) log_failure( "Failed to lex token '%c' (%d, %d)\n%s", (* ctx->scanner), c.line, c.column, context_str ); // Skip to next whitespace since we can't know if anything else is valid until then. - while ( c.left && ! char_is_space( (* ctx->scanner) ) ) - { + while ( c.left && ! char_is_space( (* ctx->scanner) ) ) { move_forward(); } } @@ -1242,18 +1222,15 @@ TokArray lex( Str content ) { lex_found_token( ctx ); TokType last_type = array_back(_ctx->Lexer_Tokens)->Type; - // TODO(Ed): Change this to just detect if its a MACRO THAT SHOULD ACCEPT NEWLINES - if ( last_type == Tok_Preprocess_Macro ) + if ( last_type == Tok_Preprocess_Macro_Stmt ) { Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; c.token = thanks_c; - if ( (* ctx->scanner) == '\r') - { + if ( (* ctx->scanner) == '\r') { move_forward(); c.token.Text.Len = 1; } - - if ( (* ctx->scanner) == '\n' ) + if ( (* ctx->scanner) == '\n' ) { c.token.Type = Tok_NewLine; c.token.Text.Len++; @@ -1266,13 +1243,10 @@ TokArray lex( Str content ) } } - if ( array_num(_ctx->Lexer_Tokens) == 0 ) - { + if ( array_num(_ctx->Lexer_Tokens) == 0 ) { log_failure( "Failed to lex any tokens" ); - { - TokArray tok_array = {}; - return tok_array; - } + TokArray tok_array = {}; + return tok_array; } TokArray result = { _ctx->Lexer_Tokens, 0 }; diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 21c2e14..e911fc1 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -64,12 +64,10 @@ StrBuilder parser_to_strbuilder(ParseContext ctx) s32 level = 0; do { - if ( curr_scope->Name.Ptr ) - { + if ( curr_scope->Name.Ptr ) { strbuilder_append_fmt(& result, "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Len, curr_scope->Name.Ptr ); } - else - { + else { strbuilder_append_fmt(& result, "\t%d: %s\n", level, curr_scope->ProcName.Ptr ); } @@ -82,8 +80,7 @@ StrBuilder parser_to_strbuilder(ParseContext ctx) bool lex__eat(TokArray* self, TokType type ) { - if ( array_num(self->Arr) - self->Idx <= 0 ) - { + if ( array_num(self->Arr) - self->Idx <= 0 ) { log_failure( "No tokens left.\n%s", parser_to_strbuilder(_ctx->parser) ); return false; } @@ -209,7 +206,7 @@ internal Code parse_operator_function_or_variable( bool expects_fu internal CodePragma parse_pragma (); internal CodeParams parse_params ( bool use_template_capture ); internal CodePreprocessCond parse_preprocess_cond (); -internal Code parse_simple_preprocess ( TokType which, bool dont_consume_braces ); +internal Code parse_simple_preprocess ( TokType which ); internal Code parse_static_assert (); internal void parse_template_args ( Token* token ); internal CodeVar parse_variable_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename type, Str name ); @@ -266,8 +263,7 @@ StrBuilder parser_strip_formatting( Str raw_text, bool preserve_newlines ) sptr last_cut = 0; char const* scanner = raw_text.Ptr; - if ( scanner[0] == ' ' ) - { + if ( scanner[0] == ' ' ) { move_fwd(); last_cut = 1; } @@ -477,8 +473,7 @@ StrBuilder parser_strip_formatting( Str raw_text, bool preserve_newlines ) move_fwd(); } - if ( last_cut < raw_text.Len ) - { + if ( last_cut < raw_text.Len ) { strbuilder_append_c_str_len( & content, cut_ptr, raw_text.Len - last_cut ); } @@ -684,8 +679,7 @@ CodeAttributes parse_attributes() internal Code parse_class_struct( TokType which, bool inplace_def ) { - if ( which != Tok_Decl_Class && which != Tok_Decl_Struct ) - { + if ( which != Tok_Decl_Class && which != Tok_Decl_Struct ) { log_failure( "Error, expected class or struct, not %s\n%s", toktype_to_str( which ), parser_to_strbuilder(_ctx->parser) ); return InvalidCode; } @@ -700,8 +694,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) Code result = InvalidCode; - if ( check(Tok_Module_Export) ) - { + if ( check(Tok_Module_Export) ) { mflags = ModuleFlag_Export; eat( Tok_Module_Export ); } @@ -713,8 +706,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) attributes = parse_attributes(); // - if ( check( Tok_Identifier ) ) - { + if ( check( Tok_Identifier ) ) { name = parse_identifier(nullptr); _ctx->parser.Scope->Name = name.Text; } @@ -733,8 +725,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) eat( Tok_Assign_Classifer ); // : - if ( tok_is_access_specifier(currtok) ) - { + if ( tok_is_access_specifier(currtok) ) { access = tok_to_access_specifier(currtok); // : eat( currtok.Type ); @@ -749,8 +740,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) eat( Tok_Comma ); // : , - if ( tok_is_access_specifier(currtok) ) - { + if ( tok_is_access_specifier(currtok) ) { eat(currtok.Type); } Token interface_tok = parse_identifier(nullptr); @@ -760,8 +750,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) } } - if ( check( Tok_BraceCurly_Open ) ) - { + if ( check( Tok_BraceCurly_Open ) ) { body = parse_class_struct_body( which, name ); } // : , ... { } @@ -821,8 +810,6 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); - b32 macro_found = true; - switch ( currtok_noskip.Type ) { case Tok_Statement_End: { @@ -840,73 +827,74 @@ CodeBody parse_class_struct_body( TokType which, Token name ) member = cast(Code, parse_comment()); break; } - case Tok_Access_Public: + case Tok_Access_Public: { member = access_public; eat( Tok_Access_Public ); eat( Tok_Assign_Classifer ); // public: - break; - - case Tok_Access_Protected: + break; + } + case Tok_Access_Protected: { member = access_protected; eat( Tok_Access_Protected ); eat( Tok_Assign_Classifer ); // protected: - break; - - case Tok_Access_Private: + break; + } + case Tok_Access_Private: { member = access_private; eat( Tok_Access_Private ); eat( Tok_Assign_Classifer ); // private: - break; - - case Tok_Decl_Class: + break; + } + case Tok_Decl_Class: { member = parse_complicated_definition( Tok_Decl_Class ); // class - break; - - case Tok_Decl_Enum: + break; + } + case Tok_Decl_Enum: { member = parse_complicated_definition( Tok_Decl_Enum ); // enum - break; - - case Tok_Decl_Friend: + break; + } + case Tok_Decl_Friend: { member = cast(Code, parser_parse_friend()); // friend - break; - - case Tok_Decl_Operator: + break; + } + case Tok_Decl_Operator: { member = cast(Code, parser_parse_operator_cast(NullCode)); // operator () - break; - - case Tok_Decl_Struct: + break; + } + case Tok_Decl_Struct: { member = parse_complicated_definition( Tok_Decl_Struct ); // struct - break; - - case Tok_Decl_Template: + break; + } + case Tok_Decl_Template: { member = cast(Code, parser_parse_template()); // template< ... > - break; - - case Tok_Decl_Typedef: + break; + } + case Tok_Decl_Typedef: { member = cast(Code, parser_parse_typedef()); // typedef - break; - - case Tok_Decl_Union: + break; + } + case Tok_Decl_Union: { member = parse_complicated_definition( Tok_Decl_Union ); // union - break; - - case Tok_Decl_Using: + break; + } + case Tok_Decl_Using: { member = cast(Code, parser_parse_using()); // using - break; - + break; + } case Tok_Operator: + { //if ( currtok.Text[0] != '~' ) //{ // log_failure( "Operator token found in global body but not destructor unary negation\n%s", to_strbuilder(_ctx->parser) ); @@ -915,17 +903,19 @@ CodeBody parse_class_struct_body( TokType which, Token name ) member = cast(Code, parser_parse_destructor(NullCode)); // ~() - break; - - case Tok_Preprocess_Define: + break; + } + case Tok_Preprocess_Define: { member = cast(Code, parse_define()); // #define - break; - + break; + } case Tok_Preprocess_Include: + { member = cast(Code, parse_include()); // #include - break; + break; + } case Tok_Preprocess_If: case Tok_Preprocess_IfDef: @@ -935,38 +925,52 @@ CodeBody parse_class_struct_body( TokType which, Token name ) // # break; - case Tok_Preprocess_Else: + case Tok_Preprocess_Else: { member = cast(Code, preprocess_else); eat( Tok_Preprocess_Else ); // #else - break; - - case Tok_Preprocess_EndIf: + break; + } + case Tok_Preprocess_EndIf: { member = cast(Code, preprocess_endif); eat( Tok_Preprocess_EndIf ); // #endif - break; + break; + } - case Tok_Preprocess_Macro: - // - macro_found = true; - goto Preprocess_Macro_Bare_In_Body; - break; + case Tok_Preprocess_Macro_Stmt: { + member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Macro_Stmt )); + break; + } - case Tok_Preprocess_Pragma: + case Tok_Preprocess_Macro_Expr: { + log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser)); + return InvalidCode; + } + + // case Tok_Preprocess_Macro: + // // + // macro_found = true; + // goto Preprocess_Macro_Bare_In_Body; + // break; + + case Tok_Preprocess_Pragma: { member = cast(Code, parse_pragma()); // #pragma - break; + break; + } - case Tok_Preprocess_Unsupported: + case Tok_Preprocess_Unsupported: { member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported, parser_consume_braces )); // # - break; + break; + } - case Tok_StaticAssert: + case Tok_StaticAssert: { member = parse_static_assert(); // static_assert - break; + break; + } case Tok_Attribute_Open: case Tok_Decl_GNU_Attribute: @@ -1059,6 +1063,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) } //! Fallthrough intentional case Tok_Identifier: + case Tok_Preprocess_Macro_Typename: case Tok_Spec_Const: case Tok_Type_Unsigned: case Tok_Type_Signed: @@ -1079,28 +1084,6 @@ CodeBody parse_class_struct_body( TokType which, Token name ) } } - if (macro_found) - { - Preprocess_Macro_Bare_In_Body: - b32 lone_macro = nexttok.Type == Tok_Statement_End || nexttok_noskip.Type == Tok_NewLine; - if (lone_macro) - { - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Macro, parser_consume_braces )); - // ; - - if ( member == Code_Invalid ) - { - log_failure( "Failed to parse member\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); - return InvalidCode; - } - continue; - } - - // We have a macro but its most likely behaving as a typename - // operator ... // or @@ -1110,13 +1093,11 @@ CodeBody parse_class_struct_body( TokType which, Token name ) default: Token untyped_tok = currtok; - while ( left && currtok.Type != Tok_BraceCurly_Close ) { untyped_tok.Text.Len = ( (sptr)currtok.Text.Ptr + currtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; eat( currtok.Type ); } - member = untyped_str( tok_to_str(untyped_tok) ); // Something unknown break; @@ -1128,7 +1109,6 @@ CodeBody parse_class_struct_body( TokType which, Token name ) parser_pop(& _ctx->parser); return InvalidCode; } - body_append(result, member ); } @@ -1320,19 +1300,46 @@ CodeDefine parse_define() CodeDefine define = (CodeDefine) make_code(); define->Type = CT_Preprocess_Define; - - if ( ! check( Tok_Identifier ) ) - { + if ( ! check( Tok_Identifier ) ) { log_failure( "Error, expected identifier after #define\n%s", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; } - _ctx->parser.Scope->Name = currtok.Text; define->Name = cache_str( tok_to_str(currtok) ); eat( Tok_Identifier ); // #define + PreprocessorMacro* macro = lookup_preprocess_macro(define->Name); + if (macro_is_functional(* macro)) { + eat( Tok_Capture_Start ); + // #define ( + CodeDefineParams params; + if ( left && currtok.Type != Tok_Capture_End ) { + params = (CodeDefineParams) make_code(); + params.Type = CT_Parameters_Define; + params.Name = currtok.text; + + define->params = params; + + eat( Tok_Preprocess_Define_Param ); + // #define ( ) + } + + while( left && currtok.Type != Tok_Capture_End ) { + CodeDefineParams next_param = (CodeDefineParams) make_code(); + next_param.Type = CT_Parameters_Define; + next_param.Name = currtok.Text; + define_params_append(params, next_param); + + // #define ( , ... + eat( Tok_Preprocess_Define_Param ); + } + + eat( Tok_Capture_End ); + // #define ( ) + } + if ( ! check( Tok_Preprocess_Content )) { log_failure( "Error, expected content after #define %s\n%s", define->Name, parser_to_strbuilder(_ctx->parser) ); @@ -1344,7 +1351,7 @@ CodeDefine parse_define() { define->Content = cache_str( tok_to_str(currtok) ); eat( Tok_Preprocess_Content ); - // #define + // #define ( ) parser_pop(& _ctx->parser); return define; @@ -1352,7 +1359,7 @@ CodeDefine parse_define() define->Content = cache_str( strbuilder_to_str( parser_strip_formatting( tok_to_str(currtok), parser_strip_formatting_dont_preserve_newlines )) ); eat( Tok_Preprocess_Content ); - // #define + // #define ( ) parser_pop(& _ctx->parser); return define; @@ -2519,24 +2526,10 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes Code result = InvalidCode; -#ifndef GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES - b32 lone_macro = false; - - if ( currtok.Type == Tok_Preprocess_Macro && ( nexttok.Type == Tok_Statement_End || nexttok.Type == Tok_Comment ) ) - { - // Were dealing with a lone macro after attributes/specifiers, there was a end statement ';' after. - result = parse_simple_preprocess( Tok_Preprocess_Macro, parser_consume_braces ); - parser_pop(& _ctx->parser); - return result; - // - } -#endif - CodeTypename type = parser_parse_type( parser_not_from_template, nullptr ); // - if ( type == InvalidCode ) - { + if ( type == InvalidCode ) { parser_pop(& _ctx->parser); return InvalidCode; } @@ -2582,7 +2575,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes // Example : // idx +1 +2 bool detected_comma = _ctx->parser.Tokens.Arr[ _ctx->parser.Tokens.Idx + 2 ].Type == Tok_Comma; - if ( detected_capture && ! detected_comma ) + if ( detected_capture && ! detected_comma ) { // Dealing with a function result = cast(Code, parse_function_after_name( ModuleFlag_None, attributes, specifiers, type, name )); @@ -2590,13 +2583,11 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes } else { - if ( expects_function ) - { + if ( expects_function ) { log_failure( "Expected function declaration (consteval was used)\n%s", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; } - // Dealing with a variable result = cast(Code, parse_variable_after_name( ModuleFlag_None, attributes, specifiers, type, tok_to_str(name) )); // ... @@ -2618,8 +2609,7 @@ CodePragma parse_pragma() eat( Tok_Preprocess_Pragma ); // #pragma - if ( ! check( Tok_Preprocess_Content )) - { + if ( ! check( Tok_Preprocess_Content )) { log_failure( "Error, expected content after #pragma\n%s", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; @@ -2640,15 +2630,14 @@ CodeParams parse_params( bool use_template_capture ) { push_scope(); - if ( ! use_template_capture ) + if ( ! use_template_capture ) { eat( Tok_Capture_Start ); - // ( - - else - { + // ( + } + else { if ( check( Tok_Operator ) && currtok.Text.Ptr[ 0 ] == '<' ) eat( Tok_Operator ); - // < + // < } if ( ! use_template_capture && check( Tok_Capture_End ) ) @@ -2658,7 +2647,7 @@ CodeParams parse_params( bool use_template_capture ) parser_pop(& _ctx->parser); return NullCode; } - else if ( check( Tok_Operator ) && currtok.Text.Ptr[ 0 ] == '>' ) + else if ( check( Tok_Operator ) && currtok.Text.Ptr[ 0 ] == '>' ) { eat( Tok_Operator ); // > @@ -2691,9 +2680,9 @@ CodeParams parse_params( bool use_template_capture ) // COREUOBJECT_API void CallFunction( FFrame& Stack, RESULT_DECL, UFunction* Function ); // and: vvvv // AddComponentByClass(UPARAM(meta = (AllowAbstract = "false")) TSubclassOf Class, bool bManualAttachment, ... - if ( check(Tok_Preprocess_Macro)) + if ( check(Tok_Preprocess_Macro_Expr)) { - macro = parse_simple_preprocess(Tok_Preprocess_Macro, parser_consume_braces); + macro = parse_simple_preprocess(Tok_Preprocess_Macro_Expr); // ( } if ( currtok.Type != Tok_Comma ) @@ -2718,9 +2707,10 @@ CodeParams parse_params( bool use_template_capture ) // template::Value)> // class T ... and then ^this^ UE_REQUIRES shows up // So we need to consume that. - if ( check( Tok_Preprocess_Macro )) + if ( check( Tok_Preprocess_Macro_Expr )) { - post_name_macro = parse_simple_preprocess( Tok_Preprocess_Macro, parser_consume_braces ); + post_name_macro = parse_simple_preprocess( Tok_Preprocess_Macro_Expr ); + // ( } // In template captures you can have a typename have direct assignment without a name @@ -2729,12 +2719,11 @@ CodeParams parse_params( bool use_template_capture ) if ( ( name.Text.Ptr || use_template_capture ) && bitfield_is_set( u32, currtok.Flags, TF_Assign ) ) { eat( Tok_Operator ); - // ( = + // ( = Token value_tok = currtok; - if ( currtok.Type == Tok_Comma ) - { + if ( currtok.Type == Tok_Comma ) { log_failure( "Expected value after assignment operator\n%s.", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; @@ -2802,10 +2791,10 @@ CodeParams parse_params( bool use_template_capture ) // COREUOBJECT_API void CallFunction( FFrame& Stack, RESULT_DECL, UFunction* Function ); // and: vvvv // AddComponentByClass(UPARAM(meta = (AllowAbstract = "false")) TSubclassOf Class, bool bManualAttachment, ... - if ( check(Tok_Preprocess_Macro)) + if ( check(Tok_Preprocess_Macro_Expr)) { - macro = parse_simple_preprocess(Tok_Preprocess_Macro, parser_dont_consume_braces); - // ( + macro = parse_simple_preprocess(Tok_Preprocess_Macro_Expr); + // ( = , } if ( currtok.Type != Tok_Comma ) { @@ -2830,9 +2819,9 @@ CodeParams parse_params( bool use_template_capture ) // template::Value)> // class T ... and then ^this^ UE_REQUIRES shows up // So we need to consume that. - if ( check( Tok_Preprocess_Macro )) - { - post_name_macro = parse_simple_preprocess( Tok_Preprocess_Macro, parser_dont_consume_braces ); + if ( check( Tok_Preprocess_Macro_Expr )) { + post_name_macro = parse_simple_preprocess( Tok_Preprocess_Macro_Expr ); + // ( = , } // In template captures you can have a typename have direct assignment without a name @@ -2841,12 +2830,11 @@ CodeParams parse_params( bool use_template_capture ) if ( ( name.Text.Ptr || use_template_capture ) && bitfield_is_set( u32, currtok.Flags, TF_Assign ) ) { eat( Tok_Operator ); - // ( = , = + // ( = , = Token value_tok = currtok; - if ( currtok.Type == Tok_Comma ) - { + if ( currtok.Type == Tok_Comma ) { log_failure( "Expected value after assignment operator\n%s", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; @@ -2878,9 +2866,9 @@ CodeParams parse_params( bool use_template_capture ) } value = untyped_str( strbuilder_to_str(parser_strip_formatting( tok_to_str(value_tok), parser_strip_formatting_dont_preserve_newlines )) ); - // ( = , = + // ( = , = } - // ( = , = , .. + // ( = , = , .. } CodeParams param = ( CodeParams )make_code(); @@ -2901,21 +2889,20 @@ CodeParams parse_params( bool use_template_capture ) } if ( ! use_template_capture ) + { eat( Tok_Capture_End ); - // ( = , = , .. ) - + // ( = , = , .. ) + } else { - if ( ! check( Tok_Operator ) || currtok.Text.Ptr[ 0 ] != '>' ) - { + if ( ! check( Tok_Operator ) || currtok.Text.Ptr[ 0 ] != '>' ) { log_failure( "Expected '<' after 'template' keyword\n%s", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; } eat( Tok_Operator ); - // < = , = , .. > + // < = , = , .. > } - parser_pop(& _ctx->parser); return result; #undef context @@ -2963,11 +2950,13 @@ Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) // If we want it to terminate or have an inline comment we can possbily use its parent typedef for that info... push_scope(); - Token tok = currtok; + Token full_macro = currtok; eat( which ); // - if ( ! dont_consume_braces && peektok.Type == Tok_BraceCurly_Open ) + PreprocessorMacro macro = * lookup_preprocess_macro( name.Text ); + + if ( macro_expects_body(macro) && peektok.Type == Tok_BraceCurly_Open ) { // Eat the block scope right after the macro. Were assuming the macro defines a function definition's signature eat( Tok_BraceCurly_Open ); @@ -2988,8 +2977,9 @@ Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) eat( Tok_BraceCurly_Close ); // { } + // TODO(Ed): Review this? Str prev_proc = _ctx->parser.Scope->Prev->ProcName; - if ( c_str_compare_len( prev_proc.Ptr, "parser_parse_typedef", prev_proc.Len ) != 0 ) + if ( macro.Type == MT_Typename && c_str_compare_len( prev_proc.Ptr, "parser_parse_typedef", prev_proc.Len ) != 0 ) { if ( check( Tok_Statement_End )) { @@ -3003,7 +2993,7 @@ Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) } } - tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)tok.Text.Ptr; + full_macro.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)full_macro.Text.Ptr; } else { @@ -3016,16 +3006,15 @@ Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) // Do nothing goto Leave_Scope_Early; } - else if (str_contains(_ctx->parser.Scope->Prev->ProcName, txt("parser_parse_typedef"))) + else if (macro.Type == MT_Typename && str_contains(_ctx->parser.Scope->Prev->ProcName, txt("parser_parse_typedef"))) { - // TODO(Ed): Reveiw the context for this? if ( peektok.Type == Tok_Statement_End ) { Token stmt_end = currtok; eat( Tok_Statement_End ); // ; - tok.Text.Len += prevtok.Text.Len; + full_macro.Text.Len += prevtok.Text.Len; // TODO(Ed): Reveiw the context for this? (ESPECIALLY THIS) if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) @@ -3033,7 +3022,7 @@ Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) eat( Tok_Comment ); // ; - tok.Text.Len += prevtok.Text.Len; + full_macro.Text.Len += prevtok.Text.Len; } } } @@ -3047,18 +3036,13 @@ Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) Token stmt_end = currtok; eat( Tok_Statement_End ); // ; - tok.Text.Len += prevtok.Text.Len; + full_macro.Text.Len += prevtok.Text.Len; } - } - // tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)tok.Text; } Leave_Scope_Early: - - char const* content = c_str_fmt_buf( "%.*s ", tok.Text.Len, tok.Text.Ptr ); - - Code result = untyped_str( to_str_from_c_str(content) ); + Code result = untyped_str( full_macro.Text ); _ctx->parser.Scope->Name = tok.Text; parser_pop(& _ctx->parser); @@ -3154,19 +3138,18 @@ CodeVar parse_variable_after_name( , CodeAttributes attributes , CodeSpecifiers specifiers , CodeTypename type - , Str name + , Str name ) { push_scope(); Code array_expr = parse_array_decl(); - Code expr = { nullptr }; - Code bitfield_expr = { nullptr }; + Code expr = NullCode; + Code bitfield_expr = NulLCode; b32 using_constructor_initializer = false; - if ( bitfield_is_set( u32, currtok.Flags, TF_Assign ) ) - { + if ( bitfield_is_set( u32, currtok.Flags, TF_Assign ) ) { // = expr = parse_assignment_expression(); } @@ -3230,15 +3213,13 @@ CodeVar parse_variable_after_name( Token expr_tok = currtok; - if ( currtok.Type == Tok_Statement_End ) - { + if ( currtok.Type == Tok_Statement_End ) { log_failure( "Expected expression after bitfield \n%SB", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; } - while ( left && currtok.Type != Tok_Statement_End ) - { + while ( left && currtok.Type != Tok_Statement_End ) { eat( currtok.Type ); } @@ -3252,8 +3233,7 @@ CodeVar parse_variable_after_name( CodeComment inline_cmt = NullCode; if ( type ) { - if ( currtok.Type == Tok_Comma ) - { + if ( currtok.Type == Tok_Comma ) { // Were dealing with a statement with more than one declaration // This is only handled this way if its the first declaration // Otherwise its looped through in parse_variable_declaration_list @@ -3272,8 +3252,7 @@ CodeVar parse_variable_after_name( // { }, ...; // Check for inline comment : = ; // - if ( left && ( currtok_noskip.Type == Tok_Comment ) && currtok_noskip.Line == stmt_end.Line ) - { + if ( left && ( currtok_noskip.Type == Tok_Comment ) && currtok_noskip.Line == stmt_end.Line ) { inline_cmt = parse_comment(); // : , ...; // = , ...; @@ -3286,35 +3265,20 @@ CodeVar parse_variable_after_name( result->Type = CT_Variable; result->Name = cache_str( name ); result->ModuleFlags = mflags; + result->ValueType = type; + result->BitfieldSize = bitfield_expr; + result->Attributes = attributes; + result->Specs = specifiers; + result->Value = expr; + result->InlineCmt = inline_cmt; - // Type can be null if we're dealing with a declaration from a variable declaration-list - if ( type ) - result->ValueType = type; - - if (array_expr ) + if (array_expr) type->ArrExpr = array_expr; - if ( bitfield_expr ) - result->BitfieldSize = bitfield_expr; - - if ( attributes ) - result->Attributes = attributes; - - if ( specifiers ) - result->Specs = specifiers; - - if ( expr ) - result->Value = expr; - - if ( inline_cmt ) - result->InlineCmt = inline_cmt; - - if ( next_var ) - { + if ( next_var ) { result->NextVar = next_var; result->NextVar->Parent = cast(Code, result); } - result->VarParenthesizedInit = using_constructor_initializer; parser_pop(& _ctx->parser); @@ -3560,7 +3524,7 @@ CodeDestructor parser_parse_destructor( CodeSpecifiers specifiers ) specifiers_append(specifiers, Spec_Pure ); } - else if ( left && c_str_compare_len( upcoming.Text.Ptr, "default", sizeof("default") - 1 ) == 0) + else if ( left && c_str_compare_len( upcoming.Text.Ptr, "default", sizeof("default") - 1 ) == 0) { body = cast(CodeBody, parse_assignment_expression()); // ~< @@ -3576,8 +3540,10 @@ CodeDestructor parser_parse_destructor( CodeSpecifiers specifiers ) } if ( ! pure_virtual && check( Tok_BraceCurly_Open ) ) + { body = cast(CodeBody, parse_function_body()); - // ~() { ... } + // ~() { ... } + } else { Token stmt_end = currtok; @@ -3591,8 +3557,7 @@ CodeDestructor parser_parse_destructor( CodeSpecifiers specifiers ) CodeDestructor result = ( CodeDestructor )make_code(); - if ( tok_is_valid(prefix_identifier) ) - { + if ( tok_is_valid(prefix_identifier) ) { prefix_identifier.Text.Len += 1 + identifier.Text.Len; result->Name = cache_str( tok_to_str(prefix_identifier) ); } @@ -3600,8 +3565,7 @@ CodeDestructor parser_parse_destructor( CodeSpecifiers specifiers ) if ( specifiers ) result->Specs = specifiers; - if ( body && body->Type == CT_Function_Body ) - { + if ( body && body->Type == CT_Function_Body ) { result->Body = cast(Code, body); result->Type = CT_Destructor; } @@ -3740,10 +3704,11 @@ CodeEnum parser_parse_enum( bool inplace_def ) eat( Tok_Preprocess_EndIf ); break; - case Tok_Preprocess_Macro: - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Macro, parser_consume_braces)); + case Tok_Preprocess_Macro_Stmt: { + member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Macro_Stmt )); // - break; + break; + } case Tok_Preprocess_Pragma: member = cast(Code, parse_pragma()); @@ -3751,11 +3716,12 @@ CodeEnum parser_parse_enum( bool inplace_def ) break; case Tok_Preprocess_Unsupported: - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported, parser_consume_braces )); + member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported )); // # break; default: + { Token entry = currtok; eat( Tok_Identifier); @@ -3766,17 +3732,15 @@ CodeEnum parser_parse_enum( bool inplace_def ) eat( Tok_Operator ); // = - while ( currtok.Type != Tok_Comma && currtok.Type != Tok_BraceCurly_Close ) - { + while ( currtok.Type != Tok_Comma && currtok.Type != Tok_BraceCurly_Close ) { eat( currtok.Type ); } } // = // Unreal UMETA macro support - if ( currtok.Type == Tok_Preprocess_Macro ) - { - eat( Tok_Preprocess_Macro ); + if ( currtok.Type == Tok_Preprocess_Macro_Expr ) { + eat( Tok_Preprocess_Macro_Expr ); // = } @@ -3800,11 +3764,11 @@ CodeEnum parser_parse_enum( bool inplace_def ) entry.Text.Len = ( (sptr)prev.Text.Ptr + prev.Text.Len ) - (sptr)entry.Text.Ptr; member = untyped_str( tok_to_str(entry) ); + } break; } - if ( member == Code_Invalid ) - { + if ( member == Code_Invalid ) { log_failure( "Failed to parse member\n%s", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; @@ -3953,8 +3917,7 @@ CodeFriend parser_parse_friend() eat( currtok.Type ); } - if ( NumSpecifiers ) - { + if ( NumSpecifiers ) { specifiers = def_specifiers( NumSpecifiers, specs_found ); } // @@ -3962,8 +3925,7 @@ CodeFriend parser_parse_friend() // Type declaration or return type CodeTypename type = parser_parse_type(parser_not_from_template, nullptr); - if ( cast(Code, type) == Code_Invalid ) - { + if ( cast(Code, type) == Code_Invalid ) { parser_pop(& _ctx->parser); return InvalidCode; } @@ -3993,8 +3955,7 @@ CodeFriend parser_parse_friend() } // Operator declaration or definition - if ( currtok.Type == Tok_Decl_Operator ) - { + if ( currtok.Type == Tok_Decl_Operator ) { op = parse_operator_after_ret_type( ModuleFlag_None, NullCode, specifiers, type ); } @@ -4041,8 +4002,7 @@ CodeFn parser_parse_function() CodeSpecifiers specifiers = { nullptr }; ModuleFlag mflags = ModuleFlag_None; - if ( check(Tok_Module_Export) ) - { + if ( check(Tok_Module_Export) ) { mflags = ModuleFlag_Export; eat( Tok_Module_Export ); } @@ -4074,15 +4034,13 @@ CodeFn parser_parse_function() eat( currtok.Type ); } - if ( NumSpecifiers ) - { + if ( NumSpecifiers ) { specifiers = def_specifiers( NumSpecifiers, specs_found ); } // CodeTypename ret_type = parser_parse_type(parser_not_from_template, nullptr); - if ( cast(Code, ret_type) == Code_Invalid ) - { + if ( cast(Code, ret_type) == Code_Invalid ) { parser_pop(& _ctx->parser); return InvalidCode; } @@ -4090,8 +4048,7 @@ CodeFn parser_parse_function() Token name = parse_identifier(nullptr); _ctx->parser.Scope->Name = name.Text; - if ( ! tok_is_valid(name) ) - { + if ( ! tok_is_valid(name) ) { parser_pop(& _ctx->parser); return InvalidCode; } @@ -4117,8 +4074,7 @@ CodeNS parser_parse_namespace() // namespace CodeBody body = parse_global_nspace( CT_Namespace_Body ); - if ( cast(Code, body) == Code_Invalid ) - { + if ( cast(Code, body) == Code_Invalid ) { parser_pop(& _ctx->parser); return InvalidCode; } @@ -4147,8 +4103,7 @@ CodeOperator parser_parse_operator() Specifier specs_found[16] = { Spec_NumSpecifiers }; s32 NumSpecifiers = 0; - if ( check(Tok_Module_Export) ) - { + if ( check(Tok_Module_Export) ) { mflags = ModuleFlag_Export; eat( Tok_Module_Export ); } @@ -4180,8 +4135,7 @@ CodeOperator parser_parse_operator() eat( currtok.Type ); } - if ( NumSpecifiers ) - { + if ( NumSpecifiers ) { specifiers = def_specifiers( NumSpecifiers, specs_found ); } // @@ -4292,13 +4246,11 @@ CodeOpCast parser_parse_operator_cast( CodeSpecifiers specifiers ) if ( tok_is_valid(name) ) result->Name = cache_str( tok_to_str(name) ); - if (body) - { + if (body) { result->Type = CT_Operator_Cast; result->Body = cast(CodeBody, body); } - else - { + else { result->Type = CT_Operator_Cast_Fwd; } @@ -4329,8 +4281,7 @@ CodeTemplate parser_parse_template() ModuleFlag mflags = ModuleFlag_None; - if ( check( Tok_Module_Export ) ) - { + if ( check( Tok_Module_Export ) ) { mflags = ModuleFlag_Export; eat( Tok_Module_Export ); } @@ -4340,8 +4291,7 @@ CodeTemplate parser_parse_template() // template CodeParams params = parse_params( UseTemplateCapture ); - if ( cast(Code, params) == Code_Invalid ) - { + if ( cast(Code, params) == Code_Invalid ) { parser_pop(& _ctx->parser); return InvalidCode; } @@ -4423,8 +4373,7 @@ CodeTemplate parser_parse_template() eat( currtok.Type ); } - if ( NumSpecifiers ) - { + if ( NumSpecifiers ) { specifiers = def_specifiers( NumSpecifiers, specs_found ); } // template< > @@ -4437,7 +4386,7 @@ CodeTemplate parser_parse_template() if (is_in_global_nspace) { Code constructor_destructor = parse_global_nspace_constructor_destructor( specifiers ); - if ( constructor_destructor ) + if ( constructor_destructor ) { definition = constructor_destructor; // :: () { ... } @@ -4471,8 +4420,7 @@ CodeTemplate parser_parse_template() break; } - if ( found_operator_cast_outside_class_implmentation ) - { + if ( found_operator_cast_outside_class_implmentation ) { definition = cast(Code, parser_parse_operator_cast( specifiers )); // :: operator () { ... } break; @@ -4531,8 +4479,7 @@ CodeTypename parser_parse_type( bool from_template, bool* typedef_is_function ) { Specifier spec = str_to_specifier( tok_to_str(currtok) ); - if ( spec != Spec_Const ) - { + if ( spec != Spec_Const ) { log_failure( "Error, invalid specifier used in type definition: %S\n%SB", tok_to_str(currtok), parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; @@ -4544,8 +4491,7 @@ CodeTypename parser_parse_type( bool from_template, bool* typedef_is_function ) } // - if ( left == 0 ) - { + if ( left == 0 ) { log_failure( "Error, unexpected end of type definition\n%SB", parser_to_strbuilder(_ctx->parser) ); parser_pop(& _ctx->parser); return InvalidCode; @@ -4645,16 +4591,16 @@ else if ( currtok.Type == Tok_DeclType ) } } // TODO(Ed): This needs updating - else if ( currtok.Type == Tok_Preprocess_Macro ) { + else if ( currtok.Type == Tok_Preprocess_Macro_Typename ) { // Typename is a macro name = currtok; - eat(Tok_Preprocess_Macro); + eat(Tok_Preprocess_Macro_Typename); } // The usual Identifier type signature that may have namespace qualifiers else { - name = parse_identifier(nullptr); + name = parse_identifier(nullptr); _ctx->parser.Scope->Name = name.Text; if ( ! tok_is_valid(name) ) { @@ -4992,13 +4938,13 @@ CodeTypedef parser_parse_typedef() #if GEN_PARSER_DISABLE_MACRO_TYPEDEF if ( false ) #else - if ( check( Tok_Preprocess_Macro )) + if ( check( Tok_Preprocess_Macro_Typename )) #endif { type = cast(Code, t_empty); name = currtok; _ctx->parser.Scope->Name = name.Text; - eat( Tok_Preprocess_Macro ); + eat( Tok_Preprocess_Macro_Typename ); // typedef if ( currtok.Type == Tok_Identifier ) @@ -5201,7 +5147,7 @@ CodeUnion parser_parse_union( bool inplace_def ) Str name = { nullptr, 0 }; if ( check( Tok_Identifier ) ) -{ + { name = tok_to_str(currtok); _ctx->parser.Scope->Name = currtok.Text; eat( Tok_Identifier ); @@ -5274,14 +5220,13 @@ CodeUnion parser_parse_union( bool inplace_def ) eat( Tok_Preprocess_EndIf ); break; - case Tok_Preprocess_Macro: - if ( nexttok.Type == Tok_Identifier ) { - // Its a variable with a macro typename - member = cast(Code, parser_parse_variable()); - break; - } + case Tok_Preprocess_Macro_Typename: + // Its a variable with a macro typename + member = cast(Code, parser_parse_variable()); + break; - member = parse_simple_preprocess( Tok_Preprocess_Macro, parser_consume_braces ); + case Tok_Preprocess_Macro_Stmt: + member = parse_simple_preprocess( Tok_Preprocess_Macro ); break; case Tok_Preprocess_Pragma: @@ -5352,7 +5297,7 @@ CodeUsing parser_parse_using() eat( Tok_Decl_Using ); // using - if ( currtok.Type == Tok_Decl_Namespace ) + if ( currtok.Type == Tok_Decl_Namespace ) { is_namespace = true; eat( Tok_Decl_Namespace ); @@ -5387,8 +5332,7 @@ CodeUsing parser_parse_using() // using = ; CodeComment inline_cmt = NullCode; - if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) - { + if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) { inline_cmt = parse_comment(); } // using = ; @@ -5435,8 +5379,7 @@ CodeVar parser_parse_variable() CodeAttributes attributes = { nullptr }; CodeSpecifiers specifiers = { nullptr }; - if ( check(Tok_Module_Export) ) - { + if ( check(Tok_Module_Export) ) { mflags = ModuleFlag_Export; eat( Tok_Module_Export ); } @@ -5468,8 +5411,7 @@ CodeVar parser_parse_variable() eat( currtok.Type ); } - if ( NumSpecifiers ) - { + if ( NumSpecifiers ) { specifiers = def_specifiers( NumSpecifiers, specs_found ); } // diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index d829850..c7eeae3 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -138,7 +138,18 @@ enum MacroType : u16 MF_UnderlyingType = GEN_U16_Max, }; -Str macro_type_to_str( MacroType type ) +forceinline +TokenType macrotype_to_toktype( MacroType type ) { + switch ( type ) { + case MT_Statement : return Tok_Preprocess_Macro_Stmt; + case MT_Expression : return Tok_Preprocess_Macro_Expr; + case MT_Typename : return Tok_Preprocess_Macro_Typename; + } + // All others unsupported for now. + return Tok_Invalid; +} + +Str macrotype_to_str( MacroType type ) { local_persist Str lookup[ (u32)Num_ModuleFlags ] = { @@ -181,4 +192,4 @@ b32 macro_is_functional( PreprocessorMacro macro ) { forceinline b32 macro_expects_body( PreprocessorMacro macro ) { return bitfield_is_set( macro->Flags, MF_Expects_Body ); -} +} \ No newline at end of file diff --git a/base/enums/ETokType.csv b/base/enums/ETokType.csv index bdf1d03..5ce4c78 100644 --- a/base/enums/ETokType.csv +++ b/base/enums/ETokType.csv @@ -51,7 +51,7 @@ Preprocess_Else, "else" Preprocess_EndIf, "endif" Preprocess_Include, "include" Preprocess_Pragma, "pragma" -Preprocess_Content, "__macro_content__" +Preprocess_Content, "__macro_content__" Preprocess_Macro_Expr, "__macro_expression__" Preprocess_Macro_Stmt, "__macro_statment__" Preprocess_Macro_Typename, "__macro_typename__" @@ -64,7 +64,7 @@ Spec_Constinit, "constinit" Spec_Explicit, "explicit" Spec_Extern, "extern" Spec_Final, "final" -Spec_ForceInline, "forceinline" +Spec_ForceInline, "forceinline" Spec_Global, "global" Spec_Inline, "inline" Spec_Internal_Linkage, "internal" diff --git a/base/helpers/base_codegen.hpp b/base/helpers/base_codegen.hpp index 8b69c89..90fb32a 100644 --- a/base/helpers/base_codegen.hpp +++ b/base/helpers/base_codegen.hpp @@ -499,35 +499,36 @@ CodeBody gen_ast_inlines() #pragma pop_macro("GEN_NS") #pragma pop_macro("CodeInvalid") - CodeBody impl_code = parse_global_body( token_fmt( "typename", Str name(Code), code_impl_tmpl )); - CodeBody impl_code_body = parse_global_body( token_fmt( "typename", Str name(CodeBody), code_impl_tmpl )); - CodeBody impl_code_attr = parse_global_body( token_fmt( "typename", Str name(CodeAttributes), code_impl_tmpl )); - CodeBody impl_code_cmt = parse_global_body( token_fmt( "typename", Str name(CodeComment), code_impl_tmpl )); - CodeBody impl_code_constr = parse_global_body( token_fmt( "typename", Str name(CodeConstructor), code_impl_tmpl )); - CodeBody impl_code_class = parse_global_body( token_fmt( "typename", Str name(CodeClass), code_impl_tmpl )); - CodeBody impl_code_define = parse_global_body( token_fmt( "typename", Str name(CodeDefine), code_impl_tmpl )); - CodeBody impl_code_destruct = parse_global_body( token_fmt( "typename", Str name(CodeDestructor), code_impl_tmpl )); - CodeBody impl_code_enum = parse_global_body( token_fmt( "typename", Str name(CodeEnum), code_impl_tmpl )); - CodeBody impl_code_exec = parse_global_body( token_fmt( "typename", Str name(CodeExec), code_impl_tmpl )); - CodeBody impl_code_extern = parse_global_body( token_fmt( "typename", Str name(CodeExtern), code_impl_tmpl )); - CodeBody impl_code_include = parse_global_body( token_fmt( "typename", Str name(CodeInclude), code_impl_tmpl )); - CodeBody impl_code_friend = parse_global_body( token_fmt( "typename", Str name(CodeFriend), code_impl_tmpl )); - CodeBody impl_code_fn = parse_global_body( token_fmt( "typename", Str name(CodeFn), code_impl_tmpl )); - CodeBody impl_code_module = parse_global_body( token_fmt( "typename", Str name(CodeModule), code_impl_tmpl )); - CodeBody impl_code_ns = parse_global_body( token_fmt( "typename", Str name(CodeNS), code_impl_tmpl )); - CodeBody impl_code_op = parse_global_body( token_fmt( "typename", Str name(CodeOperator), code_impl_tmpl )); - CodeBody impl_code_opcast = parse_global_body( token_fmt( "typename", Str name(CodeOpCast), code_impl_tmpl )); - CodeBody impl_code_params = parse_global_body( token_fmt( "typename", Str name(CodeParams), code_impl_tmpl )); - CodeBody impl_code_pragma = parse_global_body( token_fmt( "typename", Str name(CodePragma), code_impl_tmpl )); - CodeBody impl_code_precond = parse_global_body( token_fmt( "typename", Str name(CodePreprocessCond), code_impl_tmpl )); - CodeBody impl_code_specs = parse_global_body( token_fmt( "typename", Str name(CodeSpecifiers), code_impl_tmpl )); - CodeBody impl_code_struct = parse_global_body( token_fmt( "typename", Str name(CodeStruct), code_impl_tmpl )); - CodeBody impl_code_tmpl = parse_global_body( token_fmt( "typename", Str name(CodeTemplate), code_impl_tmpl )); - CodeBody impl_code_type = parse_global_body( token_fmt( "typename", Str name(CodeTypename), code_impl_tmpl )); - CodeBody impl_code_typedef = parse_global_body( token_fmt( "typename", Str name(CodeTypedef), code_impl_tmpl )); - CodeBody impl_code_union = parse_global_body( token_fmt( "typename", Str name(CodeUnion), code_impl_tmpl )); - CodeBody impl_code_using = parse_global_body( token_fmt( "typename", Str name(CodeUsing), code_impl_tmpl )); - CodeBody impl_code_var = parse_global_body( token_fmt( "typename", Str name(CodeVar), code_impl_tmpl )); + CodeBody impl_code = parse_global_body( token_fmt( "typename", Str name(Code), code_impl_tmpl )); + CodeBody impl_code_body = parse_global_body( token_fmt( "typename", Str name(CodeBody), code_impl_tmpl )); + CodeBody impl_code_attr = parse_global_body( token_fmt( "typename", Str name(CodeAttributes), code_impl_tmpl )); + CodeBody impl_code_cmt = parse_global_body( token_fmt( "typename", Str name(CodeComment), code_impl_tmpl )); + CodeBody impl_code_constr = parse_global_body( token_fmt( "typename", Str name(CodeConstructor), code_impl_tmpl )); + CodeBody impl_code_class = parse_global_body( token_fmt( "typename", Str name(CodeClass), code_impl_tmpl )); + CodeBody impl_code_define = parse_global_body( token_fmt( "typename", Str name(CodeDefine), code_impl_tmpl )); + CodeBody impl_code_define_params = parse_global_body( token_fmt( "typename", Str name(CodeDefineParams), code_impl_tmpl )); + CodeBody impl_code_destruct = parse_global_body( token_fmt( "typename", Str name(CodeDestructor), code_impl_tmpl )); + CodeBody impl_code_enum = parse_global_body( token_fmt( "typename", Str name(CodeEnum), code_impl_tmpl )); + CodeBody impl_code_exec = parse_global_body( token_fmt( "typename", Str name(CodeExec), code_impl_tmpl )); + CodeBody impl_code_extern = parse_global_body( token_fmt( "typename", Str name(CodeExtern), code_impl_tmpl )); + CodeBody impl_code_include = parse_global_body( token_fmt( "typename", Str name(CodeInclude), code_impl_tmpl )); + CodeBody impl_code_friend = parse_global_body( token_fmt( "typename", Str name(CodeFriend), code_impl_tmpl )); + CodeBody impl_code_fn = parse_global_body( token_fmt( "typename", Str name(CodeFn), code_impl_tmpl )); + CodeBody impl_code_module = parse_global_body( token_fmt( "typename", Str name(CodeModule), code_impl_tmpl )); + CodeBody impl_code_ns = parse_global_body( token_fmt( "typename", Str name(CodeNS), code_impl_tmpl )); + CodeBody impl_code_op = parse_global_body( token_fmt( "typename", Str name(CodeOperator), code_impl_tmpl )); + CodeBody impl_code_opcast = parse_global_body( token_fmt( "typename", Str name(CodeOpCast), code_impl_tmpl )); + CodeBody impl_code_params = parse_global_body( token_fmt( "typename", Str name(CodeParams), code_impl_tmpl )); + CodeBody impl_code_pragma = parse_global_body( token_fmt( "typename", Str name(CodePragma), code_impl_tmpl )); + CodeBody impl_code_precond = parse_global_body( token_fmt( "typename", Str name(CodePreprocessCond), code_impl_tmpl )); + CodeBody impl_code_specs = parse_global_body( token_fmt( "typename", Str name(CodeSpecifiers), code_impl_tmpl )); + CodeBody impl_code_struct = parse_global_body( token_fmt( "typename", Str name(CodeStruct), code_impl_tmpl )); + CodeBody impl_code_tmpl = parse_global_body( token_fmt( "typename", Str name(CodeTemplate), code_impl_tmpl )); + CodeBody impl_code_type = parse_global_body( token_fmt( "typename", Str name(CodeTypename), code_impl_tmpl )); + CodeBody impl_code_typedef = parse_global_body( token_fmt( "typename", Str name(CodeTypedef), code_impl_tmpl )); + CodeBody impl_code_union = parse_global_body( token_fmt( "typename", Str name(CodeUnion), code_impl_tmpl )); + CodeBody impl_code_using = parse_global_body( token_fmt( "typename", Str name(CodeUsing), code_impl_tmpl )); + CodeBody impl_code_var = parse_global_body( token_fmt( "typename", Str name(CodeVar), code_impl_tmpl )); body_append(impl_code_attr, parse_global_body( token_fmt( "typename", Str name(Attributes), codetype_impl_tmpl ))); body_append(impl_code_cmt, parse_global_body( token_fmt( "typename", Str name(Comment), codetype_impl_tmpl ))); @@ -563,34 +564,35 @@ CodeBody gen_ast_inlines() ); #pragma pop_macro("forceinline") - CodeBody impl_cast_body = parse_global_body( token_fmt( "typename", Str name(Body), cast_tmpl )); - CodeBody impl_cast_attribute = parse_global_body( token_fmt( "typename", Str name(Attributes), cast_tmpl )); - CodeBody impl_cast_cmt = parse_global_body( token_fmt( "typename", Str name(Comment), cast_tmpl )); - CodeBody impl_cast_constr = parse_global_body( token_fmt( "typename", Str name(Constructor), cast_tmpl )); - CodeBody impl_cast_class = parse_global_body( token_fmt( "typename", Str name(Class), cast_tmpl )); - CodeBody impl_cast_define = parse_global_body( token_fmt( "typename", Str name(Define), cast_tmpl )); - CodeBody impl_cast_destruct = parse_global_body( token_fmt( "typename", Str name(Destructor), cast_tmpl )); - CodeBody impl_cast_enum = parse_global_body( token_fmt( "typename", Str name(Enum), cast_tmpl )); - CodeBody impl_cast_exec = parse_global_body( token_fmt( "typename", Str name(Exec), cast_tmpl )); - CodeBody impl_cast_extern = parse_global_body( token_fmt( "typename", Str name(Extern), cast_tmpl )); - CodeBody impl_cast_friend = parse_global_body( token_fmt( "typename", Str name(Friend), cast_tmpl )); - CodeBody impl_cast_fn = parse_global_body( token_fmt( "typename", Str name(Fn), cast_tmpl )); - CodeBody impl_cast_include = parse_global_body( token_fmt( "typename", Str name(Include), cast_tmpl )); - CodeBody impl_cast_module = parse_global_body( token_fmt( "typename", Str name(Module), cast_tmpl )); - CodeBody impl_cast_ns = parse_global_body( token_fmt( "typename", Str name(NS), cast_tmpl )); - CodeBody impl_cast_op = parse_global_body( token_fmt( "typename", Str name(Operator), cast_tmpl )); - CodeBody impl_cast_opcast = parse_global_body( token_fmt( "typename", Str name(OpCast), cast_tmpl )); - CodeBody impl_cast_params = parse_global_body( token_fmt( "typename", Str name(Params), cast_tmpl )); - CodeBody impl_cast_pragma = parse_global_body( token_fmt( "typename", Str name(Pragma), cast_tmpl )); - CodeBody impl_cast_precond = parse_global_body( token_fmt( "typename", Str name(PreprocessCond), cast_tmpl )); - CodeBody impl_cast_specs = parse_global_body( token_fmt( "typename", Str name(Specifiers), cast_tmpl )); - CodeBody impl_cast_struct = parse_global_body( token_fmt( "typename", Str name(Struct), cast_tmpl )); - CodeBody impl_cast_tmpl = parse_global_body( token_fmt( "typename", Str name(Template), cast_tmpl )); - CodeBody impl_cast_type = parse_global_body( token_fmt( "typename", Str name(Typename), cast_tmpl )); - CodeBody impl_cast_typedef = parse_global_body( token_fmt( "typename", Str name(Typedef), cast_tmpl )); - CodeBody impl_cast_union = parse_global_body( token_fmt( "typename", Str name(Union), cast_tmpl )); - CodeBody impl_cast_using = parse_global_body( token_fmt( "typename", Str name(Using), cast_tmpl )); - CodeBody impl_cast_var = parse_global_body( token_fmt( "typename", Str name(Var), cast_tmpl )); + CodeBody impl_cast_body = parse_global_body( token_fmt( "typename", Str name(Body), cast_tmpl )); + CodeBody impl_cast_attribute = parse_global_body( token_fmt( "typename", Str name(Attributes), cast_tmpl )); + CodeBody impl_cast_cmt = parse_global_body( token_fmt( "typename", Str name(Comment), cast_tmpl )); + CodeBody impl_cast_constr = parse_global_body( token_fmt( "typename", Str name(Constructor), cast_tmpl )); + CodeBody impl_cast_class = parse_global_body( token_fmt( "typename", Str name(Class), cast_tmpl )); + CodeBody impl_cast_define = parse_global_body( token_fmt( "typename", Str name(Define), cast_tmpl )); + CodeBody impl_cast_define_params = parse_global_body( token_fmt( "typename", Str name(DefineParams), cast_tmpl )); + CodeBody impl_cast_destruct = parse_global_body( token_fmt( "typename", Str name(Destructor), cast_tmpl )); + CodeBody impl_cast_enum = parse_global_body( token_fmt( "typename", Str name(Enum), cast_tmpl )); + CodeBody impl_cast_exec = parse_global_body( token_fmt( "typename", Str name(Exec), cast_tmpl )); + CodeBody impl_cast_extern = parse_global_body( token_fmt( "typename", Str name(Extern), cast_tmpl )); + CodeBody impl_cast_friend = parse_global_body( token_fmt( "typename", Str name(Friend), cast_tmpl )); + CodeBody impl_cast_fn = parse_global_body( token_fmt( "typename", Str name(Fn), cast_tmpl )); + CodeBody impl_cast_include = parse_global_body( token_fmt( "typename", Str name(Include), cast_tmpl )); + CodeBody impl_cast_module = parse_global_body( token_fmt( "typename", Str name(Module), cast_tmpl )); + CodeBody impl_cast_ns = parse_global_body( token_fmt( "typename", Str name(NS), cast_tmpl )); + CodeBody impl_cast_op = parse_global_body( token_fmt( "typename", Str name(Operator), cast_tmpl )); + CodeBody impl_cast_opcast = parse_global_body( token_fmt( "typename", Str name(OpCast), cast_tmpl )); + CodeBody impl_cast_params = parse_global_body( token_fmt( "typename", Str name(Params), cast_tmpl )); + CodeBody impl_cast_pragma = parse_global_body( token_fmt( "typename", Str name(Pragma), cast_tmpl )); + CodeBody impl_cast_precond = parse_global_body( token_fmt( "typename", Str name(PreprocessCond), cast_tmpl )); + CodeBody impl_cast_specs = parse_global_body( token_fmt( "typename", Str name(Specifiers), cast_tmpl )); + CodeBody impl_cast_struct = parse_global_body( token_fmt( "typename", Str name(Struct), cast_tmpl )); + CodeBody impl_cast_tmpl = parse_global_body( token_fmt( "typename", Str name(Template), cast_tmpl )); + CodeBody impl_cast_type = parse_global_body( token_fmt( "typename", Str name(Typename), cast_tmpl )); + CodeBody impl_cast_typedef = parse_global_body( token_fmt( "typename", Str name(Typedef), cast_tmpl )); + CodeBody impl_cast_union = parse_global_body( token_fmt( "typename", Str name(Union), cast_tmpl )); + CodeBody impl_cast_using = parse_global_body( token_fmt( "typename", Str name(Using), cast_tmpl )); + CodeBody impl_cast_var = parse_global_body( token_fmt( "typename", Str name(Var), cast_tmpl )); CodeBody result = def_global_body( args( def_pragma( txt("region generated code inline implementation")), @@ -602,6 +604,7 @@ CodeBody gen_ast_inlines() impl_code_constr, impl_code_class, impl_code_define, + impl_code_define_params, impl_code_destruct, impl_code_enum, impl_code_exec, @@ -636,6 +639,7 @@ CodeBody gen_ast_inlines() impl_cast_constr, impl_cast_class, impl_cast_define, + impl_cast_define_params, impl_cast_destruct, impl_cast_enum, impl_cast_exec, From f93250da07d116158a626266f56337ddd63a4e47 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 14 Dec 2024 21:21:13 -0500 Subject: [PATCH 04/15] compiling again... --- base/components/ast_types.hpp | 2 +- base/components/code_serialization.cpp | 8 +- base/components/code_types.hpp | 6 +- base/components/gen/ast_inlines.hpp | 21 +++ base/components/gen/ecodetypes.hpp | 4 +- base/components/gen/eoperator.hpp | 2 +- base/components/gen/etoktype.hpp | 198 ++++++++++++------------- base/components/inlines.hpp | 8 +- base/components/interface.cpp | 6 +- base/components/interface.hpp | 13 +- base/components/interface.upfront.cpp | 6 +- base/components/lexer.cpp | 9 +- base/components/parser.cpp | 78 ++++------ base/components/parser_types.hpp | 28 ++-- base/helpers/base_codegen.hpp | 8 +- 15 files changed, 197 insertions(+), 200 deletions(-) diff --git a/base/components/ast_types.hpp b/base/components/ast_types.hpp index 8899174..43a0b80 100644 --- a/base/components/ast_types.hpp +++ b/base/components/ast_types.hpp @@ -151,7 +151,7 @@ struct AST_Define char _PAD_PROPERTIES_ [ sizeof(AST*) * 4 ]; CodeDefineParams Params; Code Body; // Should be completely serialized for now to a: StrCached Content. - char _PAD_PROPERTIES_ [ sizeof(AST*) * 1 ]; + char _PAD_PROPERTIES_2_ [ sizeof(AST*) * 1 ]; }; }; StrCached Name; diff --git a/base/components/code_serialization.cpp b/base/components/code_serialization.cpp index b219e80..c9678a1 100644 --- a/base/components/code_serialization.cpp +++ b/base/components/code_serialization.cpp @@ -252,7 +252,7 @@ void define_to_strbuilder_ref(CodeDefine define, StrBuilder* result ) GEN_ASSERT(define->Body); GEN_ASSERT(define->Body->Content); if (define->Params) { - StrBuilder params_builder = define_params_to_strbuilder(define->Params) + StrBuilder params_builder = define_params_to_strbuilder(define->Params); strbuilder_append_fmt( result, "#define %S(%S) %S", define->Name, strbuilder_to_str(params_builder), define->Body->Content ); } else { @@ -268,7 +268,7 @@ StrBuilder define_params_to_strbuilder(CodeDefineParams params) return result; } -void define_params_to_strbuilder_ref(CodeDefineParams params, StrBuilder* result) +void define_params_to_strbuilder_ref(CodeDefineParams self, StrBuilder* result) { GEN_ASSERT(self); GEN_ASSERT(result); @@ -278,9 +278,9 @@ void define_params_to_strbuilder_ref(CodeDefineParams params, StrBuilder* result } if ( self->NumEntries - 1 > 0 ) { - for ( CodeParams param = begin_CodeDefineParams(self->Next); param != end_CodeDefineParams(self->Next); param = next_CodeDefineParams(self->Next, param) ) + for ( CodeDefineParams param = begin_CodeDefineParams(self->Next); param != end_CodeDefineParams(self->Next); param = next_CodeDefineParams(self->Next, param) ) { - strbuilder_append_fmt( result, ", %SB", params_to_strbuilder(param) ); + strbuilder_append_fmt( result, ", %SB", define_params_to_strbuilder(param) ); } } } diff --git a/base/components/code_types.hpp b/base/components/code_types.hpp index 2ab1941..d1885ff 100644 --- a/base/components/code_types.hpp +++ b/base/components/code_types.hpp @@ -232,15 +232,15 @@ struct CodeDefineParams forceinline void to_strbuilder( StrBuilder& result ) { return define_params_to_strbuilder_ref(* this, & result); } #endif Using_CodeOps( CodeDefineParams ); - forceinline CodeDefineParams begin() { return begin_CodeParams( cast(CodeParams, * this)); } - forceinline CodeDefineParams end() { return end_CodeParams( cast(CodeParams, * this)); } + forceinline CodeDefineParams begin() { return (CodeDefineParams) (Code) begin_CodeParams( cast(CodeParams, * this)); } + forceinline CodeDefineParams end() { return (CodeDefineParams) (Code) end_CodeParams( cast(CodeParams, * this)); } forceinline operator Code() { return { (AST*)ast }; } forceinline CodeDefineParams operator *() { return * this; } // Required to support for-range iteration. forceinline AST_DefineParams* operator->() { GEN_ASSERT(ast); return ast; } - forceinline CodeDefineParams& operator++() { return cast(CodeParams, * this).operator ++() }; + forceinline CodeDefineParams& operator++() { return (CodeDefineParams) (Code) cast(CodeParams, * this).operator ++(); }; AST_DefineParams* ast; }; diff --git a/base/components/gen/ast_inlines.hpp b/base/components/gen/ast_inlines.hpp index 46e80ed..8f187b8 100644 --- a/base/components/gen/ast_inlines.hpp +++ b/base/components/gen/ast_inlines.hpp @@ -179,6 +179,22 @@ inline AST_Define* CodeDefine::operator->() return ast; } +inline CodeDefineParams& CodeDefineParams::operator=( Code other ) +{ + if ( other.ast != nullptr && other->Parent != nullptr ) + { + ast = rcast( decltype( ast ), code_duplicate( other ).ast ); + ast->Parent = { nullptr }; + } + ast = rcast( decltype( ast ), other.ast ); + return *this; +} + +inline CodeDefineParams::operator bool() +{ + return ast != nullptr; +} + inline CodeDestructor& CodeDestructor::operator=( Code other ) { if ( other.ast != nullptr && other->Parent != nullptr ) @@ -851,6 +867,11 @@ forceinline Code::operator CodeDefine() const return { (AST_Define*)ast }; } +forceinline Code::operator CodeDefineParams() const +{ + return { (AST_DefineParams*)ast }; +} + forceinline Code::operator CodeDestructor() const { return { (AST_Destructor*)ast }; diff --git a/base/components/gen/ecodetypes.hpp b/base/components/gen/ecodetypes.hpp index c7ad1d0..5973002 100644 --- a/base/components/gen/ecodetypes.hpp +++ b/base/components/gen/ecodetypes.hpp @@ -75,7 +75,7 @@ enum CodeType : u32 inline Str codetype_to_str( CodeType type ) { - local_persist Str lookup[61] = { + local_persist Str lookup[] = { { "Invalid", sizeof( "Invalid" ) - 1 }, { "Untyped", sizeof( "Untyped" ) - 1 }, { "NewLine", sizeof( "NewLine" ) - 1 }, @@ -144,7 +144,7 @@ inline Str codetype_to_str( CodeType type ) inline Str codetype_to_keyword_str( CodeType type ) { - local_persist Str lookup[61] = { + local_persist Str lookup[] = { { "__NA__", sizeof( "__NA__" ) - 1 }, { "__NA__", sizeof( "__NA__" ) - 1 }, { "__NA__", sizeof( "__NA__" ) - 1 }, diff --git a/base/components/gen/eoperator.hpp b/base/components/gen/eoperator.hpp index b447d7a..6a28f2b 100644 --- a/base/components/gen/eoperator.hpp +++ b/base/components/gen/eoperator.hpp @@ -60,7 +60,7 @@ enum Operator : u32 inline Str operator_to_str( Operator op ) { - local_persist Str lookup[47] = { + local_persist Str lookup[] = { { "INVALID", sizeof( "INVALID" ) - 1 }, { "=", sizeof( "=" ) - 1 }, { "+=", sizeof( "+=" ) - 1 }, diff --git a/base/components/gen/etoktype.hpp b/base/components/gen/etoktype.hpp index dc4a400..5b43e44 100644 --- a/base/components/gen/etoktype.hpp +++ b/base/components/gen/etoktype.hpp @@ -114,105 +114,105 @@ enum TokType : u32 inline Str toktype_to_str( TokType type ) { local_persist Str lookup[] = { - { "__invalid__", sizeof( "__invalid__" ) - 1 }, - { "private", sizeof( "private" ) - 1 }, - { "protected", sizeof( "protected" ) - 1 }, - { "public", sizeof( "public" ) - 1 }, - { ".", sizeof( "." ) - 1 }, - { "::", sizeof( "::" ) - 1 }, - { "&", sizeof( "&" ) - 1 }, - { "&&", sizeof( "&&" ) - 1 }, - { ":", sizeof( ":" ) - 1 }, - { "[[", sizeof( "[[" ) - 1 }, - { "]]", sizeof( "]]" ) - 1 }, - { "{", sizeof( "{" ) - 1 }, - { "}", sizeof( "}" ) - 1 }, - { "[", sizeof( "[" ) - 1 }, - { "]", sizeof( "]" ) - 1 }, - { "(", sizeof( "(" ) - 1 }, - { ")", sizeof( ")" ) - 1 }, - { "__comment__", sizeof( "__comment__" ) - 1 }, - { "__comment_end__", sizeof( "__comment_end__" ) - 1 }, - { "__comment_start__", sizeof( "__comment_start__" ) - 1 }, - { "__character__", sizeof( "__character__" ) - 1 }, - { ",", sizeof( "," ) - 1 }, - { "class", sizeof( "class" ) - 1 }, - { "__attribute__", sizeof( "__attribute__" ) - 1 }, - { "__declspec", sizeof( "__declspec" ) - 1 }, - { "enum", sizeof( "enum" ) - 1 }, - { "extern", sizeof( "extern" ) - 1 }, - { "friend", sizeof( "friend" ) - 1 }, - { "module", sizeof( "module" ) - 1 }, - { "namespace", sizeof( "namespace" ) - 1 }, - { "operator", sizeof( "operator" ) - 1 }, - { "struct", sizeof( "struct" ) - 1 }, - { "template", sizeof( "template" ) - 1 }, - { "typedef", sizeof( "typedef" ) - 1 }, - { "using", sizeof( "using" ) - 1 }, - { "union", sizeof( "union" ) - 1 }, - { "__identifier__", sizeof( "__identifier__" ) - 1 }, - { "import", sizeof( "import" ) - 1 }, - { "export", sizeof( "export" ) - 1 }, - { "__new_line__", sizeof( "__new_line__" ) - 1 }, - { "__number__", sizeof( "__number__" ) - 1 }, - { "__operator__", sizeof( "__operator__" ) - 1 }, - { "#", sizeof( "#" ) - 1 }, - { "define", sizeof( "define" ) - 1 }, - { "__define_param__", sizeof( "__define__param__" ) - 1 }, - { "if", sizeof( "if" ) - 1 }, - { "ifdef", sizeof( "ifdef" ) - 1 }, - { "ifndef", sizeof( "ifndef" ) - 1 }, - { "elif", sizeof( "elif" ) - 1 }, - { "else", sizeof( "else" ) - 1 }, - { "endif", sizeof( "endif" ) - 1 }, - { "include", sizeof( "include" ) - 1 }, - { "pragma", sizeof( "pragma" ) - 1 }, - { "__macro_content__", sizeof( "__macro_content__" ) - 1 }, - { "__macro_expression__", sizeof( "__macro_expression__" ) - 1 }, - { "__macro_statment__", sizeof( "__macro_statment__" ) - 1 }, - { "__macro_typename__", sizeof( "__macro_typename__" ) - 1 }, - { "__unsupported__", sizeof( "__unsupported__" ) - 1 }, - { "alignas", sizeof( "alignas" ) - 1 }, - { "const", sizeof( "const" ) - 1 }, - { "consteval", sizeof( "consteval" ) - 1 }, - { "constexpr", sizeof( "constexpr" ) - 1 }, - { "constinit", sizeof( "constinit" ) - 1 }, - { "explicit", sizeof( "explicit" ) - 1 }, - { "extern", sizeof( "extern" ) - 1 }, - { "final", sizeof( "final" ) - 1 }, - { "forceinline", sizeof( "forceinline" ) - 1 }, - { "global", sizeof( "global" ) - 1 }, - { "inline", sizeof( "inline" ) - 1 }, - { "internal", sizeof( "internal" ) - 1 }, - { "local_persist", sizeof( "local_persist" ) - 1 }, - { "mutable", sizeof( "mutable" ) - 1 }, - { "neverinline", sizeof( "neverinline" ) - 1 }, - { "override", sizeof( "override" ) - 1 }, - { "static", sizeof( "static" ) - 1 }, - { "thread_local", sizeof( "thread_local" ) - 1 }, - { "volatile", sizeof( "volatile" ) - 1 }, - { "virtual", sizeof( "virtual" ) - 1 }, - { "*", sizeof( "*" ) - 1 }, - { ";", sizeof( ";" ) - 1 }, - { "static_assert", sizeof( "static_assert" ) - 1 }, - { "__string__", sizeof( "__string__" ) - 1 }, - { "typename", sizeof( "typename" ) - 1 }, - { "unsigned", sizeof( "unsigned" ) - 1 }, - { "signed", sizeof( "signed" ) - 1 }, - { "short", sizeof( "short" ) - 1 }, - { "long", sizeof( "long" ) - 1 }, - { "bool", sizeof( "bool" ) - 1 }, - { "char", sizeof( "char" ) - 1 }, - { "int", sizeof( "int" ) - 1 }, - { "double", sizeof( "double" ) - 1 }, - { "__int8", sizeof( "__int8" ) - 1 }, - { "__int16", sizeof( "__int16" ) - 1 }, - { "__int32", sizeof( "__int32" ) - 1 }, - { "__int64", sizeof( "__int64" ) - 1 }, - { "_W64", sizeof( "_W64" ) - 1 }, - { "...", sizeof( "..." ) - 1 }, - { "__attrib_start__", sizeof( "__attrib_start__" ) - 1 }, - { "GEN_API", sizeof( "GEN_API" ) - 1 }, + { "__invalid__", sizeof( "__invalid__" ) - 1 }, + { "private", sizeof( "private" ) - 1 }, + { "protected", sizeof( "protected" ) - 1 }, + { "public", sizeof( "public" ) - 1 }, + { ".", sizeof( "." ) - 1 }, + { "::", sizeof( "::" ) - 1 }, + { "&", sizeof( "&" ) - 1 }, + { "&&", sizeof( "&&" ) - 1 }, + { ":", sizeof( ":" ) - 1 }, + { "[[", sizeof( "[[" ) - 1 }, + { "]]", sizeof( "]]" ) - 1 }, + { "{", sizeof( "{" ) - 1 }, + { "}", sizeof( "}" ) - 1 }, + { "[", sizeof( "[" ) - 1 }, + { "]", sizeof( "]" ) - 1 }, + { "(", sizeof( "(" ) - 1 }, + { ")", sizeof( ")" ) - 1 }, + { "__comment__", sizeof( "__comment__" ) - 1 }, + { "__comment_end__", sizeof( "__comment_end__" ) - 1 }, + { "__comment_start__", sizeof( "__comment_start__" ) - 1 }, + { "__character__", sizeof( "__character__" ) - 1 }, + { ",", sizeof( "," ) - 1 }, + { "class", sizeof( "class" ) - 1 }, + { "__attribute__", sizeof( "__attribute__" ) - 1 }, + { "__declspec", sizeof( "__declspec" ) - 1 }, + { "enum", sizeof( "enum" ) - 1 }, + { "extern", sizeof( "extern" ) - 1 }, + { "friend", sizeof( "friend" ) - 1 }, + { "module", sizeof( "module" ) - 1 }, + { "namespace", sizeof( "namespace" ) - 1 }, + { "operator", sizeof( "operator" ) - 1 }, + { "struct", sizeof( "struct" ) - 1 }, + { "template", sizeof( "template" ) - 1 }, + { "typedef", sizeof( "typedef" ) - 1 }, + { "using", sizeof( "using" ) - 1 }, + { "union", sizeof( "union" ) - 1 }, + { "__identifier__", sizeof( "__identifier__" ) - 1 }, + { "import", sizeof( "import" ) - 1 }, + { "export", sizeof( "export" ) - 1 }, + { "__new_line__", sizeof( "__new_line__" ) - 1 }, + { "__number__", sizeof( "__number__" ) - 1 }, + { "__operator__", sizeof( "__operator__" ) - 1 }, + { "#", sizeof( "#" ) - 1 }, + { "define", sizeof( "define" ) - 1 }, + { "__define_param__", sizeof( "__define_param__" ) - 1 }, + { "if", sizeof( "if" ) - 1 }, + { "ifdef", sizeof( "ifdef" ) - 1 }, + { "ifndef", sizeof( "ifndef" ) - 1 }, + { "elif", sizeof( "elif" ) - 1 }, + { "else", sizeof( "else" ) - 1 }, + { "endif", sizeof( "endif" ) - 1 }, + { "include", sizeof( "include" ) - 1 }, + { "pragma", sizeof( "pragma" ) - 1 }, + { "__macro_content__", sizeof( "__macro_content__" ) - 1 }, + { "__macro_expression__", sizeof( "__macro_expression__" ) - 1 }, + { "__macro_statment__", sizeof( "__macro_statment__" ) - 1 }, + { "__macro_typename__", sizeof( "__macro_typename__" ) - 1 }, + { "__unsupported__", sizeof( "__unsupported__" ) - 1 }, + { "alignas", sizeof( "alignas" ) - 1 }, + { "const", sizeof( "const" ) - 1 }, + { "consteval", sizeof( "consteval" ) - 1 }, + { "constexpr", sizeof( "constexpr" ) - 1 }, + { "constinit", sizeof( "constinit" ) - 1 }, + { "explicit", sizeof( "explicit" ) - 1 }, + { "extern", sizeof( "extern" ) - 1 }, + { "final", sizeof( "final" ) - 1 }, + { "forceinline", sizeof( "forceinline" ) - 1 }, + { "global", sizeof( "global" ) - 1 }, + { "inline", sizeof( "inline" ) - 1 }, + { "internal", sizeof( "internal" ) - 1 }, + { "local_persist", sizeof( "local_persist" ) - 1 }, + { "mutable", sizeof( "mutable" ) - 1 }, + { "neverinline", sizeof( "neverinline" ) - 1 }, + { "override", sizeof( "override" ) - 1 }, + { "static", sizeof( "static" ) - 1 }, + { "thread_local", sizeof( "thread_local" ) - 1 }, + { "volatile", sizeof( "volatile" ) - 1 }, + { "virtual", sizeof( "virtual" ) - 1 }, + { "*", sizeof( "*" ) - 1 }, + { ";", sizeof( ";" ) - 1 }, + { "static_assert", sizeof( "static_assert" ) - 1 }, + { "__string__", sizeof( "__string__" ) - 1 }, + { "typename", sizeof( "typename" ) - 1 }, + { "unsigned", sizeof( "unsigned" ) - 1 }, + { "signed", sizeof( "signed" ) - 1 }, + { "short", sizeof( "short" ) - 1 }, + { "long", sizeof( "long" ) - 1 }, + { "bool", sizeof( "bool" ) - 1 }, + { "char", sizeof( "char" ) - 1 }, + { "int", sizeof( "int" ) - 1 }, + { "double", sizeof( "double" ) - 1 }, + { "__int8", sizeof( "__int8" ) - 1 }, + { "__int16", sizeof( "__int16" ) - 1 }, + { "__int32", sizeof( "__int32" ) - 1 }, + { "__int64", sizeof( "__int64" ) - 1 }, + { "_W64", sizeof( "_W64" ) - 1 }, + { "...", sizeof( "..." ) - 1 }, + { "__attrib_start__", sizeof( "__attrib_start__" ) - 1 }, + { "GEN_API", sizeof( "GEN_API" ) - 1 }, }; return lookup[type]; } diff --git a/base/components/inlines.hpp b/base/components/inlines.hpp index b84ab3b..e6b4b09 100644 --- a/base/components/inlines.hpp +++ b/base/components/inlines.hpp @@ -259,12 +259,12 @@ CodeParams next_CodeParams(CodeParams params, CodeParams param_iter) #pragma region CodeDefineParams forceinline void define_params_append (CodeDefineParams appendee, CodeDefineParams other ) { params_append( cast(CodeParams, appendee), cast(CodeParams, other) ); } -forceinline CodeDefineParams define_params_get (CodeDefineParams self, s32 idx ) { return params_get( cast(CodeParams, self), idx); } +forceinline CodeDefineParams define_params_get (CodeDefineParams self, s32 idx ) { return (CodeDefineParams) (Code) params_get( cast(CodeParams, self), idx); } forceinline bool define_params_has_entries(CodeDefineParams self) { return params_has_entries( cast(CodeParams, self)); } -CodeDefineParams begin_CodeDefineParams(CodeDefineParams params) { return begin_CodeParams( cast(CodeParams, params)); } -CodeDefineParams end_CodeDefineParams (CodeDefineParams params) { return end_CodeParams ( cast(CodeParams, params)); } -CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter) { return next_CodeParams ( cast(CodeParams, params), cast(CoeParams, entry_iter)); } +CodeDefineParams begin_CodeDefineParams(CodeDefineParams params) { return (CodeDefineParams) (Code) begin_CodeParams( cast(CodeParams, (Code)params)); } +CodeDefineParams end_CodeDefineParams (CodeDefineParams params) { return (CodeDefineParams) (Code) end_CodeParams ( cast(CodeParams, (Code)params)); } +CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter) { return (CodeDefineParams) (Code) next_CodeParams ( cast(CodeParams, (Code)params), cast(CodeParams, (Code)entry_iter)); } #pragma endregion CodeDefineParams #pragma region CodeSpecifiers diff --git a/base/components/interface.cpp b/base/components/interface.cpp index 6a2d648..69d6482 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -272,8 +272,8 @@ void init(Context* ctx) ctx->CodePool_NumBlocks = kilobytes(16); } - if (ctx->InitSize_LexArena == 0 ) { - ctx->InitSize_LexArena = megabytes(4); + if (ctx->InitSize_LexerTokens == 0 ) { + ctx->InitSize_LexerTokens = kilobytes(64); } if (ctx->SizePer_StringArena == 0) { ctx->SizePer_StringArena = megabytes(1); @@ -315,7 +315,7 @@ void init(Context* ctx) if ( ctx->StrCache.Entries == nullptr ) GEN_FATAL( "gen::init: Failed to initialize the StringCache"); - ctx->PreprocessorMacros = hashtable_init(PreprocessorMacros, ctx->Allocator_DyanmicContainers); + ctx->PreprocessorMacros = hashtable_init(PreprocessorMacro, ctx->Allocator_DyanmicContainers); if (ctx->PreprocessorMacros.Hashes == nullptr || ctx->PreprocessorMacros.Entries == nullptr) { GEN_FATAL( "gen::init: Failed to initialize the PreprocessMacros table" ); } diff --git a/base/components/interface.hpp b/base/components/interface.hpp index e56a87a..f3e13fd 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -60,7 +60,7 @@ struct Context u32 CodePool_NumBlocks; // TODO(Ed): Review these... (No longer needed if using the proper allocation strategy) - u32 InitSize_LexArena; + u32 InitSize_LexerTokens; u32 SizePer_StringArena; // TODO(Ed): Symbol Table @@ -87,9 +87,6 @@ struct Context StringTable StrCache; // TODO(Ed): This needs to be just handled by a parser context - - // Arena LexArena; - // StringTable Lexer_defines; Array(Token) Lexer_Tokens; // TODO(Ed): Active parse context vs a parse result need to be separated conceptually @@ -109,8 +106,8 @@ GEN_API void reset(Context* ctx); GEN_API void set_context(Context* ctx); -// Mostly used to verify a macro entry exists for the given name -GEN_API PreprocessMacro* lookup_preprocess_macro( Str Name ); +// Mostly intended for the parser +GEN_API PreprocessorMacro* lookup_preprocess_macro( Str Name ); // Alternative way to add a preprocess define entry for the lexer & parser to utilize // if the user doesn't want to use def_define @@ -154,9 +151,9 @@ struct Opts_def_constructor { GEN_API CodeConstructor def_constructor( Opts_def_constructor opts GEN_PARAM_DEFAULT ); struct Opts_def_define { - MacroFlags flags; CodeDefineParams params; Str content; + MacroFlags flags; b32 dont_register_to_preprocess_macros; }; GEN_API CodeDefine def_define( Str name, MacroType type, Opts_def_define opts GEN_PARAM_DEFAULT ); @@ -273,7 +270,7 @@ GEN_API CodeBody def_body( CodeType type ); GEN_API CodeBody def_class_body ( s32 num, ... ); GEN_API CodeBody def_class_body ( s32 num, Code* codes ); GEN_API CodeDefineParams def_define_params ( s32 num, ... ); -GEN_API CodeDefineParams def_define_params ( s32 num, CodeDefineParams* codes ) +GEN_API CodeDefineParams def_define_params ( s32 num, CodeDefineParams* codes ); GEN_API CodeBody def_enum_body ( s32 num, ... ); GEN_API CodeBody def_enum_body ( s32 num, Code* codes ); GEN_API CodeBody def_export_body ( s32 num, ... ); diff --git a/base/components/interface.upfront.cpp b/base/components/interface.upfront.cpp index 1e4b6b1..271bb68 100644 --- a/base/components/interface.upfront.cpp +++ b/base/components/interface.upfront.cpp @@ -572,13 +572,13 @@ CodeDefine def_define( Str name, MacroType type, Opts_def_define p ) result->Name = cache_str( name ); result->Params = p.params; if ( p.content.Len <= 0 || p.content.Ptr == nullptr ) - result->Content = cache_str( txt("\n") ); + result->Body = untyped_str( txt("\n") ); else - result->Content = cache_str( strbuilder_to_str(strbuilder_fmt_buf(_ctx->Allocator_Temp, "%S\n", content)) ); + result->Body = untyped_str( strbuilder_to_str(strbuilder_fmt_buf(_ctx->Allocator_Temp, "%S\n", p.content)) ); b32 register_define = ! p.dont_register_to_preprocess_macros; if ( register_define ) { - macro_entry = { result->Name, type, p.flags }; + PreprocessorMacro macro_entry = { result->Name, type, p.flags }; register_preprocess_macro(macro_entry); } return result; diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 6dc5ff9..97f3c42 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -118,7 +118,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) Token name = { { ctx->scanner, 1 }, Tok_Identifier, ctx->line, ctx->column, TF_Preprocess }; move_forward(); - PreprocessorMacro macro = { name.Text, MT_Statement, 0 }; + PreprocessorMacro macro = { name.Text, MT_Statement, (MacroFlags)0 }; PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); if ( registered_macro == nullptr ) { log_fmt("Warning: %S is was not registered before the lexer processed its #define directive, it will be registered as a statement macro" @@ -146,6 +146,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) array_append( _ctx->Lexer_Tokens, opening_paren ); move_forward(); + Token last_parameter; // We need to tokenize the define's arguments now: while( ctx->left && * ctx->scanner != ')') { @@ -162,6 +163,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) } array_append(_ctx->Lexer_Tokens, parameter); skip_whitespace(); + last_parameter = parameter; } else { log_failure("lex_preprocessor_define(%d, %d): Expected a '_' or alpha character for a parameter name for %S\n" @@ -176,7 +178,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) log_failure("lex_preprocessor_define(%d, %d): Expected a comma after parameter %S for %S\n" , ctx->line , ctx->column - , parameter.Text + , last_parameter.Text , name.Text ); return Lex_ReturnNull; @@ -190,7 +192,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) log_failure("lex_preprocessor_define(%d, %d): Expected a ')' after last_parameter %S for %S (ran out of characters...)\n" , ctx->line , ctx->column - , parameter.Text + , last_parameter.Text , name.Text ); return Lex_ReturnNull; @@ -527,7 +529,6 @@ TokArray lex( Str content ) c.content = content; c.left = content.Len; c.scanner = content.Ptr; - c.defines = _ctx->Lexer_defines; char const* word = c.scanner; s32 word_length = 0; diff --git a/base/components/parser.cpp b/base/components/parser.cpp index e911fc1..46c71c2 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -118,11 +118,7 @@ bool lex__eat(TokArray* self, TokType type ) internal void parser_init() { - _ctx->Lexer_Tokens = array_init_reserve(Token, arena_allocator_info( & _ctx->LexArena) - , ( _ctx->InitSize_LexArena - sizeof( ArrayHeader ) ) / sizeof(Token) - ); - - _ctx->Lexer_defines = hashtable_init_reserve(Str, _ctx->Allocator_DyanmicContainers, 256 ); + _ctx->Lexer_Tokens = array_init_reserve(Token, _ctx->Allocator_DyanmicContainers, _ctx->InitSize_LexerTokens ); } internal @@ -942,7 +938,6 @@ CodeBody parse_class_struct_body( TokType which, Token name ) member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Macro_Stmt )); break; } - case Tok_Preprocess_Macro_Expr: { log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser)); return InvalidCode; @@ -961,7 +956,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) } case Tok_Preprocess_Unsupported: { - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported, parser_consume_braces )); + member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported )); // # break; } @@ -1317,10 +1312,10 @@ CodeDefine parse_define() CodeDefineParams params; if ( left && currtok.Type != Tok_Capture_End ) { params = (CodeDefineParams) make_code(); - params.Type = CT_Parameters_Define; - params.Name = currtok.text; + params->Type = CT_Parameters_Define; + params->Name = currtok.Text; - define->params = params; + define->Params = params; eat( Tok_Preprocess_Define_Param ); // #define ( ) @@ -1328,8 +1323,8 @@ CodeDefine parse_define() while( left && currtok.Type != Tok_Capture_End ) { CodeDefineParams next_param = (CodeDefineParams) make_code(); - next_param.Type = CT_Parameters_Define; - next_param.Name = currtok.Text; + next_param->Type = CT_Parameters_Define; + next_param->Name = currtok.Text; define_params_append(params, next_param); // #define ( , ... @@ -1349,7 +1344,7 @@ CodeDefine parse_define() if ( currtok.Text.Len == 0 ) { - define->Content = cache_str( tok_to_str(currtok) ); + define->Body = untyped_str( tok_to_str(currtok) ); eat( Tok_Preprocess_Content ); // #define ( ) @@ -1357,7 +1352,7 @@ CodeDefine parse_define() return define; } - define->Content = cache_str( strbuilder_to_str( parser_strip_formatting( tok_to_str(currtok), parser_strip_formatting_dont_preserve_newlines )) ); + define->Body = untyped_str( strbuilder_to_str( parser_strip_formatting( tok_to_str(currtok), parser_strip_formatting_dont_preserve_newlines )) ); eat( Tok_Preprocess_Content ); // #define ( ) @@ -1729,11 +1724,13 @@ CodeBody parse_global_nspace( CodeType which ) // #endif break; - case Tok_Preprocess_Macro: { - // - macro_found = true; - goto Preprocess_Macro_Bare_In_Body; - // TODO(Ed): MACRO UPDATE + case Tok_Preprocess_Macro_Stmt: { + member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Macro_Stmt )); + break; + } + case Tok_Preprocess_Macro_Expr: { + log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser)); + return InvalidCode; } case Tok_Preprocess_Pragma: { @@ -1743,7 +1740,7 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Preprocess_Unsupported: { - member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported, parser_consume_braces )); + member = cast(Code, parse_simple_preprocess( Tok_Preprocess_Unsupported )); // # ... } break; @@ -1828,6 +1825,7 @@ CodeBody parse_global_nspace( CodeType which ) } //! Fallthrough intentional case Tok_Identifier: + case Tok_Preprocess_Macro_Typename: case Tok_Spec_Const: case Tok_Type_Long: case Tok_Type_Short: @@ -1879,28 +1877,6 @@ CodeBody parse_global_nspace( CodeType which ) } } - if (macro_found) - { - Preprocess_Macro_Bare_In_Body: - b32 lone_macro = nexttok.Type == Tok_Statement_End || nexttok_noskip.Type == Tok_NewLine; - if (lone_macro) - { - member = parse_simple_preprocess( Tok_Preprocess_Macro, parser_consume_braces ); - // ; - - if ( member == Code_Invalid ) - { - log_failure( "Failed to parse member\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); - return InvalidCode; - } - goto Member_Resolved_To_Lone_Macro; - } - - // We have a macro but its most likely behaving as a typename - // ... } @@ -2943,7 +2919,7 @@ CodePreprocessCond parse_preprocess_cond() } internal -Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) +Code parse_simple_preprocess( TokType which ) { // TODO(Ed): We can handle a macro a bit better than this. It's AST can be made more robust.. // Make an AST_Macro, it should have an Name be the macro itself, with the function body being an optional function body node. @@ -2954,7 +2930,7 @@ Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) eat( which ); // - PreprocessorMacro macro = * lookup_preprocess_macro( name.Text ); + PreprocessorMacro macro = * lookup_preprocess_macro( full_macro.Text ); if ( macro_expects_body(macro) && peektok.Type == Tok_BraceCurly_Open ) { @@ -3043,7 +3019,7 @@ Code parse_simple_preprocess( TokType which, bool dont_consume_braces ) Leave_Scope_Early: Code result = untyped_str( full_macro.Text ); - _ctx->parser.Scope->Name = tok.Text; + _ctx->parser.Scope->Name = full_macro.Text; parser_pop(& _ctx->parser); return result; @@ -3145,7 +3121,7 @@ CodeVar parse_variable_after_name( Code array_expr = parse_array_decl(); Code expr = NullCode; - Code bitfield_expr = NulLCode; + Code bitfield_expr = NullCode; b32 using_constructor_initializer = false; @@ -3635,13 +3611,13 @@ CodeEnum parser_parse_enum( bool inplace_def ) } // enum : } - else if ( currtok.Type == Tok_Preprocess_Macro ) + else if ( currtok.Type == Tok_Preprocess_Macro_Expr ) { // We'll support the enum_underlying macro - if ( str_contains( tok_to_str(currtok), enum_underlying_sig) ) + if ( str_contains( tok_to_str(currtok), enum_underlying_macro.Name) ) { use_macro_underlying = true; - underlying_macro = parse_simple_preprocess( Tok_Preprocess_Macro, parser_dont_consume_braces ); + underlying_macro = parse_simple_preprocess( Tok_Preprocess_Macro_Typename ); } } @@ -5226,7 +5202,7 @@ CodeUnion parser_parse_union( bool inplace_def ) break; case Tok_Preprocess_Macro_Stmt: - member = parse_simple_preprocess( Tok_Preprocess_Macro ); + member = parse_simple_preprocess( Tok_Preprocess_Macro_Stmt ); break; case Tok_Preprocess_Pragma: @@ -5234,7 +5210,7 @@ CodeUnion parser_parse_union( bool inplace_def ) break; case Tok_Preprocess_Unsupported: - member = parse_simple_preprocess( Tok_Preprocess_Unsupported, parser_consume_braces ); + member = parse_simple_preprocess( Tok_Preprocess_Unsupported ); break; default: diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index c7eeae3..f198aa9 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -4,6 +4,7 @@ #include "gen/ecode.hpp" #include "gen/eoperator.hpp" #include "gen/especifier.hpp" +#include "gen/etoktype.hpp" #endif enum TokFlags : u32 @@ -135,11 +136,11 @@ enum MacroType : u16 MT_Block_End, // Not Supported yet MT_Case_Statement, // Not Supported yet - MF_UnderlyingType = GEN_U16_Max, + MT_UnderlyingType = GEN_U16_MAX, }; forceinline -TokenType macrotype_to_toktype( MacroType type ) { +TokType macrotype_to_toktype( MacroType type ) { switch ( type ) { case MT_Statement : return Tok_Preprocess_Macro_Stmt; case MT_Expression : return Tok_Preprocess_Macro_Expr; @@ -152,7 +153,7 @@ TokenType macrotype_to_toktype( MacroType type ) { Str macrotype_to_str( MacroType type ) { local_persist - Str lookup[ (u32)Num_ModuleFlags ] = { + Str lookup[] = { { "Statement", sizeof("Statement") - 1 }, { "Expression", sizeof("Expression") - 1 }, { "Typename", sizeof("Typename") - 1 }, @@ -161,21 +162,22 @@ Str macrotype_to_str( MacroType type ) { "Case_Statement", sizeof("Case_Statement") - 1 }, }; local_persist - Str invalid_flag = { "Invalid", sizeof("Invalid") }; - if ( flag > ModuleFlag_Import ) - return invalid_flag; + Str invalid = { "Invalid", sizeof("Invalid") }; + if ( type > MT_Case_Statement ) + return invalid; - return lookup[ (u32)flag ]; + return lookup[ type ]; } -enum MacroFlags : u16 +enum EMacroFlags : u16 { MF_Functional = bit(0), // Macro has parameters (args expected to be passed) MF_Expects_Body = bit(1), // Expects to assign a braced scope to its body. MF_Null = 0, - MF_UnderlyingType = GEN_U16_Max, + MF_UnderlyingType = GEN_U16_MAX, }; +typedef u16 MacroFlags; struct PreprocessorMacro { @@ -184,12 +186,12 @@ struct PreprocessorMacro MacroFlags Flags; }; -forceinine +forceinline b32 macro_is_functional( PreprocessorMacro macro ) { - return bitfield_is_set( macro->Flags, MF_Functional ); + return bitfield_is_set( b16, macro.Flags, MF_Functional ); } forceinline b32 macro_expects_body( PreprocessorMacro macro ) { - return bitfield_is_set( macro->Flags, MF_Expects_Body ); -} \ No newline at end of file + return bitfield_is_set( b16, macro.Flags, MF_Expects_Body ); +} diff --git a/base/helpers/base_codegen.hpp b/base/helpers/base_codegen.hpp index 90fb32a..1a405f3 100644 --- a/base/helpers/base_codegen.hpp +++ b/base/helpers/base_codegen.hpp @@ -47,7 +47,7 @@ CodeBody gen_ecode( char const* path, bool use_c_definition = false ) Str codetype_to_str( CodeType type ) { local_persist - Str lookup[] = { + Str lookup[] = { }; return lookup[ type ]; @@ -57,7 +57,7 @@ CodeBody gen_ecode( char const* path, bool use_c_definition = false ) Str codetype_to_keyword_str( CodeType type ) { local_persist - Str lookup[ ] = { + Str lookup[] = { }; return lookup[ type ]; @@ -139,7 +139,7 @@ CodeBody gen_eoperator( char const* path, bool use_c_definition = false ) Str operator_to_str( Operator op ) { local_persist - Str lookup[] = { + Str lookup[] = { }; @@ -353,7 +353,7 @@ CodeBody gen_etoktype( char const* etok_path, char const* attr_path, bool use_c_ #pragma push_macro("GEN_DEFINE_ATTRIBUTE_TOKENS") #undef GEN_DEFINE_ATTRIBUTE_TOKENS - CodeDefine attribute_entires_def = def_define( name(GEN_DEFINE_ATTRIBUTE_TOKENS), strbuilder_to_str(attribute_define_entries) ); + CodeDefine attribute_entires_def = def_define( name(GEN_DEFINE_ATTRIBUTE_TOKENS), MT_Statement, { {}, strbuilder_to_str(attribute_define_entries) } ); #pragma pop_macro("GEN_DEFINE_ATTRIBUTE_TOKENS") // We cannot parse this enum, it has Attribute names as enums From 956ab731309e540740f5c7c6ed950d22678945c1 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 14 Dec 2024 23:10:23 -0500 Subject: [PATCH 05/15] Unreal variant generates, doing c_library corrections next --- base/components/code_types.hpp | 14 +++--- base/components/inlines.hpp | 9 ++++ base/components/interface.cpp | 38 ++++++++++++++- base/components/interface.hpp | 4 ++ base/components/lexer.cpp | 72 ++++++++++++++++++++++++---- base/components/parser.cpp | 16 ++++--- base/components/parser_types.hpp | 16 ++++--- base/dependencies/macros.hpp | 4 +- gen_unreal_engine/enums/ETokType.csv | 4 +- gen_unreal_engine/unreal.cpp | 39 +++++++++++++++ 10 files changed, 183 insertions(+), 33 deletions(-) diff --git a/base/components/code_types.hpp b/base/components/code_types.hpp index d1885ff..26d5316 100644 --- a/base/components/code_types.hpp +++ b/base/components/code_types.hpp @@ -202,7 +202,7 @@ struct CodeParams { #if ! GEN_C_LIKE_CPP Using_Code( CodeParams ); - forceinline void append( CodeParams other ) { return params_append(* this, other) } + forceinline void append( CodeParams other ) { return params_append(* this, other); } forceinline CodeParams get( s32 idx ) { return params_get( * this, idx); } forceinline bool has_entries() { return params_has_entries(* this); } forceinline StrBuilder to_strbuilder() { return params_to_strbuilder(* this); } @@ -225,8 +225,8 @@ struct CodeDefineParams { #if ! GEN_C_LIKE_CPP Using_Code( CodeDefineParams ); - forceinline void append( CodeDefineParams other ) { return params_append( cast(CodeParams, * this), other) } - forceinline CodeDefineParams get( s32 idx ) { return params_get( cast(CodeParams, * this), idx); } + forceinline void append( CodeDefineParams other ) { return params_append( cast(CodeParams, * this), cast(CodeParams, other)); } + forceinline CodeDefineParams get( s32 idx ) { return (CodeDefineParams) (Code) params_get( cast(CodeParams, * this), idx); } forceinline bool has_entries() { return params_has_entries( cast(CodeParams, * this)); } forceinline StrBuilder to_strbuilder() { return define_params_to_strbuilder(* this); } forceinline void to_strbuilder( StrBuilder& result ) { return define_params_to_strbuilder_ref(* this, & result); } @@ -240,7 +240,7 @@ struct CodeDefineParams GEN_ASSERT(ast); return ast; } - forceinline CodeDefineParams& operator++() { return (CodeDefineParams) (Code) cast(CodeParams, * this).operator ++(); }; + forceinline CodeDefineParams& operator++(); AST_DefineParams* ast; }; @@ -1058,9 +1058,9 @@ forceinline StrBuilder to_strbuilder ( CodeClass self ) forceinline void to_strbuilder_def( CodeClass self, StrBuilder& result ) { return class_to_strbuilder_def(self, & result); } forceinline void to_strbuilder_fwd( CodeClass self, StrBuilder& result ) { return class_to_strbuilder_fwd(self, & result); } -forceinline void append (CodeDefineParams appendee, CodeDefineParams other ) { return params_append(cast(CodeParam, appendee), other); } -forceinline CodeDefineParams get (CodeDefineParams params, s32 idx) { return params_get(cast(CodeParam, params), idx); } -forceinline bool has_entries (CodeDefineParams params ) { return params_has_entries(cast(CodeParam, params)); } +forceinline void append (CodeDefineParams appendee, CodeDefineParams other ) { params_append(cast(CodeParams, appendee), cast(CodeParams, other)); } +forceinline CodeDefineParams get (CodeDefineParams params, s32 idx) { return (CodeDefineParams) (Code) params_get(cast(CodeParams, params), idx); } +forceinline bool has_entries (CodeDefineParams params ) { return params_has_entries(cast(CodeParams, params)); } forceinline StrBuilder to_strbuilder(CodeDefineParams params ) { return define_params_to_strbuilder(params); } forceinline void to_strbuilder(CodeDefineParams params, StrBuilder& result ) { return define_params_to_strbuilder_ref(params, & result); } diff --git a/base/components/inlines.hpp b/base/components/inlines.hpp index e6b4b09..b557a4f 100644 --- a/base/components/inlines.hpp +++ b/base/components/inlines.hpp @@ -265,6 +265,15 @@ forceinline bool define_params_has_entries(CodeDefineParams self) CodeDefineParams begin_CodeDefineParams(CodeDefineParams params) { return (CodeDefineParams) (Code) begin_CodeParams( cast(CodeParams, (Code)params)); } CodeDefineParams end_CodeDefineParams (CodeDefineParams params) { return (CodeDefineParams) (Code) end_CodeParams ( cast(CodeParams, (Code)params)); } CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter) { return (CodeDefineParams) (Code) next_CodeParams ( cast(CodeParams, (Code)params), cast(CodeParams, (Code)entry_iter)); } + +#if GEN_COMPILER_CPP +forceinline +CodeDefineParams& CodeDefineParams::operator ++() +{ + * this = ast->Next; + return * this; +} +#endif #pragma endregion CodeDefineParams #pragma region CodeSpecifiers diff --git a/base/components/interface.cpp b/base/components/interface.cpp index 69d6482..a5a21ec 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -207,7 +207,7 @@ void define_constants() code_set_global(cast(Code, spec_local_persist)); if (enum_underlying_macro.Name.Len == 0) { - enum_underlying_macro.Name = txt("enum_underlying("); + enum_underlying_macro.Name = txt("enum_underlying"); enum_underlying_macro.Type = MT_Expression; enum_underlying_macro.Flags = MF_Functional; } @@ -470,6 +470,42 @@ PreprocessorMacro* lookup_preprocess_macro( Str name ) { } void register_preprocess_macro( PreprocessorMacro macro ) { + GEN_ASSERT_NOT_NULL(macro.Name.Ptr); + GEN_ASSERT(macro.Name.Len > 0); u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); hashtable_set( _ctx->PreprocessorMacros, key, macro ); } + +void register_preprocess_macros( s32 num, ... ) +{ + GEN_ASSERT(num > 0); + va_list va; + va_start(va, num); + do + { + PreprocessorMacro macro = va_arg(va, PreprocessorMacro); + GEN_ASSERT_NOT_NULL(macro.Name.Ptr); + GEN_ASSERT(macro.Name.Len > 0); + + u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); + hashtable_set( _ctx->PreprocessorMacros, key, macro ); + } + while (num--, num > 0); + va_end(va); +} + +void register_preprocess_macros( s32 num, PreprocessorMacro* macros ) +{ + GEN_ASSERT(num > 0); + do + { + PreprocessorMacro macro = * macros; + GEN_ASSERT_NOT_NULL(macro.Name.Ptr); + GEN_ASSERT(macro.Name.Len > 0); + + u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); + hashtable_set( _ctx->PreprocessorMacros, key, macro ); + ++ macros; + } + while (num--, num > 0); +} diff --git a/base/components/interface.hpp b/base/components/interface.hpp index f3e13fd..d18937e 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -114,6 +114,10 @@ GEN_API PreprocessorMacro* lookup_preprocess_macro( Str Name ); // Macros are tracked by name so if the name already exists the entry will be overwritten. GEN_API void register_preprocess_macro( PreprocessorMacro macro ); +// Ease of use batch registration +GEN_API void register_preprocess_macros( s32 num, ... ); +GEN_API void register_preprocess_macros( s32 num, PreprocessorMacro* macros ); + // Used internally to retrive or make string allocations. // Strings are stored in a series of string arenas of fixed size (SizePer_StringArena) GEN_API StrCached cache_str( Str str ); diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 97f3c42..5a0a807 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -118,25 +118,34 @@ s32 lex_preprocessor_define( LexContext* ctx ) Token name = { { ctx->scanner, 1 }, Tok_Identifier, ctx->line, ctx->column, TF_Preprocess }; move_forward(); - PreprocessorMacro macro = { name.Text, MT_Statement, (MacroFlags)0 }; - PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); - if ( registered_macro == nullptr ) { - log_fmt("Warning: %S is was not registered before the lexer processed its #define directive, it will be registered as a statement macro" - , name.Text - ); - } while ( ctx->left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) { move_forward(); name.Text.Len++; } + + Specifier spec = str_to_specifier( name.Text ); + TokType attrib = str_to_toktype( name.Text ); + b32 not_specifier = spec == Spec_Invalid; + b32 not_attribute = attrib <= Tok___Attributes_Start; + + PreprocessorMacro macro = { name.Text, MT_Statement, (MacroFlags)0 }; + PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); + + if ( registered_macro == nullptr && not_specifier && not_attribute ) { + log_fmt("Warning: '%S' was not registered before the lexer processed its #define directive, it will be registered as a statement macro\n" + , name.Text + ); + GEN_DEBUG_TRAP(); + } array_append( _ctx->Lexer_Tokens, name ); if ( ctx->left && (* ctx->scanner) == '(' ) { if (registered_macro && ! macro_is_functional(* registered_macro)) { - log_fmt("Warning: %S registered macro is not flagged as functional yet the definition detects opening parenthesis '(' for arguments" + log_fmt("Warning: %S registered macro is not flagged as functional yet the definition detects opening parenthesis '(' for arguments\n" , name.Text ); + GEN_DEBUG_TRAP(); } else { macro.Flags |= MF_Functional; @@ -151,7 +160,39 @@ s32 lex_preprocessor_define( LexContext* ctx ) while( ctx->left && * ctx->scanner != ')') { skip_whitespace(); - if ( char_is_alpha( (* ctx->scanner) ) || (* ctx->scanner) == '_' ) + + Str possible_varadic = { ctx->scanner, 3 }; + if ( ctx->left > 3 && str_are_equal( txt("..."), possible_varadic ) ) { + Token parameter = { { ctx->scanner, 3 }, Tok_Preprocess_Define_Param, ctx->line, ctx->column, TF_Preprocess }; + move_forward(); + move_forward(); + move_forward(); + + array_append(_ctx->Lexer_Tokens, parameter); + skip_whitespace(); + last_parameter = parameter; + + while ( (* ctx->scanner) == '\\' ) { + move_forward(); + skip_whitespace(); + } + if (* ctx->scanner != ')' ) + { + log_failure("lex_preprocessor_define(%d, %d): Expected a ')' after '...' (varaidc macro param) %S\n" + , ctx->line + , ctx->column + , name.Text + ); + return Lex_ReturnNull; + } + break; + } + else if ( (* ctx->scanner) == '\\' ) { + move_forward(); + skip_whitespace(); + continue; + } + else if ( char_is_alpha( (* ctx->scanner) ) || (* ctx->scanner) == '_' ) { Token parameter = { { ctx->scanner, 1 }, Tok_Preprocess_Define_Param, ctx->line, ctx->column, TF_Preprocess }; move_forward(); @@ -173,6 +214,10 @@ s32 lex_preprocessor_define( LexContext* ctx ) ); return Lex_ReturnNull; } + + if (* ctx->scanner == ')' ) + break; + // There should be a comma if ( * ctx->scanner != ',' ) { log_failure("lex_preprocessor_define(%d, %d): Expected a comma after parameter %S for %S\n" @@ -201,6 +246,15 @@ s32 lex_preprocessor_define( LexContext* ctx ) array_append(_ctx->Lexer_Tokens, closing_paren); move_forward(); } + else if ( registered_macro && macro_is_functional( * registered_macro) ) { + if (registered_macro && ! macro_is_functional(* registered_macro)) { + log_fmt("Warning: %S registered macro is flagged as functional yet the definition detects no opening parenthesis '(' for arguments\n" + , name.Text + ); + GEN_DEBUG_TRAP(); + } + } + if ( registered_macro == nullptr ) { register_preprocess_macro(macro); } diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 46c71c2..749661c 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -1318,10 +1318,13 @@ CodeDefine parse_define() define->Params = params; eat( Tok_Preprocess_Define_Param ); - // #define ( ) + // #define ( } while( left && currtok.Type != Tok_Capture_End ) { + eat( Tok_Comma ); + // #define ( , + CodeDefineParams next_param = (CodeDefineParams) make_code(); next_param->Type = CT_Parameters_Define; next_param->Name = currtok.Text; @@ -1344,7 +1347,7 @@ CodeDefine parse_define() if ( currtok.Text.Len == 0 ) { - define->Body = untyped_str( tok_to_str(currtok) ); + define->Body = untyped_str( txt("\n") ); eat( Tok_Preprocess_Content ); // #define ( ) @@ -1882,7 +1885,6 @@ CodeBody parse_global_nspace( CodeType which ) } } - Member_Resolved_To_Lone_Macro: if ( member == Code_Invalid ) { log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(currtok_noskip), parser_to_strbuilder(_ctx->parser) ); @@ -2930,9 +2932,9 @@ Code parse_simple_preprocess( TokType which ) eat( which ); // - PreprocessorMacro macro = * lookup_preprocess_macro( full_macro.Text ); + PreprocessorMacro* macro = lookup_preprocess_macro( full_macro.Text ); - if ( macro_expects_body(macro) && peektok.Type == Tok_BraceCurly_Open ) + if ( macro && macro_expects_body(* macro) && peektok.Type == Tok_BraceCurly_Open ) { // Eat the block scope right after the macro. Were assuming the macro defines a function definition's signature eat( Tok_BraceCurly_Open ); @@ -2955,7 +2957,7 @@ Code parse_simple_preprocess( TokType which ) // TODO(Ed): Review this? Str prev_proc = _ctx->parser.Scope->Prev->ProcName; - if ( macro.Type == MT_Typename && c_str_compare_len( prev_proc.Ptr, "parser_parse_typedef", prev_proc.Len ) != 0 ) + if ( macro->Type == MT_Typename && c_str_compare_len( prev_proc.Ptr, "parser_parse_typedef", prev_proc.Len ) != 0 ) { if ( check( Tok_Statement_End )) { @@ -2982,7 +2984,7 @@ Code parse_simple_preprocess( TokType which ) // Do nothing goto Leave_Scope_Early; } - else if (macro.Type == MT_Typename && str_contains(_ctx->parser.Scope->Prev->ProcName, txt("parser_parse_typedef"))) + else if (macro && macro->Type == MT_Typename && str_contains(_ctx->parser.Scope->Prev->ProcName, txt("parser_parse_typedef"))) { if ( peektok.Type == Tok_Statement_End ) { diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index f198aa9..ad15bf4 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -132,6 +132,8 @@ enum MacroType : u16 MT_Statement, // A macro is assumed to be a statement if not resolved. MT_Expression, MT_Typename, + MT_Attribute, // More of a note to the parser than anythign else (attributes should be defined in the user attribues def). + MT_Specifier, // More of a note to the parser than anythign else (specifiers should be defined in the user attribues def). MT_Block_Start, // Not Supported yet MT_Block_End, // Not Supported yet MT_Case_Statement, // Not Supported yet @@ -154,12 +156,14 @@ Str macrotype_to_str( MacroType type ) { local_persist Str lookup[] = { - { "Statement", sizeof("Statement") - 1 }, - { "Expression", sizeof("Expression") - 1 }, - { "Typename", sizeof("Typename") - 1 }, - { "Block_Start", sizeof("Block_Start") - 1 }, - { "Block_End", sizeof("Block_End") - 1 }, - { "Case_Statement", sizeof("Case_Statement") - 1 }, + { "Statement", sizeof("Statement") - 1 }, + { "Expression", sizeof("Expression") - 1 }, + { "Typename", sizeof("Typename") - 1 }, + { "Attribute(Macro)", sizeof("Attribute(Macro)") - 1 }, + { "Specifier(Macro)", sizeof("Specifier(Macro)") - 1 }, + { "Block_Start", sizeof("Block_Start") - 1 }, + { "Block_End", sizeof("Block_End") - 1 }, + { "Case_Statement", sizeof("Case_Statement") - 1 }, }; local_persist Str invalid = { "Invalid", sizeof("Invalid") }; diff --git a/base/dependencies/macros.hpp b/base/dependencies/macros.hpp index cbcf0f7..a8636fa 100644 --- a/base/dependencies/macros.hpp +++ b/base/dependencies/macros.hpp @@ -248,9 +248,9 @@ # if ! GEN_COMPILER_C # define typeof decltype # elif defined(_MSC_VER) -# define typeof(x) __typeof__(x) +# define typeof __typeof__ # elif defined(__GNUC__) || defined(__clang__) -# define typeof(x) __typeof__(x) +# define typeof __typeof__ # else # error "Compiler not supported" # endif diff --git a/gen_unreal_engine/enums/ETokType.csv b/gen_unreal_engine/enums/ETokType.csv index d57ad95..739deee 100644 --- a/gen_unreal_engine/enums/ETokType.csv +++ b/gen_unreal_engine/enums/ETokType.csv @@ -51,7 +51,9 @@ Preprocess_EndIf, "endif" Preprocess_Include, "include" Preprocess_Pragma, "pragma" Preprocess_Content, "__macro_content__" -Preprocess_Macro, "__macro__" +Preprocess_Macro_Expr, "__macro_expression__" +Preprocess_Macro_Stmt, "__macro_statment__" +Preprocess_Macro_Typename, "__macro_typename__" Preprocess_Unsupported, "__unsupported__" Spec_Alignas, "alignas" Spec_Const, "const" diff --git a/gen_unreal_engine/unreal.cpp b/gen_unreal_engine/unreal.cpp index 9640340..47d0314 100644 --- a/gen_unreal_engine/unreal.cpp +++ b/gen_unreal_engine/unreal.cpp @@ -64,6 +64,45 @@ int gen_main() Code ue_forceinline = code_str(FORCEINLINE); // Code + register_preprocess_macros( args( + (PreprocessorMacro { txt("bit"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("bitfield_is_set"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_C_LIKE_CPP"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("cast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("ccast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("rcast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("pcast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("scast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("stringize_va"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("stringize"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once_defer"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once_start"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("do_once_end"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("labeled_scope_start"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("labeled_scope_end"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("num_args_impl"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("num_args"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("count_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("clamp"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("is_between"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("size_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("min"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("max"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("offset_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("static_assert"), MT_Statement, MF_Functional }), + (PreprocessorMacro { txt("typeof"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN_API_C_BEGIN"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("GEN_API_C_END"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("nullptr"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN_REMOVE_PTR"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_PARAM_DEFAULT"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("struct_init"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("GEN_OPITMIZE_MAPPINGS_END"), MT_Statement, MF_Null }) + )); + // gen_dep.hpp { CodeBody macros = def_body( CT_Global_Body ); From 4fe1a4da655092f4b27b67bfc205df85c724397c Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 01:27:57 -0500 Subject: [PATCH 06/15] c_library compiles and generates --- base/components/interface.hpp | 1 + base/components/interface.parsing.cpp | 15 ++ base/components/lexer.cpp | 56 ++++--- base/components/parser.cpp | 77 +++++++-- base/components/parser_types.hpp | 10 +- base/dependencies/memory.hpp | 2 +- gen_c_library/c_library.cpp | 150 +++++++++++++++--- gen_c_library/c_library.refactor | 1 + gen_c_library/components/containers.array.hpp | 6 +- .../components/memory.fixed_arena.hpp | 9 +- gen_c_library/components/misc.hpp | 2 +- 11 files changed, 260 insertions(+), 69 deletions(-) diff --git a/base/components/interface.hpp b/base/components/interface.hpp index d18937e..01d3815 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -336,6 +336,7 @@ CodeBody parse_file( Str path ); GEN_API CodeClass parse_class ( Str class_def ); GEN_API CodeConstructor parse_constructor ( Str constructor_def ); +GEN_API CodeDefine parse_define ( Str define_def ); GEN_API CodeDestructor parse_destructor ( Str destructor_def ); GEN_API CodeEnum parse_enum ( Str enum_def ); GEN_API CodeBody parse_export_body ( Str export_def ); diff --git a/base/components/interface.parsing.cpp b/base/components/interface.parsing.cpp index d96e0d6..eea7def 100644 --- a/base/components/interface.parsing.cpp +++ b/base/components/interface.parsing.cpp @@ -82,6 +82,21 @@ CodeConstructor parse_constructor( Str def ) return result; } +CodeDefine parse_define( Str def ) +{ + check_parse_args( def ); + + TokArray toks = lex( def ); + if ( toks.Arr == nullptr ) + return InvalidCode; + + _ctx->parser.Tokens = toks; + push_scope(); + CodeDefine result = parser_parse_define(); + parser_pop(& _ctx->parser); + return result; +} + CodeDestructor parse_destructor( Str def ) { check_parse_args( def ); diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 5a0a807..78c414b 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -123,19 +123,19 @@ s32 lex_preprocessor_define( LexContext* ctx ) name.Text.Len++; } - Specifier spec = str_to_specifier( name.Text ); - TokType attrib = str_to_toktype( name.Text ); + Specifier spec = str_to_specifier( name.Text ); + TokType attrib = str_to_toktype( name.Text ); b32 not_specifier = spec == Spec_Invalid; b32 not_attribute = attrib <= Tok___Attributes_Start; - PreprocessorMacro macro = { name.Text, MT_Statement, (MacroFlags)0 }; + PreprocessorMacro macro = { name.Text, MT_Expression, (MacroFlags)0 }; PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); if ( registered_macro == nullptr && not_specifier && not_attribute ) { - log_fmt("Warning: '%S' was not registered before the lexer processed its #define directive, it will be registered as a statement macro\n" + log_fmt("Warning: '%S' was not registered before the lexer processed its #define directive, it will be registered as a expression macro\n" , name.Text ); - GEN_DEBUG_TRAP(); + // GEN_DEBUG_TRAP(); } array_append( _ctx->Lexer_Tokens, name ); @@ -145,7 +145,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) log_fmt("Warning: %S registered macro is not flagged as functional yet the definition detects opening parenthesis '(' for arguments\n" , name.Text ); - GEN_DEBUG_TRAP(); + // GEN_DEBUG_TRAP(); } else { macro.Flags |= MF_Functional; @@ -526,35 +526,41 @@ void lex_found_token( LexContext* ctx ) } PreprocessorMacro* macro = lookup_preprocess_macro( ctx->token.Text ); - if ( macro ) + b32 has_args = ctx->left && (* ctx->scanner) == '('; + b32 resolved_to_macro = false; + if (macro) { + ctx->token.Type = macrotype_to_toktype(macro->Type); + b32 is_functional = macro_is_functional(* macro); + resolved_to_macro = has_args ? is_functional : ! is_functional; + } + if ( resolved_to_macro ) { - ctx->token.Type = macrotype_to_toktype(macro->Type); - // TODO(Ed): When we introduce a macro AST (and expression support), we'll properly lex this section. // Want to ignore any arguments the define may have as they can be execution expressions. - if ( ctx->left && (* ctx->scanner) == '(' ) + if ( has_args ) { ctx->token.Flags |= TF_Macro_Functional; - move_forward(); - ctx->token.Text.Len++; + // move_forward(); + // ctx->token.Text.Len++; - s32 level = 0; - while ( ctx->left && ((* ctx->scanner) != ')' || level > 0) ) - { - if ( (* ctx->scanner) == '(' ) - level++; + // s32 level = 0; + // while ( ctx->left && ((* ctx->scanner) != ')' || level > 0) ) + // { + // if ( (* ctx->scanner) == '(' ) + // level++; - else if ( (* ctx->scanner) == ')' && level > 0 ) - level--; + // else if ( (* ctx->scanner) == ')' && level > 0 ) + // level--; - move_forward(); - ctx->token.Text.Len++; - } + // move_forward(); + // ctx->token.Text.Len++; + // } - move_forward(); - ctx->token.Text.Len++; + // move_forward(); + // ctx->token.Text.Len++; } + //if ( (* ctx->scanner) == '\r' && ctx->scanner[1] == '\n' ) //{ // move_forward(); @@ -1277,7 +1283,7 @@ TokArray lex( Str content ) { lex_found_token( ctx ); TokType last_type = array_back(_ctx->Lexer_Tokens)->Type; - if ( last_type == Tok_Preprocess_Macro_Stmt ) + if ( last_type == Tok_Preprocess_Macro_Stmt || last_type == Tok_Preprocess_Macro_Expr ) { Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; c.token = thanks_c; diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 749661c..40a5713 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -93,7 +93,15 @@ bool lex__eat(TokArray* self, TokType type ) self->Idx ++; } - if ( at_idx.Type != type ) + b32 not_accepted = at_idx.Type != type; + b32 is_identifier = at_idx.Type == Tok_Identifier; + if ( not_accepted ) + { + PreprocessorMacro* macro = lookup_preprocess_macro(at_idx.Text); + b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier ); + not_accepted = type == Tok_Identifier && accept_as_identifier ? false : true; + } + if ( not_accepted ) { Token tok = * lex_current( self, lex_skip_formatting ); log_failure( "Parse Error, TokArray::eat, Expected: ' %s ' not ' %.*s ' (%d, %d)`\n%s" @@ -188,7 +196,7 @@ internal CodeComment parse_comment (); internal Code parse_complicated_definition ( TokType which ); internal CodeBody parse_class_struct_body ( TokType which, Token name ); internal Code parse_class_struct ( TokType which, bool inplace_def ); -internal CodeDefine parse_define (); +internal CodeDefine parser_parse_define (); internal Code parse_expression (); internal Code parse_forward_or_definition ( TokType which, bool is_inplace ); internal CodeFn parse_function_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type, Token name ); @@ -902,7 +910,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; } case Tok_Preprocess_Define: { - member = cast(Code, parse_define()); + member = cast(Code, parser_parse_define()); // #define break; } @@ -1286,9 +1294,14 @@ Code parse_complicated_definition( TokType which ) } internal inline -CodeDefine parse_define() +CodeDefine parser_parse_define() { push_scope(); + if ( check(Tok_Preprocess_Hash)) { + // If parse_define is called by the user the hash reach here. + eat(Tok_Preprocess_Hash); + } + eat( Tok_Preprocess_Define ); // #define @@ -1698,7 +1711,7 @@ CodeBody parse_global_nspace( CodeType which ) break; case Tok_Preprocess_Define: - member = cast(Code, parse_define()); + member = cast(Code, parser_parse_define()); // #define ... break; @@ -2933,6 +2946,32 @@ Code parse_simple_preprocess( TokType which ) // PreprocessorMacro* macro = lookup_preprocess_macro( full_macro.Text ); + if ( which != Tok_Preprocess_Unsupported && macro == nullptr ) { + log_failure("Expected the macro %S to be registered\n%S", full_macro, parser_to_strbuilder(_ctx->parser)); + return NullCode; + } + + // TODO(Ed) : Parse this properly later (expression and statement support) + if ( macro && macro_is_functional(* macro) ) + { + eat( Tok_Capture_Start ); + + s32 level = 0; + while ( left && ( currtok.Type != Tok_Capture_End || level > 0 ) ) + { + if ( currtok.Type == Tok_Capture_Start ) + level++; + + else if ( currtok.Type == Tok_Capture_End && level > 0 ) + level--; + + eat( currtok.Type ); + } + eat( Tok_Capture_End ); + // ( ) + + full_macro.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)full_macro.Text.Ptr; + } if ( macro && macro_expects_body(* macro) && peektok.Type == Tok_BraceCurly_Open ) { @@ -3619,7 +3658,7 @@ CodeEnum parser_parse_enum( bool inplace_def ) if ( str_contains( tok_to_str(currtok), enum_underlying_macro.Name) ) { use_macro_underlying = true; - underlying_macro = parse_simple_preprocess( Tok_Preprocess_Macro_Typename ); + underlying_macro = parse_simple_preprocess( Tok_Preprocess_Macro_Expr ); } } @@ -3660,7 +3699,7 @@ CodeEnum parser_parse_enum( bool inplace_def ) break; case Tok_Preprocess_Define: - member = cast(Code, parse_define()); + member = cast(Code, parser_parse_define()); // #define break; @@ -4916,22 +4955,32 @@ CodeTypedef parser_parse_typedef() #if GEN_PARSER_DISABLE_MACRO_TYPEDEF if ( false ) #else - if ( check( Tok_Preprocess_Macro_Typename )) + b32 valid_macro = false; + valid_macro |= left && currtok.Type == Tok_Preprocess_Macro_Typename; + valid_macro |= left && currtok.Type == Tok_Preprocess_Macro_Stmt; + // if (currtok.Type == Tok_Preprocess_Macro_Stmt) + // { + // PreprocessMacro* macro = lookup_preprocess_macro(currtok.Text); + // valid_macro |= macro && macro_expects_body(* macro)); + // } + + if ( valid_macro ) #endif { - type = cast(Code, t_empty); - name = currtok; + type = cast(Code, t_empty); + Code macro = parse_simple_preprocess(currtok.Type); + name = currtok; + name.Text.Len = macro->Content.Len; _ctx->parser.Scope->Name = name.Text; - eat( Tok_Preprocess_Macro_Typename ); // typedef if ( currtok.Type == Tok_Identifier ) { - type = untyped_str(name.Text); + type = macro; name = currtok; eat(Tok_Identifier); + // typedef } - // typedef } else { @@ -5178,7 +5227,7 @@ CodeUnion parser_parse_union( bool inplace_def ) break; case Tok_Preprocess_Define: - member = cast(Code, parse_define()); + member = cast(Code, parser_parse_define()); break; case Tok_Preprocess_If: diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index ad15bf4..4421615 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -129,8 +129,8 @@ struct ParseContext enum MacroType : u16 { - MT_Statement, // A macro is assumed to be a statement if not resolved. - MT_Expression, + MT_Expression, // A macro is assumed to be a expression if not resolved. + MT_Statement, MT_Typename, MT_Attribute, // More of a note to the parser than anythign else (attributes should be defined in the user attribues def). MT_Specifier, // More of a note to the parser than anythign else (specifiers should be defined in the user attribues def). @@ -175,8 +175,10 @@ Str macrotype_to_str( MacroType type ) enum EMacroFlags : u16 { - MF_Functional = bit(0), // Macro has parameters (args expected to be passed) - MF_Expects_Body = bit(1), // Expects to assign a braced scope to its body. + MF_Functional = bit(0), // Macro has parameters (args expected to be passed) + MF_Expects_Body = bit(1), // Expects to assign a braced scope to its body. + MF_Allow_As_Identifier = bit(2), // lex__eat wil treat this macro as an identifier if the parser attempts to consume it as one. + // ^^^ This is a sort of kludge because we don't support push/pop macro programs rn. ^^^ MF_Null = 0, MF_UnderlyingType = GEN_U16_MAX, diff --git a/base/dependencies/memory.hpp b/base/dependencies/memory.hpp index 9304b21..c28cb09 100644 --- a/base/dependencies/memory.hpp +++ b/base/dependencies/memory.hpp @@ -225,7 +225,7 @@ forceinline ssize size_remaining(Arena& arena, ssize alignment) { return // This id is defined by Unreal for asserts #pragma push_macro("check") #undef check -forceinline void check(Arena& arena) { return arena_check(& arena); }; +forceinline void check(Arena& arena) { return arena_check(& arena); } #pragma pop_macro("check") #endif diff --git a/gen_c_library/c_library.cpp b/gen_c_library/c_library.cpp index 8c7e2b3..9ed579b 100644 --- a/gen_c_library/c_library.cpp +++ b/gen_c_library/c_library.cpp @@ -65,19 +65,123 @@ int gen_main() Context ctx {}; gen::init(& ctx); - ctx.PreprocessorDefines.append(txt("GEN_API_C_BEGIN")); - ctx.PreprocessorDefines.append(txt("GEN_API_C_END")); - ctx.PreprocessorDefines.append(txt("Array(")); - ctx.PreprocessorDefines.append(txt("HashTable(")); - ctx.PreprocessorDefines.append(txt("GEN_NS_PARSER")); - ctx.PreprocessorDefines.append(txt("GEN_NS_PARSER_BEGIN")); - ctx.PreprocessorDefines.append(txt("GEN_NS_PARSER_END")); - ctx.PreprocessorDefines.append(txt("Using_Code(")); - ctx.PreprocessorDefines.append(txt("Using_CodeOps(")); - ctx.PreprocessorDefines.append(txt("GEN_OPTIMIZE_MAPPINGS_BEGIN")); - ctx.PreprocessorDefines.append(txt("GEN_OPITMIZE_MAPPINGS_END")); - ctx.PreprocessorDefines.append(txt("GEN_PARAM_DEFAULT")); - //PreprocessorDefines.append(txt("GEN_EXECUTION_EXPRESSION_SUPPORT")); + register_preprocess_macros( args( + (PreprocessorMacro { txt("bit"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("bitfield_is_set"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_C_LIKE_CPP"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("cast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("ccast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("rcast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("pcast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("scast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("stringize_va"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("stringize"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once_defer"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("do_once_start"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("do_once_end"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("labeled_scope_start"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("labeled_scope_end"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("num_args_impl"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("num_args"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("count_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("clamp"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("is_between"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("size_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("min"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("max"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("offset_of"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("static_assert"), MT_Statement, MF_Functional }), + (PreprocessorMacro { txt("typeof"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN_API_C_BEGIN"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("GEN_API_C_END"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("nullptr"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN_REMOVE_PTR"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_PARAM_DEFAULT"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("struct_init"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("GEN_OPITMIZE_MAPPINGS_END"), MT_Statement, MF_Null }), + (PreprocessorMacro { txt("Array"), MT_Typename, MF_Functional }), + (PreprocessorMacro { txt("HashTable"), MT_Typename, MF_Functional }), + (PreprocessorMacro { txt("Using_Code"), MT_Statement, MF_Functional }), + (PreprocessorMacro { txt("Using_CodeOps"), MT_Statement, MF_Functional }), + (PreprocessorMacro { txt("kilobytes"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("megabytes"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("gigabytes"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("terabytes"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN__ONES"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN__HIGHS"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN__HAS_ZERO"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("zero_item"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("zero_array"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_DEFAULT_MEMORY_ALIGNMENT"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("GEN_DEFAULT_ALLOCATOR_FLAGS"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("alloc_item"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("alloc_array"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("malloc"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("mfree"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_PRINTF_MAXLEN"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("cast_to_str"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("current"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("txt"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("GEN_FILE_OPEN_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (PreprocessorMacro { txt("GEN_FILE_READ_AT_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (PreprocessorMacro { txt("GEN_FILE_WRITE_AT_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (PreprocessorMacro { txt("GEN_FILE_SEEK_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (PreprocessorMacro { txt("GEN_FILE_CLOSE_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (PreprocessorMacro { txt("log_failure"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("operator"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("InvalidCode"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("NullCode"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("Verify_POD"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("gen_main"), MT_Statement, MF_Null }) + )); + register_preprocess_macros( args( + (PreprocessorMacro { txt("name"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("code"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("args"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("code_str"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("code_fmt"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("parse_fmt"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("token_fmt"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("check_member_val"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("check_member_str"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("check_member_content"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("check_member_ast"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("check_params"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("check_param_eq_ret"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("specs"), MT_Expression, MF_Functional | MF_Allow_As_Identifier }), + (PreprocessorMacro { txt("name_check"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("null_check"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("def_body_start"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("def_body_code_array_start"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("move_forward"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("skip_whitespace"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("end_line"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("check_parse_args"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("currtok_noskip"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("currtok"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("peektok"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("prevtok"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("nexttok"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("nexttok_noskip"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("eat"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("left"), MT_Expression, MF_Null | MF_Allow_As_Identifier }), + (PreprocessorMacro { txt("def_assign"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("CHECK_WAS_DEFINED"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("check_noskip"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("check"), MT_Expression, MF_Functional | MF_Allow_As_Identifier }), + (PreprocessorMacro { txt("push_scope"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("cut_length"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("cut_ptr"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("pos"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("move_fwd"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("Entry"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("CheckEndParams"), MT_Expression, MF_Functional }), + (PreprocessorMacro { txt("UseTemplateCapture"), MT_Expression, MF_Null }), + (PreprocessorMacro { txt("check_current"), MT_Expression, MF_Functional }) + )); Code push_ignores = scan_file( path_base "helpers/push_ignores.inline.hpp" ); Code pop_ignores = scan_file( path_base "helpers/pop_ignores.inline.hpp" ); @@ -139,8 +243,8 @@ int gen_main() CodeTemplate tmpl = cast(CodeTemplate, entry); if ( tmpl->Declaration->Name.contains(txt("swap"))) { - log_fmt("SWAPPED"); - CodeBody macro_swap = parse_global_body( txt(R"( + register_preprocess_macro({ txt("swap"), MT_Expression, MF_Functional }); + CodeDefine macro_swap = parse_define( txt(R"( #define swap( a, b ) \ do \ { \ @@ -241,7 +345,8 @@ do \ { if ( str_contains(entry->Name, txt("Msg_Invalid_Value"))) { - CodeDefine define = def_define(entry->Name, entry->Value->Content); + Opts_def_define opts = { {}, entry->Value->Content }; + CodeDefine define = def_define(entry->Name, MT_Expression, opts ); header_printing.append(define); continue; } @@ -407,7 +512,8 @@ do \ CodeVar var = cast(CodeVar, entry); if (var->Specs.has(Spec_Constexpr) > -1) { - CodeDefine define = def_define(entry->Name, entry->Value->Content); + Opts_def_define opts = { {}, entry->Value->Content }; + CodeDefine define = def_define(entry->Name, MT_Expression, opts); header_filesystem.append(define); continue; } @@ -811,7 +917,8 @@ R"(#define AST_ArrSpecs_Cap \ ast.append(def); break; } - CodeDefine def = def_define(var->Name, var->Value.to_strbuilder()); + Opts_def_define opts = { {}, var->Value.to_strbuilder() }; + CodeDefine def = def_define(var->Name, MT_Expression, opts ); ast.append(def); break; } @@ -832,6 +939,7 @@ R"(#define AST_ArrSpecs_Cap \ txt("CodeClass"), txt("CodeConstructor"), txt("CodeDefine"), + txt("CodeDefineParams"), txt("CodeDestructor"), txt("CodeEnum"), txt("CodeExec"), @@ -1209,8 +1317,10 @@ R"(#define ( code ) _Generic( (code), \ if (var->Specs) { s32 constexpr_found = var->Specs.remove( Spec_Constexpr ); - if (constexpr_found > -1) { - CodeDefine define = def_define(entry->Name, entry->Value->Content); + if (constexpr_found > -1) + { + Opts_def_define opts = { {}, entry->Value->Content }; + CodeDefine define = def_define(entry->Name, MT_Expression, opts ); header_end.append(define); continue; } diff --git a/gen_c_library/c_library.refactor b/gen_c_library/c_library.refactor index c743e5c..569be07 100644 --- a/gen_c_library/c_library.refactor +++ b/gen_c_library/c_library.refactor @@ -325,6 +325,7 @@ word CodeComment, gen_CodeComment word CodeClass, gen_CodeClass word CodeConstructor, gen_CodeConstructor word CodeDefine, gen_CodeDefine +word CodeDefineParams, gen_CodeDefineParams word CodeDestructor, gen_CodeDestructor word CodeEnum, gen_CodeEnum word CodeExec, gen_CodeExec diff --git a/gen_c_library/components/containers.array.hpp b/gen_c_library/components/containers.array.hpp index 2317375..e2fd8b3 100644 --- a/gen_c_library/components/containers.array.hpp +++ b/gen_c_library/components/containers.array.hpp @@ -22,9 +22,9 @@ CodeBody gen_array_base() Code get_header = untyped_str( txt( "#define array_get_header( self ) ( (ArrayHeader*)( self ) - 1)\n" )); Code type_define = untyped_str( txt( "#define Array(Type) gen_Array_##Type\n")); - Code array_begin = def_define(txt("array_begin(array)"), code( (array) )); - Code array_end = def_define(txt("array_end(array)"), code( (array + array_get_header(array)->Num ) )); - Code array_next = def_define(txt("array_next(array, entry)"), code( (entry + 1) )); + Code array_begin = def_define(txt("array_begin(array)"), MT_Expression, { {}, code( (array) ) } ); + Code array_end = def_define(txt("array_end(array)"), MT_Expression, { {}, code( (array + array_get_header(array)->Num ) ) } ); + Code array_next = def_define(txt("array_next(array, entry)"), MT_Expression, { {}, code( (entry + 1) ) } ); return def_global_body( args( fmt_newline, diff --git a/gen_c_library/components/memory.fixed_arena.hpp b/gen_c_library/components/memory.fixed_arena.hpp index 262f06d..c9fd7ab 100644 --- a/gen_c_library/components/memory.fixed_arena.hpp +++ b/gen_c_library/components/memory.fixed_arena.hpp @@ -89,7 +89,14 @@ CodeBody gen_fixed_arenas() result.append(arena_interface_2mb); result.append(arena_interface_4mb); - CodeDefine def = def_define(txt("fixed_arena_allocator_info(fixed_arena)"), txt("( (AllocatorInfo) { arena_allocator_proc, & (fixed_arena)->arena } )")); + register_preprocess_macros( args( + ( PreprocessorMacro { txt("fixed_arena_allocator_info"), MT_Expression, MF_Functional }), + ( PreprocessorMacro { txt("fixed_arena_init"), MT_Expression, MF_Functional }), + ( PreprocessorMacro { txt("fixed_arena_free"), MT_Expression, MF_Functional }), + ( PreprocessorMacro { txt("fixed_arena_size_remaining"), MT_Expression, MF_Functional }) + )); + + CodeDefine def = parse_define(txt("#define fixed_arena_allocator_info(fixed_arena) ( (AllocatorInfo) { arena_allocator_proc, & (fixed_arena)->arena } )\n")); result.append(def); result.append(fmt_newline); diff --git a/gen_c_library/components/misc.hpp b/gen_c_library/components/misc.hpp index 727d2b6..b288663 100644 --- a/gen_c_library/components/misc.hpp +++ b/gen_c_library/components/misc.hpp @@ -7,7 +7,7 @@ void convert_cpp_enum_to_c( CodeEnum to_convert, CodeBody to_append ) { #pragma push_macro("enum_underlying") #undef enum_underlying - StrCached type = to_convert->UnderlyingType ? to_convert->UnderlyingType : to_convert->Name + StrCached type = to_convert->UnderlyingType ? to_convert->UnderlyingType.to_strbuilder().to_str() : to_convert->Name; CodeTypedef tdef = parse_typedef(token_fmt("type", type, "name", to_convert->Name, stringize( typedef enum ; ))); if (to_convert->UnderlyingType) { From 794695401746fa65aafe20c77aae361c5e8b74ff Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 01:55:22 -0500 Subject: [PATCH 07/15] fixing post-gen c-library compiler errors (still quite a few togo --- base/components/code_serialization.cpp | 2 +- base/components/parser.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/base/components/code_serialization.cpp b/base/components/code_serialization.cpp index c9678a1..94e601a 100644 --- a/base/components/code_serialization.cpp +++ b/base/components/code_serialization.cpp @@ -250,7 +250,7 @@ void define_to_strbuilder_ref(CodeDefine define, StrBuilder* result ) { GEN_ASSERT(define); GEN_ASSERT(define->Body); - GEN_ASSERT(define->Body->Content); + GEN_ASSERT(define->Body->Content.Ptr && define->Body->Content.Len > 0); if (define->Params) { StrBuilder params_builder = define_params_to_strbuilder(define->Params); strbuilder_append_fmt( result, "#define %S(%S) %S", define->Name, strbuilder_to_str(params_builder), define->Body->Content ); diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 40a5713..f1c5634 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -1327,9 +1327,9 @@ CodeDefine parser_parse_define() params = (CodeDefineParams) make_code(); params->Type = CT_Parameters_Define; params->Name = currtok.Text; + params->NumEntries ++; define->Params = params; - eat( Tok_Preprocess_Define_Param ); // #define ( } @@ -4964,12 +4964,13 @@ CodeTypedef parser_parse_typedef() // valid_macro |= macro && macro_expects_body(* macro)); // } + Code macro; if ( valid_macro ) #endif { type = cast(Code, t_empty); - Code macro = parse_simple_preprocess(currtok.Type); name = currtok; + Code macro = parse_simple_preprocess(currtok.Type); name.Text.Len = macro->Content.Len; _ctx->parser.Scope->Name = name.Text; // typedef From e9752cb906eeb1f50f31a1fd677815009b04b813 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 10:08:28 -0500 Subject: [PATCH 08/15] generated c_library compiles --- base/components/header_end.hpp | 2 +- base/components/interface.cpp | 72 ++--- base/components/interface.hpp | 10 +- base/components/interface.upfront.cpp | 4 +- base/components/lexer.cpp | 8 +- base/components/parser.cpp | 180 +++++------ base/components/parser_types.hpp | 8 +- base/components/static_data.cpp | 2 +- gen_c_library/c_library.cpp | 292 ++++++++++-------- gen_c_library/c_library.refactor | 8 + .../components/containers.hashtable.hpp | 2 +- .../components/memory.fixed_arena.hpp | 10 +- gen_unreal_engine/unreal.cpp | 74 ++--- 13 files changed, 356 insertions(+), 316 deletions(-) diff --git a/base/components/header_end.hpp b/base/components/header_end.hpp index edf639d..282d4f8 100644 --- a/base/components/header_end.hpp +++ b/base/components/header_end.hpp @@ -6,7 +6,7 @@ #pragma region Constants -extern PreprocessorMacro enum_underlying_macro; +extern Macro enum_underlying_macro; extern Code access_public; extern Code access_protected; diff --git a/base/components/interface.cpp b/base/components/interface.cpp index a5a21ec..74ac369 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -179,28 +179,28 @@ void define_constants() #endif spec_const = def_specifier( Spec_Const); code_set_global( cast(Code, spec_const )); - spec_consteval = def_specifier( Spec_Consteval); code_set_global( cast(Code, spec_consteval ));; - spec_constexpr = def_specifier( Spec_Constexpr); code_set_global( cast(Code, spec_constexpr ));; - spec_constinit = def_specifier( Spec_Constinit); code_set_global( cast(Code, spec_constinit ));; - spec_extern_linkage = def_specifier( Spec_External_Linkage); code_set_global( cast(Code, spec_extern_linkage ));; - spec_final = def_specifier( Spec_Final); code_set_global( cast(Code, spec_final ));; - spec_forceinline = def_specifier( Spec_ForceInline); code_set_global( cast(Code, spec_forceinline ));; - spec_global = def_specifier( Spec_Global); code_set_global( cast(Code, spec_global ));; - spec_inline = def_specifier( Spec_Inline); code_set_global( cast(Code, spec_inline ));; - spec_internal_linkage = def_specifier( Spec_Internal_Linkage); code_set_global( cast(Code, spec_internal_linkage ));; - spec_local_persist = def_specifier( Spec_Local_Persist); code_set_global( cast(Code, spec_local_persist ));; - spec_mutable = def_specifier( Spec_Mutable); code_set_global( cast(Code, spec_mutable ));; - spec_neverinline = def_specifier( Spec_NeverInline); code_set_global( cast(Code, spec_neverinline ));; - spec_noexcept = def_specifier( Spec_NoExceptions); code_set_global( cast(Code, spec_noexcept ));; - spec_override = def_specifier( Spec_Override); code_set_global( cast(Code, spec_override ));; - spec_ptr = def_specifier( Spec_Ptr); code_set_global( cast(Code, spec_ptr ));; + spec_consteval = def_specifier( Spec_Consteval); code_set_global( cast(Code, spec_consteval )); + spec_constexpr = def_specifier( Spec_Constexpr); code_set_global( cast(Code, spec_constexpr )); + spec_constinit = def_specifier( Spec_Constinit); code_set_global( cast(Code, spec_constinit )); + spec_extern_linkage = def_specifier( Spec_External_Linkage); code_set_global( cast(Code, spec_extern_linkage )); + spec_final = def_specifier( Spec_Final); code_set_global( cast(Code, spec_final )); + spec_forceinline = def_specifier( Spec_ForceInline); code_set_global( cast(Code, spec_forceinline )); + spec_global = def_specifier( Spec_Global); code_set_global( cast(Code, spec_global )); + spec_inline = def_specifier( Spec_Inline); code_set_global( cast(Code, spec_inline )); + spec_internal_linkage = def_specifier( Spec_Internal_Linkage); code_set_global( cast(Code, spec_internal_linkage )); + spec_local_persist = def_specifier( Spec_Local_Persist); code_set_global( cast(Code, spec_local_persist )); + spec_mutable = def_specifier( Spec_Mutable); code_set_global( cast(Code, spec_mutable )); + spec_neverinline = def_specifier( Spec_NeverInline); code_set_global( cast(Code, spec_neverinline )); + spec_noexcept = def_specifier( Spec_NoExceptions); code_set_global( cast(Code, spec_noexcept )); + spec_override = def_specifier( Spec_Override); code_set_global( cast(Code, spec_override )); + spec_ptr = def_specifier( Spec_Ptr); code_set_global( cast(Code, spec_ptr )); spec_pure = def_specifier( Spec_Pure); code_set_global( cast(Code, spec_pure )); - spec_ref = def_specifier( Spec_Ref); code_set_global( cast(Code, spec_ref ));; - spec_register = def_specifier( Spec_Register); code_set_global( cast(Code, spec_register ));; - spec_rvalue = def_specifier( Spec_RValue); code_set_global( cast(Code, spec_rvalue ));; - spec_static_member = def_specifier( Spec_Static); code_set_global( cast(Code, spec_static_member ));; - spec_thread_local = def_specifier( Spec_Thread_Local); code_set_global( cast(Code, spec_thread_local ));; - spec_virtual = def_specifier( Spec_Virtual); code_set_global( cast(Code, spec_virtual ));; + spec_ref = def_specifier( Spec_Ref); code_set_global( cast(Code, spec_ref )); + spec_register = def_specifier( Spec_Register); code_set_global( cast(Code, spec_register )); + spec_rvalue = def_specifier( Spec_RValue); code_set_global( cast(Code, spec_rvalue )); + spec_static_member = def_specifier( Spec_Static); code_set_global( cast(Code, spec_static_member )); + spec_thread_local = def_specifier( Spec_Thread_Local); code_set_global( cast(Code, spec_thread_local )); + spec_virtual = def_specifier( Spec_Virtual); code_set_global( cast(Code, spec_virtual )); spec_volatile = def_specifier( Spec_Volatile); code_set_global( cast(Code, spec_volatile )); spec_local_persist = def_specifiers( 1, Spec_Local_Persist ); @@ -211,7 +211,7 @@ void define_constants() enum_underlying_macro.Type = MT_Expression; enum_underlying_macro.Flags = MF_Functional; } - register_preprocess_macro(enum_underlying_macro); + register_macro(enum_underlying_macro); } void init(Context* ctx) @@ -315,8 +315,8 @@ void init(Context* ctx) if ( ctx->StrCache.Entries == nullptr ) GEN_FATAL( "gen::init: Failed to initialize the StringCache"); - ctx->PreprocessorMacros = hashtable_init(PreprocessorMacro, ctx->Allocator_DyanmicContainers); - if (ctx->PreprocessorMacros.Hashes == nullptr || ctx->PreprocessorMacros.Entries == nullptr) { + ctx->Macros = hashtable_init(Macro, ctx->Allocator_DyanmicContainers); + if (ctx->Macros.Hashes == nullptr || ctx->Macros.Entries == nullptr) { GEN_FATAL( "gen::init: Failed to initialize the PreprocessMacros table" ); } } @@ -356,7 +356,7 @@ void deinit(Context* ctx) array_free( ctx->CodePools); array_free( ctx->StringArenas); - hashtable_destroy(ctx->PreprocessorMacros); + hashtable_destroy(ctx->Macros); left = array_num( ctx->Fallback_AllocatorBuckets); if (left) @@ -401,7 +401,7 @@ void reset(Context* ctx) while ( left--, left ); hashtable_clear(ctx->StrCache); - hashtable_clear(ctx->PreprocessorMacros); + hashtable_clear(ctx->Macros); define_constants(); } @@ -464,47 +464,47 @@ Code make_code() return result; } -PreprocessorMacro* lookup_preprocess_macro( Str name ) { +Macro* lookup_macro( Str name ) { u32 key = crc32( name.Ptr, name.Len ); - return hashtable_get( _ctx->PreprocessorMacros, key ); + return hashtable_get( _ctx->Macros, key ); } -void register_preprocess_macro( PreprocessorMacro macro ) { +void register_macro( Macro macro ) { GEN_ASSERT_NOT_NULL(macro.Name.Ptr); GEN_ASSERT(macro.Name.Len > 0); u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); - hashtable_set( _ctx->PreprocessorMacros, key, macro ); + hashtable_set( _ctx->Macros, key, macro ); } -void register_preprocess_macros( s32 num, ... ) +void register_macros( s32 num, ... ) { GEN_ASSERT(num > 0); va_list va; va_start(va, num); do { - PreprocessorMacro macro = va_arg(va, PreprocessorMacro); + Macro macro = va_arg(va, Macro); GEN_ASSERT_NOT_NULL(macro.Name.Ptr); GEN_ASSERT(macro.Name.Len > 0); u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); - hashtable_set( _ctx->PreprocessorMacros, key, macro ); + hashtable_set( _ctx->Macros, key, macro ); } while (num--, num > 0); va_end(va); } -void register_preprocess_macros( s32 num, PreprocessorMacro* macros ) +void register_macros( s32 num, Macro* macros ) { GEN_ASSERT(num > 0); do { - PreprocessorMacro macro = * macros; + Macro macro = * macros; GEN_ASSERT_NOT_NULL(macro.Name.Ptr); GEN_ASSERT(macro.Name.Len > 0); u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); - hashtable_set( _ctx->PreprocessorMacros, key, macro ); + hashtable_set( _ctx->Macros, key, macro ); ++ macros; } while (num--, num > 0); diff --git a/base/components/interface.hpp b/base/components/interface.hpp index 01d3815..cb70376 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -71,7 +71,7 @@ struct Context // Used by the lexer to persistently treat all these identifiers as preprocessor defines. // Populate with strings via gen::cache_str. // Functional defines must have format: id( ;at minimum to indicate that the define is only valid with arguments. - HashTable(PreprocessorMacro) PreprocessorMacros; + MacroTable Macros; // Backend @@ -107,16 +107,16 @@ GEN_API void reset(Context* ctx); GEN_API void set_context(Context* ctx); // Mostly intended for the parser -GEN_API PreprocessorMacro* lookup_preprocess_macro( Str Name ); +GEN_API Macro* lookup_macro( Str Name ); // Alternative way to add a preprocess define entry for the lexer & parser to utilize // if the user doesn't want to use def_define // Macros are tracked by name so if the name already exists the entry will be overwritten. -GEN_API void register_preprocess_macro( PreprocessorMacro macro ); +GEN_API void register_macro( Macro macro ); // Ease of use batch registration -GEN_API void register_preprocess_macros( s32 num, ... ); -GEN_API void register_preprocess_macros( s32 num, PreprocessorMacro* macros ); +GEN_API void register_macros( s32 num, ... ); +GEN_API void register_macros( s32 num, Macro* macros ); // Used internally to retrive or make string allocations. // Strings are stored in a series of string arenas of fixed size (SizePer_StringArena) diff --git a/base/components/interface.upfront.cpp b/base/components/interface.upfront.cpp index 271bb68..d6be498 100644 --- a/base/components/interface.upfront.cpp +++ b/base/components/interface.upfront.cpp @@ -578,8 +578,8 @@ CodeDefine def_define( Str name, MacroType type, Opts_def_define p ) b32 register_define = ! p.dont_register_to_preprocess_macros; if ( register_define ) { - PreprocessorMacro macro_entry = { result->Name, type, p.flags }; - register_preprocess_macro(macro_entry); + Macro macro_entry = { result->Name, type, p.flags }; + register_macro(macro_entry); } return result; } diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 78c414b..289983b 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -128,8 +128,8 @@ s32 lex_preprocessor_define( LexContext* ctx ) b32 not_specifier = spec == Spec_Invalid; b32 not_attribute = attrib <= Tok___Attributes_Start; - PreprocessorMacro macro = { name.Text, MT_Expression, (MacroFlags)0 }; - PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text); + Macro macro = { name.Text, MT_Expression, (MacroFlags)0 }; + Macro* registered_macro = lookup_macro(name.Text); if ( registered_macro == nullptr && not_specifier && not_attribute ) { log_fmt("Warning: '%S' was not registered before the lexer processed its #define directive, it will be registered as a expression macro\n" @@ -256,7 +256,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) } if ( registered_macro == nullptr ) { - register_preprocess_macro(macro); + register_macro(macro); } // Define's content handled by lex_preprocessor_directive (the original caller of this) @@ -525,7 +525,7 @@ void lex_found_token( LexContext* ctx ) return; } - PreprocessorMacro* macro = lookup_preprocess_macro( ctx->token.Text ); + Macro* macro = lookup_macro( ctx->token.Text ); b32 has_args = ctx->left && (* ctx->scanner) == '('; b32 resolved_to_macro = false; if (macro) { diff --git a/base/components/parser.cpp b/base/components/parser.cpp index f1c5634..8cbd4d8 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -97,7 +97,7 @@ bool lex__eat(TokArray* self, TokType type ) b32 is_identifier = at_idx.Type == Tok_Identifier; if ( not_accepted ) { - PreprocessorMacro* macro = lookup_preprocess_macro(at_idx.Text); + Macro* macro = lookup_macro(at_idx.Text); b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier ); not_accepted = type == Tok_Identifier && accept_as_identifier ? false : true; } @@ -196,7 +196,6 @@ internal CodeComment parse_comment (); internal Code parse_complicated_definition ( TokType which ); internal CodeBody parse_class_struct_body ( TokType which, Token name ); internal Code parse_class_struct ( TokType which, bool inplace_def ); -internal CodeDefine parser_parse_define (); internal Code parse_expression (); internal Code parse_forward_or_definition ( TokType which, bool is_inplace ); internal CodeFn parse_function_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type, Token name ); @@ -216,8 +215,9 @@ internal void parse_template_args ( Token* token ); internal CodeVar parse_variable_after_name ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename type, Str name ); internal CodeVar parse_variable_declaration_list (); -internal CodeClass parser_parse_class ( bool inplace_def ); +internal CodeClass parser_parse_class ( bool inplace_def ); internal CodeConstructor parser_parse_constructor ( CodeSpecifiers specifiers ); +internal CodeDefine parser_parse_define (); internal CodeDestructor parser_parse_destructor ( CodeSpecifiers specifiers ); internal CodeEnum parser_parse_enum ( bool inplace_def ); internal CodeBody parser_parse_export_body (); @@ -1293,89 +1293,6 @@ Code parse_complicated_definition( TokType which ) } } -internal inline -CodeDefine parser_parse_define() -{ - push_scope(); - if ( check(Tok_Preprocess_Hash)) { - // If parse_define is called by the user the hash reach here. - eat(Tok_Preprocess_Hash); - } - - eat( Tok_Preprocess_Define ); - // #define - - CodeDefine - define = (CodeDefine) make_code(); - define->Type = CT_Preprocess_Define; - if ( ! check( Tok_Identifier ) ) { - log_failure( "Error, expected identifier after #define\n%s", parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); - return InvalidCode; - } - _ctx->parser.Scope->Name = currtok.Text; - define->Name = cache_str( tok_to_str(currtok) ); - eat( Tok_Identifier ); - // #define - - PreprocessorMacro* macro = lookup_preprocess_macro(define->Name); - if (macro_is_functional(* macro)) { - eat( Tok_Capture_Start ); - // #define ( - CodeDefineParams params; - if ( left && currtok.Type != Tok_Capture_End ) { - params = (CodeDefineParams) make_code(); - params->Type = CT_Parameters_Define; - params->Name = currtok.Text; - params->NumEntries ++; - - define->Params = params; - eat( Tok_Preprocess_Define_Param ); - // #define ( - } - - while( left && currtok.Type != Tok_Capture_End ) { - eat( Tok_Comma ); - // #define ( , - - CodeDefineParams next_param = (CodeDefineParams) make_code(); - next_param->Type = CT_Parameters_Define; - next_param->Name = currtok.Text; - define_params_append(params, next_param); - - // #define ( , ... - eat( Tok_Preprocess_Define_Param ); - } - - eat( Tok_Capture_End ); - // #define ( ) - } - - if ( ! check( Tok_Preprocess_Content )) - { - log_failure( "Error, expected content after #define %s\n%s", define->Name, parser_to_strbuilder(_ctx->parser) ); - parser_pop(& _ctx->parser); - return InvalidCode; - } - - if ( currtok.Text.Len == 0 ) - { - define->Body = untyped_str( txt("\n") ); - eat( Tok_Preprocess_Content ); - // #define ( ) - - parser_pop(& _ctx->parser); - return define; - } - - define->Body = untyped_str( strbuilder_to_str( parser_strip_formatting( tok_to_str(currtok), parser_strip_formatting_dont_preserve_newlines )) ); - eat( Tok_Preprocess_Content ); - // #define ( ) - - parser_pop(& _ctx->parser); - return define; -} - internal inline Code parse_assignment_expression() { @@ -2945,7 +2862,7 @@ Code parse_simple_preprocess( TokType which ) eat( which ); // - PreprocessorMacro* macro = lookup_preprocess_macro( full_macro.Text ); + Macro* macro = lookup_macro( full_macro.Text ); if ( which != Tok_Preprocess_Unsupported && macro == nullptr ) { log_failure("Expected the macro %S to be registered\n%S", full_macro, parser_to_strbuilder(_ctx->parser)); return NullCode; @@ -3484,6 +3401,93 @@ CodeConstructor parser_parse_constructor( CodeSpecifiers specifiers ) return result; } +internal inline +CodeDefine parser_parse_define() +{ + push_scope(); + if ( check(Tok_Preprocess_Hash)) { + // If parse_define is called by the user the hash reach here. + eat(Tok_Preprocess_Hash); + } + + eat( Tok_Preprocess_Define ); + // #define + + CodeDefine + define = (CodeDefine) make_code(); + define->Type = CT_Preprocess_Define; + if ( ! check( Tok_Identifier ) ) { + log_failure( "Error, expected identifier after #define\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); + return InvalidCode; + } + _ctx->parser.Scope->Name = currtok.Text; + define->Name = cache_str( tok_to_str(currtok) ); + eat( Tok_Identifier ); + // #define + + Macro* macro = lookup_macro(define->Name); + if (macro_is_functional(* macro)) { + eat( Tok_Capture_Start ); + // #define ( + + // We provide the define params even if empty to make sure '()' are serialized. + CodeDefineParams + params = (CodeDefineParams) make_code(); + params->Type = CT_Parameters_Define; + + if ( left && currtok.Type != Tok_Capture_End ) { + params->Name = currtok.Text; + params->NumEntries ++; + + eat( Tok_Preprocess_Define_Param ); + // #define ( + } + + while( left && currtok.Type != Tok_Capture_End ) { + eat( Tok_Comma ); + // #define ( , + + CodeDefineParams next_param = (CodeDefineParams) make_code(); + next_param->Type = CT_Parameters_Define; + next_param->Name = currtok.Text; + define_params_append(params, next_param); + + // #define ( , ... + eat( Tok_Preprocess_Define_Param ); + } + + eat( Tok_Capture_End ); + // #define ( ) + + define->Params = params; + } + + if ( ! check( Tok_Preprocess_Content )) + { + log_failure( "Error, expected content after #define %s\n%s", define->Name, parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); + return InvalidCode; + } + + if ( currtok.Text.Len == 0 ) + { + define->Body = untyped_str( txt("\n") ); + eat( Tok_Preprocess_Content ); + // #define ( ) + + parser_pop(& _ctx->parser); + return define; + } + + define->Body = untyped_str( strbuilder_to_str( parser_strip_formatting( tok_to_str(currtok), parser_strip_formatting_dont_preserve_newlines )) ); + eat( Tok_Preprocess_Content ); + // #define ( ) + + parser_pop(& _ctx->parser); + return define; +} + internal CodeDestructor parser_parse_destructor( CodeSpecifiers specifiers ) { @@ -4960,7 +4964,7 @@ CodeTypedef parser_parse_typedef() valid_macro |= left && currtok.Type == Tok_Preprocess_Macro_Stmt; // if (currtok.Type == Tok_Preprocess_Macro_Stmt) // { - // PreprocessMacro* macro = lookup_preprocess_macro(currtok.Text); + // PreprocessMacro* macro = lookup_macro(currtok.Text); // valid_macro |= macro && macro_expects_body(* macro)); // } diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index 4421615..fd03708 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -185,7 +185,7 @@ enum EMacroFlags : u16 }; typedef u16 MacroFlags; -struct PreprocessorMacro +struct Macro { StrCached Name; MacroType Type; @@ -193,11 +193,13 @@ struct PreprocessorMacro }; forceinline -b32 macro_is_functional( PreprocessorMacro macro ) { +b32 macro_is_functional( Macro macro ) { return bitfield_is_set( b16, macro.Flags, MF_Functional ); } forceinline -b32 macro_expects_body( PreprocessorMacro macro ) { +b32 macro_expects_body( Macro macro ) { return bitfield_is_set( b16, macro.Flags, MF_Expects_Body ); } + +typedef HashTable(Macro) MacroTable; diff --git a/base/components/static_data.cpp b/base/components/static_data.cpp index 70a6ff2..3c9c0f1 100644 --- a/base/components/static_data.cpp +++ b/base/components/static_data.cpp @@ -9,7 +9,7 @@ global Context* _ctx; #pragma region Constants global u32 context_counter; -global PreprocessorMacro enum_underlying_macro; +global Macro enum_underlying_macro; global Code Code_Global; global Code Code_Invalid; diff --git a/gen_c_library/c_library.cpp b/gen_c_library/c_library.cpp index 9ed579b..275ddff 100644 --- a/gen_c_library/c_library.cpp +++ b/gen_c_library/c_library.cpp @@ -65,122 +65,122 @@ int gen_main() Context ctx {}; gen::init(& ctx); - register_preprocess_macros( args( - (PreprocessorMacro { txt("bit"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("bitfield_is_set"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_C_LIKE_CPP"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("cast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("ccast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("rcast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("pcast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("scast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("stringize_va"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("stringize"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("do_once"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("do_once_defer"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("do_once_start"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("do_once_end"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("labeled_scope_start"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("labeled_scope_end"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("num_args_impl"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("num_args"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("count_of"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("clamp"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("is_between"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("size_of"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("min"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("max"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("offset_of"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("static_assert"), MT_Statement, MF_Functional }), - (PreprocessorMacro { txt("typeof"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("GEN_API_C_BEGIN"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("GEN_API_C_END"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("nullptr"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("GEN_REMOVE_PTR"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_PARAM_DEFAULT"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("struct_init"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("GEN_OPITMIZE_MAPPINGS_END"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("Array"), MT_Typename, MF_Functional }), - (PreprocessorMacro { txt("HashTable"), MT_Typename, MF_Functional }), - (PreprocessorMacro { txt("Using_Code"), MT_Statement, MF_Functional }), - (PreprocessorMacro { txt("Using_CodeOps"), MT_Statement, MF_Functional }), - (PreprocessorMacro { txt("kilobytes"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("megabytes"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("gigabytes"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("terabytes"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN__ONES"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("GEN__HIGHS"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("GEN__HAS_ZERO"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("zero_item"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("zero_array"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_DEFAULT_MEMORY_ALIGNMENT"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("GEN_DEFAULT_ALLOCATOR_FLAGS"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("alloc_item"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("alloc_array"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("malloc"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("mfree"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_PRINTF_MAXLEN"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("cast_to_str"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("current"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("txt"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_FILE_OPEN_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), - (PreprocessorMacro { txt("GEN_FILE_READ_AT_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), - (PreprocessorMacro { txt("GEN_FILE_WRITE_AT_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), - (PreprocessorMacro { txt("GEN_FILE_SEEK_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), - (PreprocessorMacro { txt("GEN_FILE_CLOSE_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), - (PreprocessorMacro { txt("log_failure"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("operator"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("InvalidCode"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("NullCode"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("Verify_POD"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("gen_main"), MT_Statement, MF_Null }) + register_macros( args( + (Macro { txt("bit"), MT_Expression, MF_Functional }), + (Macro { txt("bitfield_is_set"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_C_LIKE_CPP"), MT_Expression, MF_Null }), + (Macro { txt("cast"), MT_Expression, MF_Functional }), + (Macro { txt("ccast"), MT_Expression, MF_Functional }), + (Macro { txt("rcast"), MT_Expression, MF_Functional }), + (Macro { txt("pcast"), MT_Expression, MF_Functional }), + (Macro { txt("scast"), MT_Expression, MF_Functional }), + (Macro { txt("stringize_va"), MT_Expression, MF_Functional }), + (Macro { txt("stringize"), MT_Expression, MF_Functional }), + (Macro { txt("do_once"), MT_Expression, MF_Functional }), + (Macro { txt("do_once_defer"), MT_Expression, MF_Functional }), + (Macro { txt("do_once_start"), MT_Statement, MF_Null }), + (Macro { txt("do_once_end"), MT_Statement, MF_Null }), + (Macro { txt("labeled_scope_start"), MT_Statement, MF_Null }), + (Macro { txt("labeled_scope_end"), MT_Statement, MF_Null }), + (Macro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null }), + (Macro { txt("num_args_impl"), MT_Expression, MF_Functional }), + (Macro { txt("num_args"), MT_Expression, MF_Functional }), + (Macro { txt("count_of"), MT_Expression, MF_Functional }), + (Macro { txt("clamp"), MT_Expression, MF_Functional }), + (Macro { txt("is_between"), MT_Expression, MF_Functional }), + (Macro { txt("size_of"), MT_Expression, MF_Functional }), + (Macro { txt("min"), MT_Expression, MF_Functional }), + (Macro { txt("max"), MT_Expression, MF_Functional }), + (Macro { txt("offset_of"), MT_Expression, MF_Functional }), + (Macro { txt("static_assert"), MT_Statement, MF_Functional }), + (Macro { txt("typeof"), MT_Expression, MF_Null }), + (Macro { txt("GEN_API_C_BEGIN"), MT_Statement, MF_Null }), + (Macro { txt("GEN_API_C_END"), MT_Statement, MF_Null }), + (Macro { txt("nullptr"), MT_Expression, MF_Null }), + (Macro { txt("GEN_REMOVE_PTR"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_PARAM_DEFAULT"), MT_Expression, MF_Null }), + (Macro { txt("struct_init"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"), MT_Statement, MF_Null }), + (Macro { txt("GEN_OPITMIZE_MAPPINGS_END"), MT_Statement, MF_Null }), + (Macro { txt("Array"), MT_Typename, MF_Functional }), + (Macro { txt("HashTable"), MT_Typename, MF_Functional }), + (Macro { txt("Using_Code"), MT_Statement, MF_Functional }), + (Macro { txt("Using_CodeOps"), MT_Statement, MF_Functional }), + (Macro { txt("kilobytes"), MT_Expression, MF_Functional }), + (Macro { txt("megabytes"), MT_Expression, MF_Functional }), + (Macro { txt("gigabytes"), MT_Expression, MF_Functional }), + (Macro { txt("terabytes"), MT_Expression, MF_Functional }), + (Macro { txt("GEN__ONES"), MT_Expression, MF_Null }), + (Macro { txt("GEN__HIGHS"), MT_Expression, MF_Null }), + (Macro { txt("GEN__HAS_ZERO"), MT_Expression, MF_Functional }), + (Macro { txt("zero_item"), MT_Expression, MF_Functional }), + (Macro { txt("zero_array"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_DEFAULT_MEMORY_ALIGNMENT"), MT_Expression, MF_Null }), + (Macro { txt("GEN_DEFAULT_ALLOCATOR_FLAGS"), MT_Expression, MF_Null }), + (Macro { txt("alloc_item"), MT_Expression, MF_Functional }), + (Macro { txt("alloc_array"), MT_Expression, MF_Functional }), + (Macro { txt("malloc"), MT_Expression, MF_Functional }), + (Macro { txt("mfree"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_PRINTF_MAXLEN"), MT_Expression, MF_Null }), + (Macro { txt("cast_to_str"), MT_Expression, MF_Functional }), + (Macro { txt("current"), MT_Expression, MF_Null }), + (Macro { txt("txt"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_FILE_OPEN_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (Macro { txt("GEN_FILE_READ_AT_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (Macro { txt("GEN_FILE_WRITE_AT_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (Macro { txt("GEN_FILE_SEEK_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (Macro { txt("GEN_FILE_CLOSE_PROC"), MT_Statement, MF_Functional | MF_Expects_Body }), + (Macro { txt("log_failure"), MT_Expression, MF_Null }), + (Macro { txt("operator"), MT_Expression, MF_Null }), + (Macro { txt("InvalidCode"), MT_Expression, MF_Null }), + (Macro { txt("NullCode"), MT_Expression, MF_Null }), + (Macro { txt("Verify_POD"), MT_Expression, MF_Functional }), + (Macro { txt("gen_main"), MT_Statement, MF_Null }) )); - register_preprocess_macros( args( - (PreprocessorMacro { txt("name"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("code"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("args"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("code_str"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("code_fmt"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("parse_fmt"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("token_fmt"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("check_member_val"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("check_member_str"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("check_member_content"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("check_member_ast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("check_params"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("check_param_eq_ret"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("specs"), MT_Expression, MF_Functional | MF_Allow_As_Identifier }), - (PreprocessorMacro { txt("name_check"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("null_check"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("def_body_start"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("def_body_code_array_start"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("move_forward"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("skip_whitespace"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("end_line"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("check_parse_args"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("currtok_noskip"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("currtok"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("peektok"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("prevtok"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("nexttok"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("nexttok_noskip"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("eat"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("left"), MT_Expression, MF_Null | MF_Allow_As_Identifier }), - (PreprocessorMacro { txt("def_assign"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("CHECK_WAS_DEFINED"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("check_noskip"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("check"), MT_Expression, MF_Functional | MF_Allow_As_Identifier }), - (PreprocessorMacro { txt("push_scope"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("cut_length"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("cut_ptr"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("pos"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("move_fwd"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("Entry"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("CheckEndParams"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("UseTemplateCapture"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("check_current"), MT_Expression, MF_Functional }) + register_macros( args( + (Macro { txt("name"), MT_Expression, MF_Functional }), + (Macro { txt("code"), MT_Expression, MF_Functional }), + (Macro { txt("args"), MT_Expression, MF_Functional }), + (Macro { txt("code_str"), MT_Expression, MF_Functional }), + (Macro { txt("code_fmt"), MT_Expression, MF_Functional }), + (Macro { txt("parse_fmt"), MT_Expression, MF_Functional }), + (Macro { txt("token_fmt"), MT_Expression, MF_Functional }), + (Macro { txt("check_member_val"), MT_Expression, MF_Functional }), + (Macro { txt("check_member_str"), MT_Expression, MF_Functional }), + (Macro { txt("check_member_content"), MT_Expression, MF_Functional }), + (Macro { txt("check_member_ast"), MT_Expression, MF_Functional }), + (Macro { txt("check_params"), MT_Expression, MF_Functional }), + (Macro { txt("check_param_eq_ret"), MT_Expression, MF_Functional }), + (Macro { txt("specs"), MT_Expression, MF_Functional | MF_Allow_As_Identifier }), + (Macro { txt("name_check"), MT_Expression, MF_Functional }), + (Macro { txt("null_check"), MT_Expression, MF_Functional }), + (Macro { txt("def_body_start"), MT_Expression, MF_Functional }), + (Macro { txt("def_body_code_array_start"), MT_Expression, MF_Functional }), + (Macro { txt("move_forward"), MT_Expression, MF_Functional }), + (Macro { txt("skip_whitespace"), MT_Expression, MF_Functional }), + (Macro { txt("end_line"), MT_Expression, MF_Functional }), + (Macro { txt("check_parse_args"), MT_Expression, MF_Functional }), + (Macro { txt("currtok_noskip"), MT_Expression, MF_Null }), + (Macro { txt("currtok"), MT_Expression, MF_Null }), + (Macro { txt("peektok"), MT_Expression, MF_Null }), + (Macro { txt("prevtok"), MT_Expression, MF_Null }), + (Macro { txt("nexttok"), MT_Expression, MF_Null }), + (Macro { txt("nexttok_noskip"), MT_Expression, MF_Null }), + (Macro { txt("eat"), MT_Expression, MF_Functional }), + (Macro { txt("left"), MT_Expression, MF_Null | MF_Allow_As_Identifier }), + (Macro { txt("def_assign"), MT_Expression, MF_Functional }), + (Macro { txt("CHECK_WAS_DEFINED"), MT_Expression, MF_Null }), + (Macro { txt("check_noskip"), MT_Expression, MF_Functional }), + (Macro { txt("check"), MT_Expression, MF_Functional | MF_Allow_As_Identifier }), + (Macro { txt("push_scope"), MT_Expression, MF_Functional }), + (Macro { txt("cut_length"), MT_Expression, MF_Null }), + (Macro { txt("cut_ptr"), MT_Expression, MF_Null }), + (Macro { txt("pos"), MT_Expression, MF_Null }), + (Macro { txt("move_fwd"), MT_Expression, MF_Functional }), + (Macro { txt("Entry"), MT_Expression, MF_Functional }), + (Macro { txt("CheckEndParams"), MT_Expression, MF_Functional }), + (Macro { txt("UseTemplateCapture"), MT_Expression, MF_Null }), + (Macro { txt("check_current"), MT_Expression, MF_Functional }) )); Code push_ignores = scan_file( path_base "helpers/push_ignores.inline.hpp" ); @@ -243,7 +243,7 @@ int gen_main() CodeTemplate tmpl = cast(CodeTemplate, entry); if ( tmpl->Declaration->Name.contains(txt("swap"))) { - register_preprocess_macro({ txt("swap"), MT_Expression, MF_Functional }); + register_macro({ txt("swap"), MT_Expression, MF_Functional }); CodeDefine macro_swap = parse_define( txt(R"( #define swap( a, b ) \ do \ @@ -1113,8 +1113,9 @@ R"(#define ( code ) _Generic( (code), \ break; } - CodeBody array_arena = gen_array(txt("gen_Arena"), txt("Array_gen_Arena")); - CodeBody array_pool = gen_array(txt("gen_Pool"), txt("Array_gen_Pool")); + CodeBody array_arena = gen_array(txt("gen_Arena"), txt("Array_gen_Arena")); + CodeBody array_pool = gen_array(txt("gen_Pool"), txt("Array_gen_Pool")); + CodeBody ht_preprocessor_macro = gen_hashtable(txt("gen_Macro"), txt("MacroTable")); CodeBody parsed_interface = parse_file( path_base "components/interface.hpp" ); CodeBody interface = def_body(CT_Global_Body); @@ -1354,11 +1355,37 @@ R"(#define ( code ) _Generic( (code), \ Code src_static_data = scan_file( path_base "components/static_data.cpp" ); Code src_ast_case_macros = scan_file( path_base "components/ast_case_macros.cpp" ); Code src_code_serialization = scan_file( path_base "components/code_serialization.cpp" ); - Code src_interface = scan_file( path_base "components/interface.cpp" ); + Code src_parsing_interface = scan_file( path_base "components/interface.parsing.cpp" ); Code src_untyped = scan_file( path_base "components/interface.untyped.cpp" ); Code src_parser_case_macros = scan_file( path_base "components/parser_case_macros.cpp" ); + CodeBody parsed_src_interface = parse_file( path_base "components/interface.cpp" ); + CodeBody src_interface = def_body(CT_Global_Body); + for ( Code entry = parsed_src_interface.begin(); entry != parsed_src_interface.end(); ++ entry ) switch( entry ->Type ) + { + case CT_Function: + { + CodeFn fn = cast(CodeFn, entry); + Code prev = entry->Prev; + for ( CodeParams arr_param : fn->Params ) + { + b32 repeat_register_macros = fn->Name.is_equal(txt("register_macros")) && arr_param->Name.is_equal(txt("num")) && ! arr_param->Next->Name.is_equal(txt("...")); + if ( repeat_register_macros ) { + // rename second definition so there isn't a symbol conflict + StrBuilder postfix_arr = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "%S_arr", fn->Name); + fn->Name = cache_str(postfix_arr.to_str()); + } + } + src_interface.append(fn); + } + break; + + default: + src_interface.append(entry); + break; + } + CodeBody parsed_src_ast = parse_file( path_base "components/ast.cpp" ); CodeBody src_ast = def_body(CT_Global_Body); for ( Code entry = parsed_src_ast.begin(); entry != parsed_src_ast.end(); ++ entry ) switch( entry ->Type ) @@ -1418,26 +1445,22 @@ R"(#define ( code ) _Generic( (code), \ Code prev = entry->Prev; for ( CodeParams arr_param : fn->Params ) - if ( fn->Name.starts_with(txt("def_")) - && ( (arr_param->ValueType->Name.starts_with(txt("Specifier")) && fn->Params->NumEntries > 1) - || arr_param->ValueType->Name.starts_with(txt("Code")) ) - ) - { + { + b32 repeat_def_array = fn->Name.starts_with(txt("def_")) && arr_param->Name.is_equal(txt("num")) && ! arr_param->Next->Name.is_equal(txt("...")); + if ( repeat_def_array ) { // rename second definition so there isn't a symbol conflict StrBuilder postfix_arr = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "%S_arr", fn->Name); fn->Name = cache_str(postfix_arr.to_str()); - postfix_arr.free(); } - + } for ( CodeParams opt_param : fn->Params ) if (opt_param->ValueType->Name.starts_with(txt("Opts_"))) { + // The frontend names are warapped in macros so we need to give it the intenral symbol name Str prefix = txt("def_"); Str actual_name = { fn->Name.Ptr + prefix.Len, fn->Name.Len - prefix.Len }; Str new_name = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "def__%S", actual_name ).to_str(); - fn->Name = cache_str(new_name); } - src_upfront.append(fn); } break; @@ -1588,10 +1611,11 @@ R"(#define ( code ) _Generic( (code), \ Code rf_interface = refactor_and_format(interface); Code rf_inlines = refactor_and_format(inlines); - Code rf_array_string_cached = refactor_and_format(array_string_cached); - Code rf_header_end = refactor_and_format(header_end); - Code rf_header_builder = refactor_and_format(header_builder); - Code rf_header_scanner = refactor_and_format( scan_file( path_base "auxillary/scanner.hpp" )); + Code rf_ht_preprocessor_macro = refactor_and_format(ht_preprocessor_macro); + Code rf_array_string_cached = refactor_and_format(array_string_cached); + Code rf_header_end = refactor_and_format(header_end); + Code rf_header_builder = refactor_and_format(header_builder); + Code rf_header_scanner = refactor_and_format( scan_file( path_base "auxillary/scanner.hpp" )); Code r_src_dep_start = refactor(src_dep_start); Code r_src_debug = refactor(src_debug); @@ -1613,7 +1637,7 @@ R"(#define ( code ) _Generic( (code), \ Code r_src_code_serialization = refactor(src_code_serialization); Code r_src_parser_case_macros = refactor(src_parser_case_macros); - Code r_src_interface = refactor(src_interface); + Code r_src_interface = refactor_and_format(src_interface); Code r_src_upfront = refactor_and_format(src_upfront); Code r_src_lexer = refactor_and_format(src_lexer); Code rf_array_code_typename = refactor_and_format(array_code_typename); @@ -1696,6 +1720,8 @@ R"(#define ( code ) _Generic( (code), \ header.print( rf_array_pool); header.print( fmt_newline); header.print( rf_array_string_cached ); + header.print( fmt_newline); + header.print( rf_ht_preprocessor_macro ); header.print( rf_interface ); header.print(fmt_newline); diff --git a/gen_c_library/c_library.refactor b/gen_c_library/c_library.refactor index 569be07..1dc2e85 100644 --- a/gen_c_library/c_library.refactor +++ b/gen_c_library/c_library.refactor @@ -310,6 +310,14 @@ word spec_to_str, gen_spec_to_str word spec_is_trailing, gen_spec_is_trailing // word str_to_specifier, gen_str_to_specifier +word MacroType, gen_MacroType +word EMacroFlags, gen_EMacroFlags +word MacroFlags, gen_MacroFlags +word Macro, gen_Macro + +namespace macro_, gen_macro_ +namespace macrotype, gen_macrotype_ + // AST word AST, gen_AST diff --git a/gen_c_library/components/containers.hashtable.hpp b/gen_c_library/components/containers.hashtable.hpp index 000a995..8303eb7 100644 --- a/gen_c_library/components/containers.hashtable.hpp +++ b/gen_c_library/components/containers.hashtable.hpp @@ -19,7 +19,7 @@ CodeBody gen_hashtable_base() )); Code define_type = untyped_str(txt( -R"(#define HashTable(_type) struct _type +R"(#define HashTable(_type) struct gen_HashTable_##_type )" )); diff --git a/gen_c_library/components/memory.fixed_arena.hpp b/gen_c_library/components/memory.fixed_arena.hpp index c9fd7ab..f61b720 100644 --- a/gen_c_library/components/memory.fixed_arena.hpp +++ b/gen_c_library/components/memory.fixed_arena.hpp @@ -89,11 +89,11 @@ CodeBody gen_fixed_arenas() result.append(arena_interface_2mb); result.append(arena_interface_4mb); - register_preprocess_macros( args( - ( PreprocessorMacro { txt("fixed_arena_allocator_info"), MT_Expression, MF_Functional }), - ( PreprocessorMacro { txt("fixed_arena_init"), MT_Expression, MF_Functional }), - ( PreprocessorMacro { txt("fixed_arena_free"), MT_Expression, MF_Functional }), - ( PreprocessorMacro { txt("fixed_arena_size_remaining"), MT_Expression, MF_Functional }) + register_macros( args( + ( Macro { txt("fixed_arena_allocator_info"), MT_Expression, MF_Functional }), + ( Macro { txt("fixed_arena_init"), MT_Expression, MF_Functional }), + ( Macro { txt("fixed_arena_free"), MT_Expression, MF_Functional }), + ( Macro { txt("fixed_arena_size_remaining"), MT_Expression, MF_Functional }) )); CodeDefine def = parse_define(txt("#define fixed_arena_allocator_info(fixed_arena) ( (AllocatorInfo) { arena_allocator_proc, & (fixed_arena)->arena } )\n")); diff --git a/gen_unreal_engine/unreal.cpp b/gen_unreal_engine/unreal.cpp index 47d0314..5719a7c 100644 --- a/gen_unreal_engine/unreal.cpp +++ b/gen_unreal_engine/unreal.cpp @@ -64,43 +64,43 @@ int gen_main() Code ue_forceinline = code_str(FORCEINLINE); // Code - register_preprocess_macros( args( - (PreprocessorMacro { txt("bit"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("bitfield_is_set"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_C_LIKE_CPP"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("cast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("ccast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("rcast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("pcast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("scast"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("stringize_va"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("stringize"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("do_once"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("do_once_defer"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("do_once_start"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("do_once_end"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("labeled_scope_start"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("labeled_scope_end"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("num_args_impl"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("num_args"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("count_of"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("clamp"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("is_between"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("size_of"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("min"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("max"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("offset_of"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("static_assert"), MT_Statement, MF_Functional }), - (PreprocessorMacro { txt("typeof"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("GEN_API_C_BEGIN"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("GEN_API_C_END"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("nullptr"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("GEN_REMOVE_PTR"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_PARAM_DEFAULT"), MT_Expression, MF_Null }), - (PreprocessorMacro { txt("struct_init"), MT_Expression, MF_Functional }), - (PreprocessorMacro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"), MT_Statement, MF_Null }), - (PreprocessorMacro { txt("GEN_OPITMIZE_MAPPINGS_END"), MT_Statement, MF_Null }) + register_macros( args( + (Macro { txt("bit"), MT_Expression, MF_Functional }), + (Macro { txt("bitfield_is_set"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_C_LIKE_CPP"), MT_Expression, MF_Null }), + (Macro { txt("cast"), MT_Expression, MF_Functional }), + (Macro { txt("ccast"), MT_Expression, MF_Functional }), + (Macro { txt("rcast"), MT_Expression, MF_Functional }), + (Macro { txt("pcast"), MT_Expression, MF_Functional }), + (Macro { txt("scast"), MT_Expression, MF_Functional }), + (Macro { txt("stringize_va"), MT_Expression, MF_Functional }), + (Macro { txt("stringize"), MT_Expression, MF_Functional }), + (Macro { txt("do_once"), MT_Expression, MF_Functional }), + (Macro { txt("do_once_defer"), MT_Expression, MF_Functional }), + (Macro { txt("do_once_start"), MT_Statement, MF_Null }), + (Macro { txt("do_once_end"), MT_Statement, MF_Null }), + (Macro { txt("labeled_scope_start"), MT_Statement, MF_Null }), + (Macro { txt("labeled_scope_end"), MT_Statement, MF_Null }), + (Macro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null }), + (Macro { txt("num_args_impl"), MT_Expression, MF_Functional }), + (Macro { txt("num_args"), MT_Expression, MF_Functional }), + (Macro { txt("count_of"), MT_Expression, MF_Functional }), + (Macro { txt("clamp"), MT_Expression, MF_Functional }), + (Macro { txt("is_between"), MT_Expression, MF_Functional }), + (Macro { txt("size_of"), MT_Expression, MF_Functional }), + (Macro { txt("min"), MT_Expression, MF_Functional }), + (Macro { txt("max"), MT_Expression, MF_Functional }), + (Macro { txt("offset_of"), MT_Expression, MF_Functional }), + (Macro { txt("static_assert"), MT_Statement, MF_Functional }), + (Macro { txt("typeof"), MT_Expression, MF_Null }), + (Macro { txt("GEN_API_C_BEGIN"), MT_Statement, MF_Null }), + (Macro { txt("GEN_API_C_END"), MT_Statement, MF_Null }), + (Macro { txt("nullptr"), MT_Expression, MF_Null }), + (Macro { txt("GEN_REMOVE_PTR"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_PARAM_DEFAULT"), MT_Expression, MF_Null }), + (Macro { txt("struct_init"), MT_Expression, MF_Functional }), + (Macro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"), MT_Statement, MF_Null }), + (Macro { txt("GEN_OPITMIZE_MAPPINGS_END"), MT_Statement, MF_Null }) )); // gen_dep.hpp From 70872c29d14c86f96e68563d776708845489b0a6 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 10:34:47 -0500 Subject: [PATCH 09/15] correction to Specifier codegen --- base/components/gen/especifier.hpp | 2 +- base/helpers/base_codegen.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/components/gen/especifier.hpp b/base/components/gen/especifier.hpp index f75f94d..bfbcdd1 100644 --- a/base/components/gen/especifier.hpp +++ b/base/components/gen/especifier.hpp @@ -39,7 +39,7 @@ enum Specifier : u32 inline Str spec_to_str( Specifier type ) { - local_persist Str lookup[26] = { + local_persist Str lookup[] = { { "INVALID", sizeof( "INVALID" ) - 1 }, { "consteval", sizeof( "consteval" ) - 1 }, { "constexpr", sizeof( "constexpr" ) - 1 }, diff --git a/base/helpers/base_codegen.hpp b/base/helpers/base_codegen.hpp index 1a405f3..47293fc 100644 --- a/base/helpers/base_codegen.hpp +++ b/base/helpers/base_codegen.hpp @@ -237,7 +237,7 @@ CodeBody gen_especifier( char const* path, bool use_c_definition = false ) Str spec_to_str( Specifier type ) { local_persist - Str lookup[] = { + Str lookup[] = { }; From 63bc3bebed43b4bbdc38512fae0844799bd3622c Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 11:23:04 -0500 Subject: [PATCH 10/15] add GEN_NS to txt macro --- base/dependencies/strings.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/dependencies/strings.hpp b/base/dependencies/strings.hpp index 788a553..3c60b68 100644 --- a/base/dependencies/strings.hpp +++ b/base/dependencies/strings.hpp @@ -40,9 +40,9 @@ struct Str #ifndef txt # if GEN_COMPILER_CPP -# define txt( text ) Str { ( text ), sizeof( text ) - 1 } +# define txt( text ) GEN_NS Str { ( text ), sizeof( text ) - 1 } # else -# define txt( text ) (Str){ ( text ), sizeof( text ) - 1 } +# define txt( text ) (GEN_NS Str){ ( text ), sizeof( text ) - 1 } # endif #endif From 572e957c17b66a1d17fea8cbb2de3cdc87e86e45 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 11:23:23 -0500 Subject: [PATCH 11/15] remove UE_DEPRECATED from unrela's attribute tokens, it must be handled as a statement macro --- gen_unreal_engine/enums/AttributeTokens.csv | 1 - gen_unreal_engine/enums/ETokType.csv | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/gen_unreal_engine/enums/AttributeTokens.csv b/gen_unreal_engine/enums/AttributeTokens.csv index 49f51a4..4ac524b 100644 --- a/gen_unreal_engine/enums/AttributeTokens.csv +++ b/gen_unreal_engine/enums/AttributeTokens.csv @@ -4,4 +4,3 @@ COREUOBJECT_API, COREUOBJECT_API ENGINE_API, ENGINE_API GAMEPLAYABILITIES_API, GAMEPLAYABILITIES_API UMG_API, UMG_API -UE_DEPRECATED, UE_DEPRECATED diff --git a/gen_unreal_engine/enums/ETokType.csv b/gen_unreal_engine/enums/ETokType.csv index 739deee..cd8b917 100644 --- a/gen_unreal_engine/enums/ETokType.csv +++ b/gen_unreal_engine/enums/ETokType.csv @@ -42,6 +42,7 @@ Number, "__number__" Operator, "__operator__" Preprocess_Hash, "#" Preprocess_Define, "define" +Preprocess_Define_Param, "__define_param__" Preprocess_If, "if" Preprocess_IfDef, "ifdef" Preprocess_IfNotDef, "ifndef" From 0b03b3cd925588a816fff693292c37a5c5e61ed0 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 13:39:00 -0500 Subject: [PATCH 12/15] Unreal parsing support: Added MF_Allow_As_Attribute & MF_Allow_As_Definition --- base/components/ast_types.hpp | 3 +- base/components/interface.cpp | 3 ++ base/components/lexer.cpp | 43 ++++++----------------- base/components/parser.cpp | 58 ++++++++++++++++++++++++++++++-- base/components/parser_types.hpp | 18 ++++++++-- 5 files changed, 88 insertions(+), 37 deletions(-) diff --git a/base/components/ast_types.hpp b/base/components/ast_types.hpp index 43a0b80..5efd39a 100644 --- a/base/components/ast_types.hpp +++ b/base/components/ast_types.hpp @@ -42,13 +42,14 @@ struct AST_Body }; static_assert( sizeof(AST_Body) == sizeof(AST), "ERROR: AST_Body is not the same size as AST"); +// TODO(Ed): Support chaining attributes (Use parameter linkage pattern) struct AST_Attributes { union { char _PAD_[ sizeof(Specifier) * AST_ArrSpecs_Cap + sizeof(AST*) ]; StrCached Content; }; - StrCached Name; + StrCached Name; Code Prev; Code Next; Token* Tok; diff --git a/base/components/interface.cpp b/base/components/interface.cpp index 74ac369..83e44e6 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -473,6 +473,7 @@ void register_macro( Macro macro ) { GEN_ASSERT_NOT_NULL(macro.Name.Ptr); GEN_ASSERT(macro.Name.Len > 0); u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); + macro.Name = cache_str(macro.Name); hashtable_set( _ctx->Macros, key, macro ); } @@ -486,6 +487,7 @@ void register_macros( s32 num, ... ) Macro macro = va_arg(va, Macro); GEN_ASSERT_NOT_NULL(macro.Name.Ptr); GEN_ASSERT(macro.Name.Len > 0); + macro.Name = cache_str(macro.Name); u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); hashtable_set( _ctx->Macros, key, macro ); @@ -502,6 +504,7 @@ void register_macros( s32 num, Macro* macros ) Macro macro = * macros; GEN_ASSERT_NOT_NULL(macro.Name.Ptr); GEN_ASSERT(macro.Name.Len > 0); + macro.Name = cache_str(macro.Name); u32 key = crc32( macro.Name.Ptr, macro.Name.Len ); hashtable_set( _ctx->Macros, key, macro ); diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 289983b..1d7f4ee 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -532,45 +532,24 @@ void lex_found_token( LexContext* ctx ) ctx->token.Type = macrotype_to_toktype(macro->Type); b32 is_functional = macro_is_functional(* macro); resolved_to_macro = has_args ? is_functional : ! is_functional; + if ( ! resolved_to_macro ) { + log_fmt("Info(%d, %d): %S identified as a macro but usage here does not resolve to one (interpreting as identifier)\n" + , ctx->token.Line + , ctx->token.Line + , macro->Name + ); + } } if ( resolved_to_macro ) { // TODO(Ed): When we introduce a macro AST (and expression support), we'll properly lex this section. // Want to ignore any arguments the define may have as they can be execution expressions. - if ( has_args ) - { + if ( has_args ) { ctx->token.Flags |= TF_Macro_Functional; - - // move_forward(); - // ctx->token.Text.Len++; - - // s32 level = 0; - // while ( ctx->left && ((* ctx->scanner) != ')' || level > 0) ) - // { - // if ( (* ctx->scanner) == '(' ) - // level++; - - // else if ( (* ctx->scanner) == ')' && level > 0 ) - // level--; - - // move_forward(); - // ctx->token.Text.Len++; - // } - - // move_forward(); - // ctx->token.Text.Len++; } - - //if ( (* ctx->scanner) == '\r' && ctx->scanner[1] == '\n' ) - //{ - // move_forward(); - // ctx->token..Text.Length++; - //} - //else if ( (* ctx->scanner) == '\n' ) - //{ - // move_forward(); - // ctx->token..Text.Length++; - //} + if ( bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Attribute) ) { + ctx->token.Flags |= TF_Attribute; + } } else { diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 8cbd4d8..be520ba 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -204,6 +204,7 @@ internal CodeBody parse_global_nspace ( CodeType which internal Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ); internal Token parse_identifier ( bool* possible_member_function ); internal CodeInclude parse_include (); +internal Code parse_macro_as_definiton ( CodeAttributes attributes, CodeSpecifiers specifiers ); internal CodeOperator parse_operator_after_ret_type ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type ); internal Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ); internal CodePragma parse_pragma (); @@ -578,7 +579,7 @@ CodeAttributes parse_attributes() s32 len = 0; // There can be more than one attribute. If there is flatten them to a single string. - // TODO(Ed): Support keeping an linked list of attributes similar to parameters + // TODO(Ed): Support chaining attributes (Use parameter linkage pattern) while ( left && tok_is_attribute(currtok) ) { if ( check( Tok_Attribute_Open ) ) @@ -2434,6 +2435,12 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes Code result = InvalidCode; + Code macro_stmt = parse_macro_as_definiton(attributes, specifiers); + if (macro_stmt) { + parser_pop(& _ctx->parser); + return macro_stmt; + } + CodeTypename type = parser_parse_type( parser_not_from_template, nullptr ); // @@ -2506,6 +2513,36 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes return result; } +internal +Code parse_macro_as_definiton( CodeAttributes attributes, CodeSpecifiers specifiers ) +{ + push_scope(); + + if (currtok.Type != Tok_Preprocess_Macro_Stmt ) { + parser_pop(& _ctx->parser); + return NullCode; + } + Macro* macro = lookup_macro(currtok.Text); + b32 can_resolve_to_definition = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Definition); + if ( ! can_resolve_to_definition) { + parser_pop(& _ctx->parser); + return NullCode; + } + + // TODO(Ed): When AST_Macro is made, have it support attributs and specifiers for when its behaving as a declaration/definition. + Code code = parse_simple_preprocess( Tok_Preprocess_Macro_Stmt ); + + // Attributes and sepcifiers will be collapsed into the macro's serialization. + StrBuilder resolved_definition = strbuilder_fmt_buf(_ctx->Allocator_Temp, "%S %S %S" + , attributes ? strbuilder_to_str( attributes_to_strbuilder(attributes)) : txt("") + , specifiers ? strbuilder_to_str( specifiers_to_strbuilder(specifiers)) : txt("") + , code->Content + ); + Code result = untyped_str( strbuilder_to_str(resolved_definition) ); + parser_pop(& _ctx->parser); + return result; +} + internal CodePragma parse_pragma() { @@ -3761,8 +3798,11 @@ CodeEnum parser_parse_enum( bool inplace_def ) // Unreal UMETA macro support if ( currtok.Type == Tok_Preprocess_Macro_Expr ) { - eat( Tok_Preprocess_Macro_Expr ); + Code macro = parse_simple_preprocess( Tok_Preprocess_Macro_Expr ); // = + + // We're intentially ignoring this code as its going to be serialized as an untyped string with the rest of the enum "entry". + // TODO(Ed): We need a CodeEnumEntry, AST_EnumEntry types } if ( currtok.Type == Tok_Comma ) @@ -4060,6 +4100,13 @@ CodeFn parser_parse_function() } // + // Note(Ed): We're enforcing that using this codepath requires non-macro jank. + // Code macro_stmt = parse_macro_as_definiton(attributes, specifiers); + // if (macro_stmt) { + // parser_pop(& _ctx->parser); + // return macro_stmt; + // } + CodeTypename ret_type = parser_parse_type(parser_not_from_template, nullptr); if ( cast(Code, ret_type) == Code_Invalid ) { parser_pop(& _ctx->parser); @@ -5448,6 +5495,13 @@ CodeVar parser_parse_variable() } // + // Note(Ed): We're enforcing that using this codepath requires non-macro jank. + // Code macro_stmt = parse_macro_as_definiton(attributes, specifiers); + // if (macro_stmt) { + // parser_pop(& _ctx->parser); + // return macro_stmt; + // } + CodeTypename type = parser_parse_type(parser_not_from_template, nullptr); // diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index fd03708..9275b53 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -177,8 +177,22 @@ enum EMacroFlags : u16 { MF_Functional = bit(0), // Macro has parameters (args expected to be passed) MF_Expects_Body = bit(1), // Expects to assign a braced scope to its body. - MF_Allow_As_Identifier = bit(2), // lex__eat wil treat this macro as an identifier if the parser attempts to consume it as one. - // ^^^ This is a sort of kludge because we don't support push/pop macro programs rn. ^^^ + + // lex__eat wil treat this macro as an identifier if the parser attempts to consume it as one. + // ^^^ This is a kludge because we don't support push/pop macro pragmas rn. + MF_Allow_As_Identifier = bit(2), + + // lex__eat wil treat this macro as an attribute if the parser attempts to consume it as one. + // ^^^ This a kludge because unreal has a macro that behaves as both a 'statement' and an attribute (UE_DEPRECATED, PRAGMA_ENABLE_DEPRECATION_WARNINGS, etc) + // TODO(Ed): We can keep the MF_Allow_As_Attribute flag for macros, however, we need to add the ability of AST_Attributes to chain themselves. + // Its thats already a thing in the standard language anyway + // & it would allow UE_DEPRECATED, (UE_PROPERTY / UE_FUNCTION) to chain themselves as attributes of a resolved member function/varaible definition + MF_Allow_As_Attribute = bit(3), + + // When a macro is encountered after attributs and specifiers while parsing a function, or variable: + // It will consume the macro and treat it as resolving the definition. (Yes this is for Unreal Engine) + // (MUST BE OF MT_Statement TYPE) + MF_Allow_As_Definition = bit(4), MF_Null = 0, MF_UnderlyingType = GEN_U16_MAX, From 868b93cdd0fd27bf340ff89584efc59e5696ad42 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 17:52:31 -0500 Subject: [PATCH 13/15] bugfixes while testing with unreal (still more needs fixing --- base/auxillary/builder.cpp | 4 +++- base/auxillary/builder.hpp | 13 ++++++----- base/auxillary/scanner.cpp | 4 ++-- base/components/inlines.hpp | 6 ++--- base/components/interface.cpp | 4 ++++ base/components/interface.hpp | 3 +++ base/components/lexer.cpp | 2 +- base/components/parser.cpp | 40 ++++++++++++++++++++++++++++---- base/components/parser_types.hpp | 8 ++++++- base/dependencies/containers.hpp | 6 ++--- base/dependencies/debug.hpp | 2 +- base/dependencies/memory.cpp | 34 +++++++++++++-------------- base/dependencies/memory.hpp | 7 +++--- base/dependencies/strings.hpp | 1 + gen_unreal_engine/unreal.cpp | 18 +++++++++----- 15 files changed, 103 insertions(+), 49 deletions(-) diff --git a/base/auxillary/builder.cpp b/base/auxillary/builder.cpp index 606ab33..02adda8 100644 --- a/base/auxillary/builder.cpp +++ b/base/auxillary/builder.cpp @@ -15,7 +15,9 @@ Builder builder_open( char const* path ) return result; } - result.Buffer = strbuilder_make_reserve( _ctx->Allocator_Temp, _ctx->InitSize_BuilderBuffer ); + Context* ctx = get_context(); + GEN_ASSERT_NOT_NULL(ctx); + result.Buffer = strbuilder_make_reserve( ctx->Allocator_Temp, ctx->InitSize_BuilderBuffer ); // log_fmt("$Builder - Opened file: %s\n", result.File.filename ); return result; diff --git a/base/auxillary/builder.hpp b/base/auxillary/builder.hpp index 1de05c3..1729fe0 100644 --- a/base/auxillary/builder.hpp +++ b/base/auxillary/builder.hpp @@ -25,13 +25,14 @@ Builder builder_open ( char const* path ); void builder_pad_lines ( Builder* builder, s32 num ); void builder_print ( Builder* builder, Code code ); void builder_print_fmt_va( Builder* builder, char const* fmt, va_list va ); -void builder_print_fmt ( Builder* builder, char const* fmt, ... ) { +void builder_write ( Builder* builder ); + +forceinline void builder_print_fmt ( Builder* builder, char const* fmt, ... ) { va_list va; va_start( va, fmt ); builder_print_fmt_va( builder, fmt, va ); va_end( va ); } -void builder_write( Builder* builder ); struct Builder { @@ -56,10 +57,10 @@ struct Builder }; #if GEN_COMPILER_CPP && ! GEN_C_LIKE_CPP -void builder_pad_lines( Builder& builder, s32 num ) { return builder_pad_lines(& builder, num); } -void builder_print ( Builder& builder, Code code ) { return builder_print(& builder, code); } -void builder_write ( Builder& builder ) { return builder_write(& builder ); } -void builder_print_fmt( Builder& builder, char const* fmt, ...) { +forceinline void builder_pad_lines( Builder& builder, s32 num ) { return builder_pad_lines(& builder, num); } +forceinline void builder_print ( Builder& builder, Code code ) { return builder_print(& builder, code); } +forceinline void builder_write ( Builder& builder ) { return builder_write(& builder ); } +forceinline void builder_print_fmt( Builder& builder, char const* fmt, ...) { va_list va; va_start( va, fmt ); builder_print_fmt_va( & builder, fmt, va ); diff --git a/base/auxillary/scanner.cpp b/base/auxillary/scanner.cpp index 94f5e48..6cde239 100644 --- a/base/auxillary/scanner.cpp +++ b/base/auxillary/scanner.cpp @@ -20,7 +20,7 @@ Code scan_file( char const* path ) GEN_FATAL("scan_file: %s is empty", path ); } - StrBuilder str = strbuilder_make_reserve( _ctx->Allocator_Temp, fsize ); + StrBuilder str = strbuilder_make_reserve( get_context()->Allocator_Temp, fsize ); file_read( & file, str, fsize ); strbuilder_get_header(str)->Length = fsize; @@ -117,7 +117,7 @@ Code scan_file( char const* path ) } CodeBody parse_file( const char* path ) { - FileContents file = file_read_contents( _ctx->Allocator_Temp, true, path ); + FileContents file = file_read_contents( get_context()->Allocator_Temp, true, path ); Str content = { (char const*)file.data, file.size }; CodeBody code = parse_global_body( content ); log_fmt("\nParsed: %s\n", path); diff --git a/base/components/inlines.hpp b/base/components/inlines.hpp index b557a4f..87e2710 100644 --- a/base/components/inlines.hpp +++ b/base/components/inlines.hpp @@ -262,9 +262,9 @@ forceinline void define_params_append (CodeDefineParams appendee forceinline CodeDefineParams define_params_get (CodeDefineParams self, s32 idx ) { return (CodeDefineParams) (Code) params_get( cast(CodeParams, self), idx); } forceinline bool define_params_has_entries(CodeDefineParams self) { return params_has_entries( cast(CodeParams, self)); } -CodeDefineParams begin_CodeDefineParams(CodeDefineParams params) { return (CodeDefineParams) (Code) begin_CodeParams( cast(CodeParams, (Code)params)); } -CodeDefineParams end_CodeDefineParams (CodeDefineParams params) { return (CodeDefineParams) (Code) end_CodeParams ( cast(CodeParams, (Code)params)); } -CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter) { return (CodeDefineParams) (Code) next_CodeParams ( cast(CodeParams, (Code)params), cast(CodeParams, (Code)entry_iter)); } +forceinline CodeDefineParams begin_CodeDefineParams(CodeDefineParams params) { return (CodeDefineParams) (Code) begin_CodeParams( cast(CodeParams, (Code)params)); } +forceinline CodeDefineParams end_CodeDefineParams (CodeDefineParams params) { return (CodeDefineParams) (Code) end_CodeParams ( cast(CodeParams, (Code)params)); } +forceinline CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter) { return (CodeDefineParams) (Code) next_CodeParams ( cast(CodeParams, (Code)params), cast(CodeParams, (Code)entry_iter)); } #if GEN_COMPILER_CPP forceinline diff --git a/base/components/interface.cpp b/base/components/interface.cpp index 83e44e6..ab2f580 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -378,6 +378,10 @@ void deinit(Context* ctx) -- context_counter; } +Context* get_context() { + return _ctx; +} + void reset(Context* ctx) { s32 index = 0; diff --git a/base/components/interface.hpp b/base/components/interface.hpp index cb70376..0ee2422 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -100,6 +100,9 @@ GEN_API void init(Context* ctx); // However on Windows at least, it doesn't need to occur as the OS will clean up after the process. GEN_API void deinit(Context* ctx); +// Retrieves the active context (not usually needed, but here in case...) +GEN_API Context* get_context(); + // Clears the allocations, but doesn't free the memoery, then calls init() again. // Ease of use. GEN_API void reset(Context* ctx); diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 1d7f4ee..07cf62f 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -155,7 +155,7 @@ s32 lex_preprocessor_define( LexContext* ctx ) array_append( _ctx->Lexer_Tokens, opening_paren ); move_forward(); - Token last_parameter; + Token last_parameter = {}; // We need to tokenize the define's arguments now: while( ctx->left && * ctx->scanner != ')') { diff --git a/base/components/parser.cpp b/base/components/parser.cpp index be520ba..47b5813 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -2490,7 +2490,40 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes // Example : // idx +1 +2 bool detected_comma = _ctx->parser.Tokens.Arr[ _ctx->parser.Tokens.Idx + 2 ].Type == Tok_Comma; - if ( detected_capture && ! detected_comma ) + + b32 detected_non_varadic_unpaired_param = detected_comma && nexttok.Type != Tok_Varadic_Argument; + if (! detected_non_varadic_unpaired_param && nexttok.Type == Tok_Preprocess_Macro_Expr) for( s32 break_scope = 0; break_scope == 0; ++ break_scope) + { + Macro* macro = lookup_macro( nexttok.Text ); + if (macro == nullptr || ! macro_is_functional(* macro)) + break; + + // ( ( + // Idx +1 +2 + s32 idx = _ctx->parser.Tokens.Idx + 1; + s32 level = 0; + + // Find end of the token expression + for ( ; idx < array_num(_ctx->parser.Tokens.Arr); idx++ ) + { + Token tok = _ctx->parser.Tokens.Arr[ idx ]; + + if ( tok.Type == Tok_Capture_Start ) + level++; + else if ( tok.Type == Tok_Capture_End && level > 0 ) + level--; + if (level == 0 && tok.Type == Tok_Capture_End) + break; + } + ++ idx; // Will incremnt to possible comma position + + if ( _ctx->parser.Tokens.Arr[ idx ].Type != Tok_Comma ) + break; + + detected_non_varadic_unpaired_param = true; + } + + if ( detected_capture && ! detected_non_varadic_unpaired_param ) { // Dealing with a function result = cast(Code, parse_function_after_name( ModuleFlag_None, attributes, specifiers, type, name )); @@ -2509,7 +2542,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes } } - parser_pop(& _ctx->parser); + parser_pop(& _ctx->parser); return result; } @@ -3002,7 +3035,7 @@ Code parse_simple_preprocess( TokType which ) || str_contains(calling_proc, txt("parse_class_struct_body")) ) { - if (peektok.Type == Tok_Statement_End) + if (left && peektok.Type == Tok_Statement_End) { Token stmt_end = currtok; eat( Tok_Statement_End ); @@ -5015,7 +5048,6 @@ CodeTypedef parser_parse_typedef() // valid_macro |= macro && macro_expects_body(* macro)); // } - Code macro; if ( valid_macro ) #endif { diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index 9275b53..f98291e 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -9,7 +9,7 @@ enum TokFlags : u32 { - TF_Operator = bit(0), + TF_Operator = bit(0), TF_Assign = bit(1), TF_Preprocess = bit(2), TF_Preprocess_Cond = bit(3), @@ -152,6 +152,7 @@ TokType macrotype_to_toktype( MacroType type ) { return Tok_Invalid; } +inline Str macrotype_to_str( MacroType type ) { local_persist @@ -216,4 +217,9 @@ b32 macro_expects_body( Macro macro ) { return bitfield_is_set( b16, macro.Flags, MF_Expects_Body ); } +#if GEN_COMPILER_CPP && ! GEN_C_LIKE_CPP +forceinline b32 is_functional( Macro macro ) { return bitfield_is_set( b16, macro.Flags, MF_Functional ); } +forceinline b32 expects_body ( Macro macro ) { return bitfield_is_set( b16, macro.Flags, MF_Expects_Body ); } +#endif + typedef HashTable(Macro) MacroTable; diff --git a/base/dependencies/containers.hpp b/base/dependencies/containers.hpp index 13386e6..a84d148 100644 --- a/base/dependencies/containers.hpp +++ b/base/dependencies/containers.hpp @@ -143,6 +143,7 @@ Array array_init_reserve(AllocatorInfo allocator, ssize capacity) return {rcast(Type*, header + 1)}; } +forceinline usize array_grow_formula(ssize value) { return 2 * value + 8; } @@ -202,7 +203,7 @@ bool array_append_at(Array* array, Type item, usize idx) ArrayHeader* header = array_get_header(* array); ssize slot = idx; - if (slot >= header->Num) + if (slot >= (ssize)(header->Num)) slot = header->Num - 1; if (slot < 0) @@ -354,7 +355,6 @@ bool array_reserve(Array* array, usize new_capacity) { GEN_ASSERT( array != nullptr); GEN_ASSERT(* array != nullptr); - GEN_ASSERT(num > 0) ArrayHeader* header = array_get_header(array); if (header->Capacity < new_capacity) @@ -763,7 +763,7 @@ HashTableFindResult hashtable__find(HashTable table, u64 key) } template forceinline -bool hashtable_full(HashTable table) { +b32 hashtable_full(HashTable table) { GEN_ASSERT_NOT_NULL(table.Hashes); GEN_ASSERT_NOT_NULL(table.Entries); usize critical_load = usize(HashTable_CriticalLoadScale * f32(array_num(table.Hashes))); diff --git a/base/dependencies/debug.hpp b/base/dependencies/debug.hpp index 6870261..858b277 100644 --- a/base/dependencies/debug.hpp +++ b/base/dependencies/debug.hpp @@ -44,7 +44,7 @@ // NOTE: Things that shouldn't happen with a message! #define GEN_PANIC( msg, ... ) GEN_ASSERT_MSG( 0, msg, ##__VA_ARGS__ ) -#if GEN_BULD_DEBUG +#if GEN_BUILD_DEBUG #define GEN_FATAL( ... ) \ do \ { \ diff --git a/base/dependencies/memory.cpp b/base/dependencies/memory.cpp index 3d8ad2d..b38097a 100644 --- a/base/dependencies/memory.cpp +++ b/base/dependencies/memory.cpp @@ -346,25 +346,25 @@ void* arena_allocator_proc( void* allocator_data, AllocType type, ssize size, ss switch ( type ) { case EAllocation_ALLOC : + { + void* end = pointer_add( arena->PhysicalStart, arena->TotalUsed ); + ssize total_size = align_forward_s64( size, alignment ); + + // NOTE: Out of memory + if ( arena->TotalUsed + total_size > (ssize) arena->TotalSize ) { - void* end = pointer_add( arena->PhysicalStart, arena->TotalUsed ); - ssize total_size = align_forward_s64( size, alignment ); - - // NOTE: Out of memory - if ( arena->TotalUsed + total_size > (ssize) arena->TotalSize ) - { - // zpl__printf_err("%s", "Arena out of memory\n"); - GEN_FATAL("Arena out of memory! (Possibly could not fit for the largest size Arena!!)"); - return nullptr; - } - - ptr = align_forward( end, alignment ); - arena->TotalUsed += total_size; - - if ( flags & ALLOCATOR_FLAG_CLEAR_TO_ZERO ) - zero_size( ptr, size ); + // zpl__printf_err("%s", "Arena out of memory\n"); + GEN_FATAL("Arena out of memory! (Possibly could not fit for the largest size Arena!!)"); } - break; + + + ptr = align_forward( end, alignment ); + arena->TotalUsed += total_size; + + if ( flags & ALLOCATOR_FLAG_CLEAR_TO_ZERO ) + zero_size( ptr, size ); + } + break; case EAllocation_FREE : // NOTE: Free all at once diff --git a/base/dependencies/memory.hpp b/base/dependencies/memory.hpp index c28cb09..ff04129 100644 --- a/base/dependencies/memory.hpp +++ b/base/dependencies/memory.hpp @@ -229,7 +229,6 @@ forceinline void check(Arena& arena) { return arena_check(& arena); } #pragma pop_macro("check") #endif - inline AllocatorInfo arena_allocator_info( Arena* arena ) { GEN_ASSERT(arena != nullptr); @@ -392,9 +391,9 @@ void pool_clear(Pool* pool); void pool_free(Pool* pool); #if GEN_COMPILER_CPP && ! GEN_C_LIKE_CPP -AllocatorInfo allocator_info(Pool& pool) { return pool_allocator_info(& pool); } -void clear(Pool& pool) { return pool_clear(& pool); } -void free(Pool& pool) { return pool_free(& pool); } +forceinline AllocatorInfo allocator_info(Pool& pool) { return pool_allocator_info(& pool); } +forceinline void clear(Pool& pool) { return pool_clear(& pool); } +forceinline void free(Pool& pool) { return pool_free(& pool); } #endif struct Pool diff --git a/base/dependencies/strings.hpp b/base/dependencies/strings.hpp index 3c60b68..76b7ce1 100644 --- a/base/dependencies/strings.hpp +++ b/base/dependencies/strings.hpp @@ -392,6 +392,7 @@ bool strbuilder_append_string(StrBuilder* str, StrBuilder const other) { return strbuilder_append_c_str_len(str, (char const*)other, strbuilder_length(other)); } +inline bool strbuilder_append_fmt(StrBuilder* str, char const* fmt, ...) { GEN_ASSERT(str != nullptr); ssize res; diff --git a/gen_unreal_engine/unreal.cpp b/gen_unreal_engine/unreal.cpp index 5719a7c..2afa5ae 100644 --- a/gen_unreal_engine/unreal.cpp +++ b/gen_unreal_engine/unreal.cpp @@ -147,6 +147,7 @@ int gen_main() Code strings = scan_file( path_base "dependencies/strings.hpp" ); Code filesystem = scan_file( path_base "dependencies/filesystem.hpp" ); Code timing = scan_file( path_base "dependencies/timing.hpp" ); + Code parsing = scan_file( path_base "dependencies/parsing.hpp" ); Builder header = Builder::open("gen/gen.dep.hpp"); @@ -168,6 +169,7 @@ int gen_main() header.print( strings ); header.print( filesystem ); header.print( timing ); + header.print(parsing); header.print_fmt( "\nGEN_NS_END\n" ); header.print( fmt_newline ); @@ -186,6 +188,7 @@ int gen_main() Code strings = scan_file( path_base "dependencies/strings.cpp" ); Code filesystem = scan_file( path_base "dependencies/filesystem.cpp" ); Code timing = scan_file( path_base "dependencies/timing.cpp" ); + Code parsing = scan_file( path_base "dependencies/parsing.cpp" ); Builder src = Builder::open( "gen/gen.dep.cpp" ); @@ -204,6 +207,7 @@ int gen_main() src.print( strings ); src.print( filesystem ); src.print( timing ); + src.print( parsing ); src.print_fmt( "\nGEN_NS_END\n" ); src.print( fmt_newline ); @@ -310,14 +314,20 @@ int gen_main() src.print_fmt( "\n#pragma region Interface\n" ); src.print( interface ); + src.print( upfront ); + src.print_fmt( "\n#pragma region Parsing\n\n" ); src.print( lexer ); src.print( parser_case_macros ); src.print( parser ); src.print( parsing_interface ); - src.print( untyped ); src.print_fmt( "\n#pragma endregion Parsing\n\n" ); + + src.print_fmt( "\n#pragma region Untyped\n\n" ); + src.print( untyped ); + src.print_fmt( "#pragma endregion \n\n" ); + src.print_fmt( "#pragma endregion Interface\n\n" ); src.print_fmt( "GEN_NS_END\n\n"); @@ -364,7 +374,6 @@ int gen_main() // gen_scanner.hpp { - Code parsing = scan_file( path_base "dependencies/parsing.hpp" ); Code scanner = scan_file( path_base "auxillary/scanner.hpp" ); Builder @@ -375,7 +384,6 @@ int gen_main() header.print( fmt_newline ); header.print( def_include( txt("gen.hpp") ) ); header.print_fmt( "\nGEN_NS_BEGIN\n" ); - header.print( parsing ); header.print( scanner ); header.print_fmt( "\nGEN_NS_END\n" ); header.print( fmt_newline ); @@ -385,7 +393,6 @@ int gen_main() // gen.scanner.cpp { - Code parsing = scan_file( path_base "dependencies/parsing.cpp" ); Code scanner = scan_file( path_base "auxillary/scanner.cpp" ); Builder @@ -395,8 +402,7 @@ int gen_main() src.print( fmt_newline ); src.print( def_include( txt("gen.scanner.hpp") ) ); src.print_fmt( "\nGEN_NS_BEGIN\n" ); - src.print( parsing ); - // src.print( scanner ); + src.print( scanner ); src.print_fmt( "\nGEN_NS_END\n" ); src.print( fmt_newline ); src.print( pop_ignores ); From a6143e12b4292851f021e570bd1ddf3d7265ad8b Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 18:21:03 -0500 Subject: [PATCH 14/15] fix null check in def_variable --- base/components/interface.upfront.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/components/interface.upfront.cpp b/base/components/interface.upfront.cpp index d6be498..d064a8b 100644 --- a/base/components/interface.upfront.cpp +++ b/base/components/interface.upfront.cpp @@ -1274,7 +1274,7 @@ CodeUsing def_using_namespace( Str name ) CodeVar def_variable( CodeTypename type, Str name, Opts_def_variable p ) { - if ( ! name_check( def_variable, name ) || null_check( def_variable, type ) ) { + if ( ! name_check( def_variable, name ) || ! null_check( def_variable, type ) ) { GEN_DEBUG_TRAP(); return InvalidCode; } From b0277783288d6e7f1376fbb16cc2552c13e0a443 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Dec 2024 19:54:27 -0500 Subject: [PATCH 15/15] last set of fixes for UE --- base/components/interface.cpp | 3 + base/components/interface.hpp | 14 ++- base/components/interface.untyped.cpp | 13 +- base/dependencies/macros.hpp | 16 +-- base/helpers/base_codegen.hpp | 166 +++++++++++++------------- 5 files changed, 113 insertions(+), 99 deletions(-) diff --git a/base/components/interface.cpp b/base/components/interface.cpp index ab2f580..f0e97ee 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -376,6 +376,9 @@ void deinit(Context* ctx) if (_ctx == ctx) _ctx = nullptr; -- context_counter; + + Context wipe = {}; + * ctx = wipe; } Context* get_context() { diff --git a/base/components/interface.hpp b/base/components/interface.hpp index 0ee2422..8a849ad 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -79,6 +79,8 @@ struct Context u32 InitSize_Fallback_Allocator_Bucket_Size; Array(Arena) Fallback_AllocatorBuckets; + StringTable token_fmt_map; + // Array(Token) LexerTokens; Array(Pool) CodePools; @@ -381,12 +383,20 @@ GEN_API Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... ); #ifndef name // Convienence for defining any name used with the gen api. // Lets you provide the length and string literal to the functions without the need for the DSL. -#define name( Id_ ) { stringize(Id_), sizeof(stringize( Id_ )) - 1 } +# if GEN_COMPILER_C +# define name( Id_ ) (Str){ stringize(Id_), sizeof(stringize( Id_ )) - 1 } +# else +# define name( Id_ ) Str { stringize(Id_), sizeof(stringize( Id_ )) - 1 } +# endif #endif #ifndef code // Same as name just used to indicate intention of literal for code instead of names. -#define code( ... ) { stringize( __VA_ARGS__ ), sizeof(stringize(__VA_ARGS__)) - 1 } +# if GEN_COMPILER_C +# define code( ... ) (Str){ stringize( __VA_ARGS__ ), sizeof(stringize(__VA_ARGS__)) - 1 } +# else +# define code( ... ) Str { stringize( __VA_ARGS__ ), sizeof(stringize(__VA_ARGS__)) - 1 } +# endif #endif #ifndef args diff --git a/base/components/interface.untyped.cpp b/base/components/interface.untyped.cpp index e95abf5..bc8afab 100644 --- a/base/components/interface.untyped.cpp +++ b/base/components/interface.untyped.cpp @@ -8,9 +8,8 @@ ssize token_fmt_va( char* buf, usize buf_size, s32 num_tokens, va_list va ) char const* buf_begin = buf; ssize remaining = buf_size; - local_persist StringTable tok_map; - do_once() { - tok_map = hashtable_init(Str, _ctx->Allocator_DyanmicContainers ); + if (_ctx->token_fmt_map.Hashes == nullptr) { + _ctx->token_fmt_map = hashtable_init(Str, _ctx->Allocator_DyanmicContainers ); } // Populate token pairs { @@ -22,7 +21,7 @@ ssize token_fmt_va( char* buf, usize buf_size, s32 num_tokens, va_list va ) Str value = va_arg( va, Str ); u32 key = crc32( token, c_str_len(token) ); - hashtable_set( tok_map, key, value ); + hashtable_set( _ctx->token_fmt_map, key, value ); } } @@ -57,8 +56,8 @@ ssize token_fmt_va( char* buf, usize buf_size, s32 num_tokens, va_list va ) char const* token = fmt + 1; - u32 key = crc32( token, tok_len ); - Str* value = hashtable_get(tok_map, key ); + u32 key = crc32( token, tok_len ); + Str* value = hashtable_get(_ctx->token_fmt_map, key ); if ( value ) { @@ -87,7 +86,7 @@ ssize token_fmt_va( char* buf, usize buf_size, s32 num_tokens, va_list va ) current = * fmt; } } - hashtable_clear(tok_map); + hashtable_clear(_ctx->token_fmt_map); ssize result = buf_size - remaining; return result; } diff --git a/base/dependencies/macros.hpp b/base/dependencies/macros.hpp index a8636fa..f582d65 100644 --- a/base/dependencies/macros.hpp +++ b/base/dependencies/macros.hpp @@ -86,14 +86,16 @@ #define stringize( ... ) stringize_va( __VA_ARGS__ ) #endif -#ifndef do_once -#define do_once() \ - static int __do_once_counter_##__LINE__ = 0; \ - for(; __do_once_counter_##__LINE__ != 1; __do_once_counter_##__LINE__ = 1 ) \ +#define src_line_str stringize(__LINE__) -#define do_once_defer( expression ) \ - static int __do_once_counter_##__LINE__ = 0; \ - for(; __do_once_counter_##__LINE__ != 1; __do_once_counter_##__LINE__ = 1, (expression)) \ +#ifndef do_once +#define do_once() \ + static int __do_once_counter_##src_line_str = 0; \ + for(; __do_once_counter_##src_line_str != 1; __do_once_counter_##src_line_str = 1 ) \ + +#define do_once_defer( expression ) \ + static int __do_once_counter_##src_line_str = 0; \ + for(; __do_once_counter_##src_line_str != 1; __do_once_counter_##src_line_str = 1, (expression)) \ #define do_once_start \ do \ diff --git a/base/helpers/base_codegen.hpp b/base/helpers/base_codegen.hpp index 47293fc..10d1765 100644 --- a/base/helpers/base_codegen.hpp +++ b/base/helpers/base_codegen.hpp @@ -283,7 +283,7 @@ CodeBody gen_especifier( char const* path, bool use_c_definition = false ) body_append(result, enum_code); if (use_c_definition) { - CodeTypedef specifier_t = parse_typedef( code(typedef u32 Specifier; )); + CodeTypedef specifier_t = parse_typedef( code(typedef enum Specifier Specifier; )); body_append(result, specifier_t); } @@ -499,60 +499,60 @@ CodeBody gen_ast_inlines() #pragma pop_macro("GEN_NS") #pragma pop_macro("CodeInvalid") - CodeBody impl_code = parse_global_body( token_fmt( "typename", Str name(Code), code_impl_tmpl )); - CodeBody impl_code_body = parse_global_body( token_fmt( "typename", Str name(CodeBody), code_impl_tmpl )); - CodeBody impl_code_attr = parse_global_body( token_fmt( "typename", Str name(CodeAttributes), code_impl_tmpl )); - CodeBody impl_code_cmt = parse_global_body( token_fmt( "typename", Str name(CodeComment), code_impl_tmpl )); - CodeBody impl_code_constr = parse_global_body( token_fmt( "typename", Str name(CodeConstructor), code_impl_tmpl )); - CodeBody impl_code_class = parse_global_body( token_fmt( "typename", Str name(CodeClass), code_impl_tmpl )); - CodeBody impl_code_define = parse_global_body( token_fmt( "typename", Str name(CodeDefine), code_impl_tmpl )); - CodeBody impl_code_define_params = parse_global_body( token_fmt( "typename", Str name(CodeDefineParams), code_impl_tmpl )); - CodeBody impl_code_destruct = parse_global_body( token_fmt( "typename", Str name(CodeDestructor), code_impl_tmpl )); - CodeBody impl_code_enum = parse_global_body( token_fmt( "typename", Str name(CodeEnum), code_impl_tmpl )); - CodeBody impl_code_exec = parse_global_body( token_fmt( "typename", Str name(CodeExec), code_impl_tmpl )); - CodeBody impl_code_extern = parse_global_body( token_fmt( "typename", Str name(CodeExtern), code_impl_tmpl )); - CodeBody impl_code_include = parse_global_body( token_fmt( "typename", Str name(CodeInclude), code_impl_tmpl )); - CodeBody impl_code_friend = parse_global_body( token_fmt( "typename", Str name(CodeFriend), code_impl_tmpl )); - CodeBody impl_code_fn = parse_global_body( token_fmt( "typename", Str name(CodeFn), code_impl_tmpl )); - CodeBody impl_code_module = parse_global_body( token_fmt( "typename", Str name(CodeModule), code_impl_tmpl )); - CodeBody impl_code_ns = parse_global_body( token_fmt( "typename", Str name(CodeNS), code_impl_tmpl )); - CodeBody impl_code_op = parse_global_body( token_fmt( "typename", Str name(CodeOperator), code_impl_tmpl )); - CodeBody impl_code_opcast = parse_global_body( token_fmt( "typename", Str name(CodeOpCast), code_impl_tmpl )); - CodeBody impl_code_params = parse_global_body( token_fmt( "typename", Str name(CodeParams), code_impl_tmpl )); - CodeBody impl_code_pragma = parse_global_body( token_fmt( "typename", Str name(CodePragma), code_impl_tmpl )); - CodeBody impl_code_precond = parse_global_body( token_fmt( "typename", Str name(CodePreprocessCond), code_impl_tmpl )); - CodeBody impl_code_specs = parse_global_body( token_fmt( "typename", Str name(CodeSpecifiers), code_impl_tmpl )); - CodeBody impl_code_struct = parse_global_body( token_fmt( "typename", Str name(CodeStruct), code_impl_tmpl )); - CodeBody impl_code_tmpl = parse_global_body( token_fmt( "typename", Str name(CodeTemplate), code_impl_tmpl )); - CodeBody impl_code_type = parse_global_body( token_fmt( "typename", Str name(CodeTypename), code_impl_tmpl )); - CodeBody impl_code_typedef = parse_global_body( token_fmt( "typename", Str name(CodeTypedef), code_impl_tmpl )); - CodeBody impl_code_union = parse_global_body( token_fmt( "typename", Str name(CodeUnion), code_impl_tmpl )); - CodeBody impl_code_using = parse_global_body( token_fmt( "typename", Str name(CodeUsing), code_impl_tmpl )); - CodeBody impl_code_var = parse_global_body( token_fmt( "typename", Str name(CodeVar), code_impl_tmpl )); + CodeBody impl_code = parse_global_body( token_fmt( "typename", name(Code), code_impl_tmpl )); + CodeBody impl_code_body = parse_global_body( token_fmt( "typename", name(CodeBody), code_impl_tmpl )); + CodeBody impl_code_attr = parse_global_body( token_fmt( "typename", name(CodeAttributes), code_impl_tmpl )); + CodeBody impl_code_cmt = parse_global_body( token_fmt( "typename", name(CodeComment), code_impl_tmpl )); + CodeBody impl_code_constr = parse_global_body( token_fmt( "typename", name(CodeConstructor), code_impl_tmpl )); + CodeBody impl_code_class = parse_global_body( token_fmt( "typename", name(CodeClass), code_impl_tmpl )); + CodeBody impl_code_define = parse_global_body( token_fmt( "typename", name(CodeDefine), code_impl_tmpl )); + CodeBody impl_code_define_params = parse_global_body( token_fmt( "typename", name(CodeDefineParams), code_impl_tmpl )); + CodeBody impl_code_destruct = parse_global_body( token_fmt( "typename", name(CodeDestructor), code_impl_tmpl )); + CodeBody impl_code_enum = parse_global_body( token_fmt( "typename", name(CodeEnum), code_impl_tmpl )); + CodeBody impl_code_exec = parse_global_body( token_fmt( "typename", name(CodeExec), code_impl_tmpl )); + CodeBody impl_code_extern = parse_global_body( token_fmt( "typename", name(CodeExtern), code_impl_tmpl )); + CodeBody impl_code_include = parse_global_body( token_fmt( "typename", name(CodeInclude), code_impl_tmpl )); + CodeBody impl_code_friend = parse_global_body( token_fmt( "typename", name(CodeFriend), code_impl_tmpl )); + CodeBody impl_code_fn = parse_global_body( token_fmt( "typename", name(CodeFn), code_impl_tmpl )); + CodeBody impl_code_module = parse_global_body( token_fmt( "typename", name(CodeModule), code_impl_tmpl )); + CodeBody impl_code_ns = parse_global_body( token_fmt( "typename", name(CodeNS), code_impl_tmpl )); + CodeBody impl_code_op = parse_global_body( token_fmt( "typename", name(CodeOperator), code_impl_tmpl )); + CodeBody impl_code_opcast = parse_global_body( token_fmt( "typename", name(CodeOpCast), code_impl_tmpl )); + CodeBody impl_code_params = parse_global_body( token_fmt( "typename", name(CodeParams), code_impl_tmpl )); + CodeBody impl_code_pragma = parse_global_body( token_fmt( "typename", name(CodePragma), code_impl_tmpl )); + CodeBody impl_code_precond = parse_global_body( token_fmt( "typename", name(CodePreprocessCond), code_impl_tmpl )); + CodeBody impl_code_specs = parse_global_body( token_fmt( "typename", name(CodeSpecifiers), code_impl_tmpl )); + CodeBody impl_code_struct = parse_global_body( token_fmt( "typename", name(CodeStruct), code_impl_tmpl )); + CodeBody impl_code_tmpl = parse_global_body( token_fmt( "typename", name(CodeTemplate), code_impl_tmpl )); + CodeBody impl_code_type = parse_global_body( token_fmt( "typename", name(CodeTypename), code_impl_tmpl )); + CodeBody impl_code_typedef = parse_global_body( token_fmt( "typename", name(CodeTypedef), code_impl_tmpl )); + CodeBody impl_code_union = parse_global_body( token_fmt( "typename", name(CodeUnion), code_impl_tmpl )); + CodeBody impl_code_using = parse_global_body( token_fmt( "typename", name(CodeUsing), code_impl_tmpl )); + CodeBody impl_code_var = parse_global_body( token_fmt( "typename", name(CodeVar), code_impl_tmpl )); - body_append(impl_code_attr, parse_global_body( token_fmt( "typename", Str name(Attributes), codetype_impl_tmpl ))); - body_append(impl_code_cmt, parse_global_body( token_fmt( "typename", Str name(Comment), codetype_impl_tmpl ))); - body_append(impl_code_constr, parse_global_body( token_fmt( "typename", Str name(Constructor), codetype_impl_tmpl ))); - body_append(impl_code_define, parse_global_body( token_fmt( "typename", Str name(Define), codetype_impl_tmpl ))); - body_append(impl_code_destruct, parse_global_body( token_fmt( "typename", Str name(Destructor), codetype_impl_tmpl ))); - body_append(impl_code_enum, parse_global_body( token_fmt( "typename", Str name(Enum), codetype_impl_tmpl ))); - body_append(impl_code_exec, parse_global_body( token_fmt( "typename", Str name(Exec), codetype_impl_tmpl ))); - body_append(impl_code_extern, parse_global_body( token_fmt( "typename", Str name(Extern), codetype_impl_tmpl ))); - body_append(impl_code_include, parse_global_body( token_fmt( "typename", Str name(Include), codetype_impl_tmpl ))); - body_append(impl_code_friend, parse_global_body( token_fmt( "typename", Str name(Friend), codetype_impl_tmpl ))); - body_append(impl_code_fn, parse_global_body( token_fmt( "typename", Str name(Fn), codetype_impl_tmpl ))); - body_append(impl_code_module, parse_global_body( token_fmt( "typename", Str name(Module), codetype_impl_tmpl ))); - body_append(impl_code_ns, parse_global_body( token_fmt( "typename", Str name(NS), codetype_impl_tmpl ))); - body_append(impl_code_op, parse_global_body( token_fmt( "typename", Str name(Operator), codetype_impl_tmpl ))); - body_append(impl_code_opcast, parse_global_body( token_fmt( "typename", Str name(OpCast), codetype_impl_tmpl ))); - body_append(impl_code_pragma, parse_global_body( token_fmt( "typename", Str name(Pragma), codetype_impl_tmpl ))); - body_append(impl_code_precond, parse_global_body( token_fmt( "typename", Str name(PreprocessCond), codetype_impl_tmpl ))); - body_append(impl_code_tmpl, parse_global_body( token_fmt( "typename", Str name(Template), codetype_impl_tmpl ))); - body_append(impl_code_type, parse_global_body( token_fmt( "typename", Str name(Typename), codetype_impl_tmpl ))); - body_append(impl_code_typedef, parse_global_body( token_fmt( "typename", Str name(Typedef), codetype_impl_tmpl ))); - body_append(impl_code_union, parse_global_body( token_fmt( "typename", Str name(Union), codetype_impl_tmpl ))); - body_append(impl_code_using, parse_global_body( token_fmt( "typename", Str name(Using), codetype_impl_tmpl ))); - body_append(impl_code_var, parse_global_body( token_fmt( "typename", Str name(Var), codetype_impl_tmpl ))); + body_append(impl_code_attr, parse_global_body( token_fmt( "typename", name(Attributes), codetype_impl_tmpl ))); + body_append(impl_code_cmt, parse_global_body( token_fmt( "typename", name(Comment), codetype_impl_tmpl ))); + body_append(impl_code_constr, parse_global_body( token_fmt( "typename", name(Constructor), codetype_impl_tmpl ))); + body_append(impl_code_define, parse_global_body( token_fmt( "typename", name(Define), codetype_impl_tmpl ))); + body_append(impl_code_destruct, parse_global_body( token_fmt( "typename", name(Destructor), codetype_impl_tmpl ))); + body_append(impl_code_enum, parse_global_body( token_fmt( "typename", name(Enum), codetype_impl_tmpl ))); + body_append(impl_code_exec, parse_global_body( token_fmt( "typename", name(Exec), codetype_impl_tmpl ))); + body_append(impl_code_extern, parse_global_body( token_fmt( "typename", name(Extern), codetype_impl_tmpl ))); + body_append(impl_code_include, parse_global_body( token_fmt( "typename", name(Include), codetype_impl_tmpl ))); + body_append(impl_code_friend, parse_global_body( token_fmt( "typename", name(Friend), codetype_impl_tmpl ))); + body_append(impl_code_fn, parse_global_body( token_fmt( "typename", name(Fn), codetype_impl_tmpl ))); + body_append(impl_code_module, parse_global_body( token_fmt( "typename", name(Module), codetype_impl_tmpl ))); + body_append(impl_code_ns, parse_global_body( token_fmt( "typename", name(NS), codetype_impl_tmpl ))); + body_append(impl_code_op, parse_global_body( token_fmt( "typename", name(Operator), codetype_impl_tmpl ))); + body_append(impl_code_opcast, parse_global_body( token_fmt( "typename", name(OpCast), codetype_impl_tmpl ))); + body_append(impl_code_pragma, parse_global_body( token_fmt( "typename", name(Pragma), codetype_impl_tmpl ))); + body_append(impl_code_precond, parse_global_body( token_fmt( "typename", name(PreprocessCond), codetype_impl_tmpl ))); + body_append(impl_code_tmpl, parse_global_body( token_fmt( "typename", name(Template), codetype_impl_tmpl ))); + body_append(impl_code_type, parse_global_body( token_fmt( "typename", name(Typename), codetype_impl_tmpl ))); + body_append(impl_code_typedef, parse_global_body( token_fmt( "typename", name(Typedef), codetype_impl_tmpl ))); + body_append(impl_code_union, parse_global_body( token_fmt( "typename", name(Union), codetype_impl_tmpl ))); + body_append(impl_code_using, parse_global_body( token_fmt( "typename", name(Using), codetype_impl_tmpl ))); + body_append(impl_code_var, parse_global_body( token_fmt( "typename", name(Var), codetype_impl_tmpl ))); #pragma push_macro("forceinline") #undef forceinline @@ -564,35 +564,35 @@ CodeBody gen_ast_inlines() ); #pragma pop_macro("forceinline") - CodeBody impl_cast_body = parse_global_body( token_fmt( "typename", Str name(Body), cast_tmpl )); - CodeBody impl_cast_attribute = parse_global_body( token_fmt( "typename", Str name(Attributes), cast_tmpl )); - CodeBody impl_cast_cmt = parse_global_body( token_fmt( "typename", Str name(Comment), cast_tmpl )); - CodeBody impl_cast_constr = parse_global_body( token_fmt( "typename", Str name(Constructor), cast_tmpl )); - CodeBody impl_cast_class = parse_global_body( token_fmt( "typename", Str name(Class), cast_tmpl )); - CodeBody impl_cast_define = parse_global_body( token_fmt( "typename", Str name(Define), cast_tmpl )); - CodeBody impl_cast_define_params = parse_global_body( token_fmt( "typename", Str name(DefineParams), cast_tmpl )); - CodeBody impl_cast_destruct = parse_global_body( token_fmt( "typename", Str name(Destructor), cast_tmpl )); - CodeBody impl_cast_enum = parse_global_body( token_fmt( "typename", Str name(Enum), cast_tmpl )); - CodeBody impl_cast_exec = parse_global_body( token_fmt( "typename", Str name(Exec), cast_tmpl )); - CodeBody impl_cast_extern = parse_global_body( token_fmt( "typename", Str name(Extern), cast_tmpl )); - CodeBody impl_cast_friend = parse_global_body( token_fmt( "typename", Str name(Friend), cast_tmpl )); - CodeBody impl_cast_fn = parse_global_body( token_fmt( "typename", Str name(Fn), cast_tmpl )); - CodeBody impl_cast_include = parse_global_body( token_fmt( "typename", Str name(Include), cast_tmpl )); - CodeBody impl_cast_module = parse_global_body( token_fmt( "typename", Str name(Module), cast_tmpl )); - CodeBody impl_cast_ns = parse_global_body( token_fmt( "typename", Str name(NS), cast_tmpl )); - CodeBody impl_cast_op = parse_global_body( token_fmt( "typename", Str name(Operator), cast_tmpl )); - CodeBody impl_cast_opcast = parse_global_body( token_fmt( "typename", Str name(OpCast), cast_tmpl )); - CodeBody impl_cast_params = parse_global_body( token_fmt( "typename", Str name(Params), cast_tmpl )); - CodeBody impl_cast_pragma = parse_global_body( token_fmt( "typename", Str name(Pragma), cast_tmpl )); - CodeBody impl_cast_precond = parse_global_body( token_fmt( "typename", Str name(PreprocessCond), cast_tmpl )); - CodeBody impl_cast_specs = parse_global_body( token_fmt( "typename", Str name(Specifiers), cast_tmpl )); - CodeBody impl_cast_struct = parse_global_body( token_fmt( "typename", Str name(Struct), cast_tmpl )); - CodeBody impl_cast_tmpl = parse_global_body( token_fmt( "typename", Str name(Template), cast_tmpl )); - CodeBody impl_cast_type = parse_global_body( token_fmt( "typename", Str name(Typename), cast_tmpl )); - CodeBody impl_cast_typedef = parse_global_body( token_fmt( "typename", Str name(Typedef), cast_tmpl )); - CodeBody impl_cast_union = parse_global_body( token_fmt( "typename", Str name(Union), cast_tmpl )); - CodeBody impl_cast_using = parse_global_body( token_fmt( "typename", Str name(Using), cast_tmpl )); - CodeBody impl_cast_var = parse_global_body( token_fmt( "typename", Str name(Var), cast_tmpl )); + CodeBody impl_cast_body = parse_global_body( token_fmt( "typename", name(Body), cast_tmpl )); + CodeBody impl_cast_attribute = parse_global_body( token_fmt( "typename", name(Attributes), cast_tmpl )); + CodeBody impl_cast_cmt = parse_global_body( token_fmt( "typename", name(Comment), cast_tmpl )); + CodeBody impl_cast_constr = parse_global_body( token_fmt( "typename", name(Constructor), cast_tmpl )); + CodeBody impl_cast_class = parse_global_body( token_fmt( "typename", name(Class), cast_tmpl )); + CodeBody impl_cast_define = parse_global_body( token_fmt( "typename", name(Define), cast_tmpl )); + CodeBody impl_cast_define_params = parse_global_body( token_fmt( "typename", name(DefineParams), cast_tmpl )); + CodeBody impl_cast_destruct = parse_global_body( token_fmt( "typename", name(Destructor), cast_tmpl )); + CodeBody impl_cast_enum = parse_global_body( token_fmt( "typename", name(Enum), cast_tmpl )); + CodeBody impl_cast_exec = parse_global_body( token_fmt( "typename", name(Exec), cast_tmpl )); + CodeBody impl_cast_extern = parse_global_body( token_fmt( "typename", name(Extern), cast_tmpl )); + CodeBody impl_cast_friend = parse_global_body( token_fmt( "typename", name(Friend), cast_tmpl )); + CodeBody impl_cast_fn = parse_global_body( token_fmt( "typename", name(Fn), cast_tmpl )); + CodeBody impl_cast_include = parse_global_body( token_fmt( "typename", name(Include), cast_tmpl )); + CodeBody impl_cast_module = parse_global_body( token_fmt( "typename", name(Module), cast_tmpl )); + CodeBody impl_cast_ns = parse_global_body( token_fmt( "typename", name(NS), cast_tmpl )); + CodeBody impl_cast_op = parse_global_body( token_fmt( "typename", name(Operator), cast_tmpl )); + CodeBody impl_cast_opcast = parse_global_body( token_fmt( "typename", name(OpCast), cast_tmpl )); + CodeBody impl_cast_params = parse_global_body( token_fmt( "typename", name(Params), cast_tmpl )); + CodeBody impl_cast_pragma = parse_global_body( token_fmt( "typename", name(Pragma), cast_tmpl )); + CodeBody impl_cast_precond = parse_global_body( token_fmt( "typename", name(PreprocessCond), cast_tmpl )); + CodeBody impl_cast_specs = parse_global_body( token_fmt( "typename", name(Specifiers), cast_tmpl )); + CodeBody impl_cast_struct = parse_global_body( token_fmt( "typename", name(Struct), cast_tmpl )); + CodeBody impl_cast_tmpl = parse_global_body( token_fmt( "typename", name(Template), cast_tmpl )); + CodeBody impl_cast_type = parse_global_body( token_fmt( "typename", name(Typename), cast_tmpl )); + CodeBody impl_cast_typedef = parse_global_body( token_fmt( "typename", name(Typedef), cast_tmpl )); + CodeBody impl_cast_union = parse_global_body( token_fmt( "typename", name(Union), cast_tmpl )); + CodeBody impl_cast_using = parse_global_body( token_fmt( "typename", name(Using), cast_tmpl )); + CodeBody impl_cast_var = parse_global_body( token_fmt( "typename", name(Var), cast_tmpl )); CodeBody result = def_global_body( args( def_pragma( txt("region generated code inline implementation")),