From 16d0e0834f37e90d984774bae8a662ff65d4d791 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Fri, 13 Dec 2024 20:40:18 -0500 Subject: [PATCH] All global vars (except concepts) have been retrofitetd to the library's new Context struct --- base/base.cpp | 2 +- base/components/ast.cpp | 4 - base/components/ast.hpp | 17 +- .../gen/{etoktype.cpp => etoktype.hpp} | 4 - base/components/interface.cpp | 42 +- base/components/interface.hpp | 20 +- base/components/interface.parsing.cpp | 69 +-- base/components/interface.upfront.cpp | 169 +---- base/components/lexer.cpp | 162 +---- base/components/parser.cpp | 583 +++++++++--------- base/components/parser_types.hpp | 125 ++++ base/components/static_data.cpp | 5 + base/dependencies/filesystem.cpp | 2 +- base/dependencies/platform.hpp | 12 - base/gen.cpp | 1 - base/gen.hpp | 2 + base/helpers/base_codegen.hpp | 2 - gen_segmented/segmented.cpp | 11 +- gen_singleheader/singleheader.cpp | 27 +- gen_unreal_engine/components/header_start.hpp | 20 +- gen_unreal_engine/unreal.cpp | 21 +- 21 files changed, 584 insertions(+), 716 deletions(-) rename base/components/gen/{etoktype.cpp => etoktype.hpp} (99%) create mode 100644 base/components/parser_types.hpp diff --git a/base/base.cpp b/base/base.cpp index 2a7b475..6e87efc 100644 --- a/base/base.cpp +++ b/base/base.cpp @@ -60,7 +60,7 @@ int gen_main() builder_print( & header_especifier, format(especifier) ); builder_write( & header_especifier); - Builder header_etoktype = builder_open( "components/gen/etoktype.cpp" ); + Builder header_etoktype = builder_open( "components/gen/etoktype.hpp" ); builder_print( & header_etoktype, gen_component_header ); builder_print( & header_etoktype, format(etoktype) ); builder_write( & header_etoktype); diff --git a/base/components/ast.cpp b/base/components/ast.cpp index f1efe5d..f9a4d8c 100644 --- a/base/components/ast.cpp +++ b/base/components/ast.cpp @@ -3,9 +3,6 @@ #include "static_data.cpp" #endif -global Code Code_Global; -global Code Code_Invalid; - // This serializes all the data-members in a "debug" format, where each member is printed with its associated value. Str code_debug_str(Code self) { @@ -1283,6 +1280,5 @@ bool code_validate_body(Code self) log_failure( "AST::validate_body: Invalid this AST does not have a body %S", code_debug_str(self) ); return false; } - return false; } diff --git a/base/components/ast.hpp b/base/components/ast.hpp index 9f39db4..487cb5f 100644 --- a/base/components/ast.hpp +++ b/base/components/ast.hpp @@ -230,21 +230,6 @@ struct CodeUsing; struct CodeVar; #endif -GEN_NS_PARSER_BEGIN - -struct Token; - -GEN_NS_PARSER_END - -#if GEN_COMPILER_CPP -// Note(Ed): This is to alleviate an edge case with parsing usings or typedefs where I don't really have it setup -// to parse a 'namespace' macro or a type with a macro. -// I have ideas for ways to pack that into the typedef/using ast, but for now just keeping it like this -#define ParserTokenType GEN_NS_PARSER Token -typedef ParserTokenType Token; -#undef ParserTokenType -#endif - #if GEN_COMPILER_CPP template< class Type> forceinline Type tmpl_cast( Code self ) { return * rcast( Type*, & self ); } #endif @@ -455,7 +440,7 @@ static_assert( sizeof(AST) == AST_POD_Size, "ERROR: AST is not size of AST_POD_S struct InvalidCode_ImplictCaster; #define InvalidCode (InvalidCode_ImplictCaster{}) #else -#define InvalidCode (void*){ (void*)Code_Invalid } +#define InvalidCode (void*){ (void*)_ctx->Code_Invalid } #endif #if GEN_COMPILER_CPP diff --git a/base/components/gen/etoktype.cpp b/base/components/gen/etoktype.hpp similarity index 99% rename from base/components/gen/etoktype.cpp rename to base/components/gen/etoktype.hpp index e5b4c68..627424a 100644 --- a/base/components/gen/etoktype.cpp +++ b/base/components/gen/etoktype.hpp @@ -5,8 +5,6 @@ // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) -GEN_NS_PARSER_BEGIN - #define GEN_DEFINE_ATTRIBUTE_TOKENS Entry( Tok_Attribute_API_Export, "GEN_API_Export_Code" ) Entry( Tok_Attribute_API_Import, "GEN_API_Import_Code" ) enum TokType : u32 @@ -231,5 +229,3 @@ inline TokType str_to_toktype( Str str ) } return Tok_Invalid; } - -GEN_NS_PARSER_END diff --git a/base/components/interface.cpp b/base/components/interface.cpp index 7d2ebf3..761ccbf 100644 --- a/base/components/interface.cpp +++ b/base/components/interface.cpp @@ -3,10 +3,8 @@ #include "code_serialization.cpp" #endif -GEN_NS_PARSER_BEGIN internal void parser_init(); internal void parser_deinit(); -GEN_NS_PARSER_END internal void* fallback_allocator_proc( void* allocator_data, AllocType type, ssize size, ssize alignment, void* old_memory, ssize old_size, u64 flags ) @@ -76,6 +74,10 @@ void* fallback_allocator_proc( void* allocator_data, AllocType type, ssize size, internal void define_constants() { + // We only initalize these if there is no base context. + if ( context_counter > 0 ) + return; + Code_Global = make_code(); Code_Global->Name = cache_str( txt("Global Code") ); Code_Global->Content = Code_Global->Name; @@ -208,12 +210,13 @@ void define_constants() enum_underlying_sig = txt("enum_underlying("); } array_append( _ctx->PreprocessorDefines, enum_underlying_sig); - -# undef def_constant_spec } void init(Context* ctx) { + do_once() { + context_counter = 0; + } AllocatorInfo fallback_allocator = { & fallback_allocator_proc, nullptr }; b32 using_fallback_allocator = false; @@ -307,23 +310,25 @@ void init(Context* ctx) GEN_FATAL( "gen::init: Failed to initialize the string arena" ); array_append( ctx->StringArenas, strbuilder_arena ); } - // Setup the hash tables { ctx->StrCache = hashtable_init(StrCached, ctx->Allocator_DyanmicContainers); if ( ctx->StrCache.Entries == nullptr ) GEN_FATAL( "gen::init: Failed to initialize the StringCache"); } - // Preprocessor Defines ctx->PreprocessorDefines = array_init_reserve(StrCached, ctx->Allocator_DyanmicContainers, kilobytes(1) ); define_constants(); - GEN_NS_PARSER parser_init(); + parser_init(); + + ++ context_counter; } void deinit(Context* ctx) { + GEN_ASSERT(context_counter); + GEN_ASSERT_MSG(context_counter > 0, "Attempted to deinit a context that for some reason wan't accounted for!"); usize index = 0; usize left = array_num(ctx->CodePools); do @@ -353,21 +358,24 @@ void deinit(Context* ctx) array_free(ctx->PreprocessorDefines); - index = 0; left = array_num( ctx->Fallback_AllocatorBuckets); - do + if (left) { - Arena* bucket = & ctx->Fallback_AllocatorBuckets[ index ]; - arena_free(bucket); - index++; + index = 0; + do + { + Arena* bucket = & ctx->Fallback_AllocatorBuckets[ index ]; + arena_free(bucket); + index++; + } + while ( left--, left ); + array_free( ctx->Fallback_AllocatorBuckets); } - while ( left--, left ); - - array_free( ctx->Fallback_AllocatorBuckets); - GEN_NS_PARSER parser_deinit(); + parser_deinit(); if (_ctx == ctx) _ctx = nullptr; + -- context_counter; } void reset(Context* ctx) @@ -413,7 +421,6 @@ AllocatorInfo get_cached_str_allocator( s32 str_length ) last = array_back( _ctx->StringArenas); } - return arena_allocator_info(last); } @@ -451,7 +458,6 @@ Code make_code() allocator = array_back( _ctx->CodePools); } - Code result = { rcast( AST*, alloc( pool_allocator_info(allocator), sizeof(AST) )) }; mem_set( rcast(void*, cast(AST*, result)), 0, sizeof(AST) ); return result; diff --git a/base/components/interface.hpp b/base/components/interface.hpp index 4aacd76..afe429b 100644 --- a/base/components/interface.hpp +++ b/base/components/interface.hpp @@ -33,6 +33,7 @@ struct LogEntry typedef void LoggerCallback(LogEntry entry); #endif + // Note(Ed): This is subject to heavily change // with upcoming changes to the library's fallback (default) allocations strategy; // and major changes to lexer/parser context usage. @@ -64,6 +65,9 @@ struct Context u32 InitSize_LexArena; u32 SizePer_StringArena; +// TODO(Ed): Symbol Table + // Keep track of all resolved symbols (naemspaced identifiers) + // Parser // Used by the lexer to persistently treat all these identifiers as preprocessor defines. @@ -72,6 +76,7 @@ struct Context Array(StrCached) PreprocessorDefines; // Backend + // The fallback allocator is utilized if any fo the three above allocators is not specified by the user. u32 InitSize_Fallback_Allocator_Bucket_Size; Array(Arena) Fallback_AllocatorBuckets; @@ -81,12 +86,19 @@ struct Context Array(Pool) CodePools; Array(Arena) StringArenas; - Arena LexArena; - StringTable StrCache; + + // TODO(Ed): This needs to be just handled by a parser context + + Arena LexArena; + StringTable Lexer_defines; + Array(Token) Lexer_Tokens; + + // TODO(Ed): Active parse context vs a parse result need to be separated conceptually + ParseContext parser; }; -// Initialize the library. +// Initialize the library. There first ctx initialized must exist for lifetime of other contextes that come after as its the one that void init(Context* ctx); // Currently manually free's the arenas, code for checking for leaks. @@ -283,7 +295,6 @@ CodeBody def_union_body ( s32 num, Code* codes ); // TODO(Ed) : Implmeent the new parser API design. #if 0 -GEN_NS_PARSER_BEGIN struct StackNode { StackNode* Prev; @@ -299,7 +310,6 @@ struct Error StrBuilder message; StackNode* context_stack; }; -GEN_NS_PARSER_END struct ParseInfo { diff --git a/base/components/interface.parsing.cpp b/base/components/interface.parsing.cpp index 76dd9eb..d96e0d6 100644 --- a/base/components/interface.parsing.cpp +++ b/base/components/interface.parsing.cpp @@ -10,23 +10,21 @@ CodeClass parse_class( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; push_scope(); CodeClass result = (CodeClass) parse_class_struct( Tok_Decl_Class, parser_not_inplace_def ); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } CodeConstructor parse_constructor( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); @@ -59,8 +57,8 @@ CodeConstructor parse_constructor( Str def ) break; default : - log_failure( "Invalid specifier %s for variable\n%S", spec_to_str( spec ), parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Invalid specifier %s for variable\n%S", spec_to_str( spec ), parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -79,14 +77,13 @@ CodeConstructor parse_constructor( Str def ) // ... } - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; CodeConstructor result = parser_parse_constructor( specifiers ); return result; } CodeDestructor parse_destructor( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); @@ -96,225 +93,209 @@ CodeDestructor parse_destructor( Str def ) // TODO(Ed): Destructors can have prefix attributes // TODO(Ed): Destructors can have virtual - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; CodeDestructor result = parser_parse_destructor(NullCode); return result; } CodeEnum parse_enum( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) { - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return InvalidCode; } - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_enum( parser_not_inplace_def); } CodeBody parse_export_body( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_export_body(); } CodeExtern parse_extern_link( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_extern_link(); } CodeFriend parse_friend( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_friend(); } CodeFn parse_function( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return (CodeFn) parser_parse_function(); } CodeBody parse_global_body( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; push_scope(); CodeBody result = parse_global_nspace( CT_Global_Body ); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } CodeNS parse_namespace( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_namespace(); } CodeOperator parse_operator( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return (CodeOperator) parser_parse_operator(); } CodeOpCast parse_operator_cast( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_operator_cast(NullCode); } CodeStruct parse_struct( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; push_scope(); CodeStruct result = (CodeStruct) parse_class_struct( Tok_Decl_Struct, parser_not_inplace_def ); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } CodeTemplate parse_template( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_template(); } CodeTypename parse_type( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_type( parser_not_from_template, nullptr); } CodeTypedef parse_typedef( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_typedef(); } CodeUnion parse_union( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_union( parser_not_inplace_def); } CodeUsing parse_using( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_using(); } CodeVar parse_variable( Str def ) { - GEN_USING_NS_PARSER; check_parse_args( def ); TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return InvalidCode; - parser_ctx.Tokens = toks; + _ctx->parser.Tokens = toks; return parser_parse_variable(); } diff --git a/base/components/interface.upfront.cpp b/base/components/interface.upfront.cpp index 4d0c487..a8c720f 100644 --- a/base/components/interface.upfront.cpp +++ b/base/components/interface.upfront.cpp @@ -1352,15 +1352,12 @@ CodeBody def_class_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast(Code, pod); - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_class_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_CLASS_UNALLOWED_TYPES: @@ -1370,7 +1367,6 @@ CodeBody def_class_body( s32 num, ... ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1386,18 +1382,14 @@ CodeBody def_class_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Function_Body; - do { Code entry = *codes; codes++; - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_class_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_CLASS_UNALLOWED_TYPES: @@ -1407,7 +1399,6 @@ CodeBody def_class_body( s32 num, Code* codes ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1429,19 +1420,14 @@ CodeBody def_enum_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast(Code, pod); - - if ( ! entry ) - { + if ( ! entry ) { log_failure("gen::def_enum_body: Provided a null entry"); return InvalidCode; } - - if ( entry->Type != CT_Untyped && entry->Type != CT_Comment ) - { + if ( entry->Type != CT_Untyped && entry->Type != CT_Comment ) { log_failure("gen::def_enum_body: Entry type is not allowed - %s. Must be of untyped or comment type.", code_debug_str(entry) ); return InvalidCode; } - body_append(result, entry ); } while ( num--, num > 0 ); @@ -1457,23 +1443,17 @@ CodeBody def_enum_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Enum_Body; - do { Code entry = *codes; - - if ( ! entry ) - { + if ( ! entry ) { log_failure("gen::def_enum_body: Provided a null entry"); return InvalidCode; } - - if ( entry->Type != CT_Untyped && entry->Type != CT_Comment ) - { + if ( entry->Type != CT_Untyped && entry->Type != CT_Comment ) { log_failure("gen::def_enum_body: Entry type is not allowed: %s", code_debug_str(entry) ); return InvalidCode; } - body_append(result, entry ); } while ( codes++, num--, num > 0 ); @@ -1495,13 +1475,11 @@ CodeBody def_export_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast(Code, pod); - - if (!entry) + if ( ! entry) { log_failure("gen::" "def_export_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_EXPORT_UNALLOWED_TYPES: @@ -1511,7 +1489,6 @@ CodeBody def_export_body( s32 num, ... ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1527,18 +1504,14 @@ CodeBody def_export_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Export_Body; - do { Code entry = *codes; codes++; - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_export_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_EXPORT_UNALLOWED_TYPES: @@ -1548,7 +1521,6 @@ CodeBody def_export_body( s32 num, Code* codes ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1570,13 +1542,10 @@ CodeBody def_extern_link_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast(Code, pod); - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_extern_linkage_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES: @@ -1586,7 +1555,6 @@ CodeBody def_extern_link_body( s32 num, ... ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1602,18 +1570,15 @@ CodeBody def_extern_link_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Extern_Linkage_Body; - do { Code entry = *codes; codes++; - if (!entry) { log_failure("gen::" "def_extern_linkage_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES: @@ -1623,7 +1588,6 @@ CodeBody def_extern_link_body( s32 num, Code* codes ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1645,16 +1609,12 @@ CodeBody def_function_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast(Code, pod); - - if (!entry) - { + if ( ! entry) { log_failure("gen::" stringize(def_function_body) ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { - GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES: log_failure("gen::" stringize(def_function_body) ": Entry type is not allowed: %s", code_debug_str(entry)); return InvalidCode; @@ -1662,7 +1622,6 @@ CodeBody def_function_body( s32 num, ... ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1678,18 +1637,14 @@ CodeBody def_function_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Function_Body; - do { Code entry = *codes; codes++; - - if (!entry) - { + if (!entry) { log_failure("gen::" "def_function_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES: @@ -1720,13 +1675,10 @@ CodeBody def_global_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast(Code, pod); - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_global_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { case CT_Global_Body: @@ -1741,7 +1693,6 @@ CodeBody def_global_body( s32 num, ... ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1757,18 +1708,14 @@ CodeBody def_global_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Global_Body; - do { Code entry = *codes; codes++; - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_global_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { case CT_Global_Body: @@ -1804,13 +1751,10 @@ CodeBody def_namespace_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast(Code, pod); - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_namespace_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES: @@ -1820,7 +1764,6 @@ CodeBody def_namespace_body( s32 num, ... ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1836,18 +1779,14 @@ CodeBody def_namespace_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Global_Body; - do { Code entry = *codes; codes++; - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_namespace_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES: @@ -1856,7 +1795,6 @@ CodeBody def_namespace_body( s32 num, Code* codes ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -1875,26 +1813,20 @@ CodeParams def_params( s32 num, ... ) CodeParams param = pcast( CodeParams, pod ); null_check( def_params, param ); - - if ( param->Type != CT_Parameters ) - { + if ( param->Type != CT_Parameters ) { log_failure( "gen::def_params: param %d is not a Parameters", num - num + 1 ); return InvalidCode; } CodeParams result = (CodeParams) code_duplicate(param); - while ( -- num ) { pod = va_arg(va, Code_POD); param = pcast( CodeParams, pod ); - - if ( param->Type != CT_Parameters ) - { + if ( param->Type != CT_Parameters ) { log_failure( "gen::def_params: param %d is not a Parameters", num - num + 1 ); return InvalidCode; } - params_append(result, param ); } va_end(va); @@ -1907,18 +1839,14 @@ CodeParams def_params( s32 num, CodeParams* codes ) def_body_code_array_start( def_params ); # define check_current(current) \ - if ( current == nullptr ) \ - { \ + if ( current == nullptr ) { \ log_failure("gen::def_params: Provide a null code in codes array"); \ return InvalidCode; \ } \ - \ - if (current->Type != CT_Parameters ) \ - { \ + if (current->Type != CT_Parameters ) { \ log_failure("gen::def_params: Code in coes array is not of paramter type - %s", code_debug_str(current) ); \ return InvalidCode; \ } - CodeParams current = (CodeParams)code_duplicate(* codes); check_current(current); @@ -1927,9 +1855,7 @@ CodeParams def_params( s32 num, CodeParams* codes ) result->Name = current->Name; result->Type = current->Type; result->ValueType = current->ValueType; - - while( codes++, current = * codes, num--, num > 0 ) - { + while( codes++, current = * codes, num--, num > 0 ) { check_current(current); params_append(result, current ); } @@ -1940,28 +1866,22 @@ CodeParams def_params( s32 num, CodeParams* codes ) CodeSpecifiers def_specifiers( s32 num, ... ) { - if ( num <= 0 ) - { + if ( num <= 0 ) { log_failure("gen::def_specifiers: num cannot be zero or less"); return InvalidCode; } - - if ( num > AST_ArrSpecs_Cap ) - { + if ( num > AST_ArrSpecs_Cap ) { log_failure("gen::def_specifiers: num of speciifers to define AST larger than AST specicifier capacity - %d", num); return InvalidCode; } - CodeSpecifiers result = (CodeSpecifiers) make_code(); result->Type = CT_Specifiers; va_list va; va_start(va, num); - do - { + do { Specifier type = (Specifier)va_arg(va, int); - specifiers_append(result, type ); } while ( --num, num ); @@ -1972,25 +1892,20 @@ CodeSpecifiers def_specifiers( s32 num, ... ) CodeSpecifiers def_specifiers( s32 num, Specifier* specs ) { - if ( num <= 0 ) - { + if ( num <= 0 ) { log_failure("gen::def_specifiers: num cannot be zero or less"); return InvalidCode; } - - if ( num > AST_ArrSpecs_Cap ) - { + if ( num > AST_ArrSpecs_Cap ) { log_failure("gen::def_specifiers: num of speciifers to define AST larger than AST specicifier capacity - %d", num); return InvalidCode; } - CodeSpecifiers result = (CodeSpecifiers) make_code(); result->Type = CT_Specifiers; s32 idx = 0; - do - { + do { specifiers_append(result, specs[idx] ); idx++; } @@ -2013,13 +1928,10 @@ CodeBody def_struct_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast(Code, pod); - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_struct_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_STRUCT_UNALLOWED_TYPES: @@ -2029,7 +1941,6 @@ CodeBody def_struct_body( s32 num, ... ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -2045,18 +1956,14 @@ CodeBody def_struct_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Struct_Body; - do { Code entry = *codes; codes++; - - if (!entry) - { + if ( ! entry) { log_failure("gen::" "def_struct_body" ": Provided an null entry"); return InvalidCode; } - switch (entry->Type) { GEN_AST_BODY_STRUCT_UNALLOWED_TYPES: @@ -2066,7 +1973,6 @@ CodeBody def_struct_body( s32 num, Code* codes ) default: break; } - body_append(result, entry); } while (num--, num > 0); @@ -2088,19 +1994,14 @@ CodeBody def_union_body( s32 num, ... ) { Code_POD pod = va_arg(va, Code_POD); Code entry = pcast( Code, pod ); - - if ( ! entry ) - { + if ( ! entry ) { log_failure("gen::def_union_body: Provided a null entry"); return InvalidCode; } - - if ( entry->Type != CT_Untyped && entry->Type != CT_Comment ) - { + if ( entry->Type != CT_Untyped && entry->Type != CT_Comment ) { log_failure("gen::def_union_body: Entry type is not allowed - %s. Must be of untyped or comment type.", code_debug_str(entry) ); return InvalidCode; } - body_append(result, entry ); } while ( num--, num > 0 ); @@ -2116,23 +2017,17 @@ CodeBody def_union_body( s32 num, Code* codes ) CodeBody result = (CodeBody) make_code(); result->Type = CT_Union_Body; - do { Code entry = *codes; - - if ( ! entry ) - { + if ( ! entry ) { log_failure("gen::def_union_body: Provided a null entry"); return InvalidCode; } - - if ( entry->Type != CT_Untyped && entry->Type != CT_Comment ) - { + if ( entry->Type != CT_Untyped && entry->Type != CT_Comment ) { log_failure("gen::def_union_body: Entry type is not allowed: %s", code_debug_str(entry) ); return InvalidCode; } - body_append(result, entry ); } while ( codes++, num--, num > 0 ); diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index c6df466..a5c8eb6 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -4,92 +4,6 @@ #include "gen/etoktype.cpp" #endif -GEN_NS_PARSER_BEGIN - -enum TokFlags : u32 -{ - TF_Operator = bit(0), - TF_Assign = bit(1), - TF_Preprocess = bit(2), - TF_Preprocess_Cond = bit(3), - TF_Attribute = bit(6), - TF_AccessOperator = bit( 7 ), - TF_AccessSpecifier = bit( 8 ), - TF_Specifier = bit( 9 ), - TF_EndDefinition = bit( 10 ), // Either ; or } - TF_Formatting = bit( 11 ), - TF_Literal = bit( 12 ), - - TF_Null = 0, - TF_UnderlyingType = GEN_U32_MAX, -}; - -struct Token -{ - Str Text; - TokType Type; - s32 Line; - s32 Column; - u32 Flags; -}; - -constexpr Token NullToken { nullptr, 0, Tok_Invalid, false, 0, TF_Null }; - -forceinline -AccessSpec tok_to_access_specifier(Token tok) { - return scast(AccessSpec, tok.Type); -} - -forceinline -Str tok_to_str(Token tok) { - return tok.Text; -} - -forceinline -bool tok_is_valid( Token tok ) { - return tok.Text.Ptr && tok.Text.Len && tok.Type != Tok_Invalid; -} - -forceinline -bool tok_is_access_operator(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_AccessOperator ); -} - -forceinline -bool tok_is_access_specifier(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_AccessSpecifier ); -} - -forceinline -bool tok_is_attribute(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Attribute ); -} - -forceinline -bool tok_is_operator(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Operator ); -} - -forceinline -bool tok_is_preprocessor(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Preprocess ); -} - -forceinline -bool tok_is_preprocess_cond(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Preprocess_Cond ); -} - -forceinline -bool tok_is_specifier(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_Specifier ); -} - -forceinline -bool tok_is_end_definition(Token tok) { - return bitfield_is_equal( u32, tok.Flags, TF_EndDefinition ); -} - StrBuilder tok_to_strbuilder(Token tok) { StrBuilder result = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(4) ); @@ -103,11 +17,6 @@ StrBuilder tok_to_strbuilder(Token tok) return result; } -struct TokArray -{ - Array(Token) Arr; - s32 Idx; -}; bool lex__eat( TokArray* self, TokType type ); @@ -160,27 +69,12 @@ Token* lex_next(TokArray self, bool skip_formatting) return & self.Arr[idx + 1]; } -global FixedArena_256KB Lexer_defines_map_arena; -global StringTable Lexer_defines; -global Array(Token) Lexer_Tokens; - enum { Lex_Continue, Lex_ReturnNull, }; -struct LexContext -{ - Str content; - s32 left; - char const* scanner; - s32 line; - s32 column; - StringTable defines; - Token token; -}; - forceinline void lexer_move_forward( LexContext* ctx ) { @@ -224,7 +118,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) { char const* hash = ctx->scanner; Token hash_tok = { { hash, 1 }, Tok_Preprocess_Hash, ctx->line, ctx->column, TF_Preprocess }; - array_append( Lexer_Tokens, hash_tok ); + array_append( _ctx->Lexer_Tokens, hash_tok ); move_forward(); skip_whitespace(); @@ -300,14 +194,14 @@ s32 lex_preprocessor_directive( LexContext* ctx ) ctx->token.Text.Len = ctx->token.Text.Len + ctx->token.Text.Ptr - hash; ctx->token.Text.Ptr = hash; - array_append( Lexer_Tokens, ctx->token ); + array_append( _ctx->Lexer_Tokens, ctx->token ); return Lex_Continue; // Skip found token, its all handled here. } if ( ctx->token.Type == Tok_Preprocess_Else || ctx->token.Type == Tok_Preprocess_EndIf ) { ctx->token.Flags |= TF_Preprocess_Cond; - array_append( Lexer_Tokens, ctx->token ); + array_append( _ctx->Lexer_Tokens, ctx->token ); end_line(); return Lex_Continue; } @@ -316,7 +210,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) ctx->token.Flags |= TF_Preprocess_Cond; } - array_append( Lexer_Tokens, ctx->token ); + array_append( _ctx->Lexer_Tokens, ctx->token ); skip_whitespace(); @@ -340,7 +234,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) name.Text.Len++; } - array_append( Lexer_Tokens, name ); + array_append( _ctx->Lexer_Tokens, name ); u64 key = crc32( name.Text.Ptr, name.Text.Len ); hashtable_set(ctx->defines, key, tok_to_str(name) ); @@ -386,7 +280,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) move_forward(); } - array_append( Lexer_Tokens, preprocess_content ); + array_append( _ctx->Lexer_Tokens, preprocess_content ); return Lex_Continue; // Skip found token, its all handled here. } @@ -449,7 +343,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) preprocess_content.Text.Len++; } - array_append( Lexer_Tokens, preprocess_content ); + array_append( _ctx->Lexer_Tokens, preprocess_content ); return Lex_Continue; // Skip found token, its all handled here. } @@ -458,7 +352,7 @@ void lex_found_token( LexContext* ctx ) { if ( ctx->token.Type != Tok_Invalid ) { - array_append( Lexer_Tokens, ctx->token ); + array_append( _ctx->Lexer_Tokens, ctx->token ); return; } @@ -485,7 +379,7 @@ void lex_found_token( LexContext* ctx ) } ctx->token.Type = type; - array_append( Lexer_Tokens, ctx->token ); + array_append( _ctx->Lexer_Tokens, ctx->token ); return; } @@ -495,7 +389,7 @@ void lex_found_token( LexContext* ctx ) { ctx->token.Type = type; ctx->token.Flags |= TF_Specifier; - array_append( Lexer_Tokens, ctx->token ); + array_append( _ctx->Lexer_Tokens, ctx->token ); return; } @@ -503,7 +397,7 @@ void lex_found_token( LexContext* ctx ) if ( type != Tok_Invalid ) { ctx->token.Type = type; - array_append( Lexer_Tokens, ctx->token ); + array_append( _ctx->Lexer_Tokens, ctx->token ); return; } @@ -557,7 +451,7 @@ void lex_found_token( LexContext* ctx ) ctx->token.Type = Tok_Identifier; } - array_append( Lexer_Tokens, ctx->token ); + array_append( _ctx->Lexer_Tokens, ctx->token ); } neverinline @@ -568,7 +462,7 @@ TokArray lex( Str content ) c.content = content; c.left = content.Len; c.scanner = content.Ptr; - c.defines = Lexer_defines; + c.defines = _ctx->Lexer_defines; char const* word = c.scanner; s32 word_length = 0; @@ -602,7 +496,7 @@ TokArray lex( Str content ) hashtable_set(c.defines, key, * entry ); } - array_clear(Lexer_Tokens); + array_clear(_ctx->Lexer_Tokens); while (c.left ) { @@ -635,7 +529,7 @@ TokArray lex( Str content ) c.token.Type = Tok_NewLine; c.token.Text.Len++; - array_append( Lexer_Tokens, c.token ); + array_append( _ctx->Lexer_Tokens, c.token ); continue; } } @@ -674,7 +568,7 @@ TokArray lex( Str content ) c.token.Text.Len++; move_forward(); - array_append( Lexer_Tokens, c.token ); + array_append( _ctx->Lexer_Tokens, c.token ); } } continue; @@ -1131,7 +1025,7 @@ TokArray lex( Str content ) move_forward(); c.token.Text.Len++; } - array_append( Lexer_Tokens, c.token ); + array_append( _ctx->Lexer_Tokens, c.token ); continue; } else if ( (* ctx->scanner) == '*' ) @@ -1167,7 +1061,7 @@ TokArray lex( Str content ) move_forward(); c.token.Text.Len++; } - array_append( Lexer_Tokens, c.token ); + array_append( _ctx->Lexer_Tokens, c.token ); // end_line(); continue; } @@ -1260,14 +1154,14 @@ TokArray lex( Str content ) } else { - s32 start = max( 0, array_num(Lexer_Tokens) - 100 ); + s32 start = max( 0, array_num(_ctx->Lexer_Tokens) - 100 ); log_fmt("\n%d\n", start); - for ( s32 idx = start; idx < array_num(Lexer_Tokens); idx++ ) + for ( s32 idx = start; idx < array_num(_ctx->Lexer_Tokens); idx++ ) { log_fmt( "Token %d Type: %s : %.*s\n" , idx - , toktype_to_str( Lexer_Tokens[ idx ].Type ).Ptr - , Lexer_Tokens[ idx ].Text.Len, Lexer_Tokens[ idx ].Text.Ptr + , toktype_to_str( _ctx->Lexer_Tokens[ idx ].Type ).Ptr + , _ctx->Lexer_Tokens[ idx ].Text.Len, _ctx->Lexer_Tokens[ idx ].Text.Ptr ); } @@ -1284,7 +1178,7 @@ TokArray lex( Str content ) FoundToken: { lex_found_token( ctx ); - TokType last_type = array_back(Lexer_Tokens)->Type; + TokType last_type = array_back(_ctx->Lexer_Tokens)->Type; if ( last_type == Tok_Preprocess_Macro ) { Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; @@ -1301,14 +1195,14 @@ TokArray lex( Str content ) c.token.Text.Len++; move_forward(); - array_append( Lexer_Tokens, c.token ); + array_append( _ctx->Lexer_Tokens, c.token ); continue; } } } } - if ( array_num(Lexer_Tokens) == 0 ) + if ( array_num(_ctx->Lexer_Tokens) == 0 ) { log_failure( "Failed to lex any tokens" ); { @@ -1317,13 +1211,11 @@ TokArray lex( Str content ) } } - hashtable_clear(Lexer_defines); + hashtable_clear(_ctx->Lexer_defines); // defines_map_arena.free(); - TokArray result = { Lexer_Tokens, 0 }; + TokArray result = { _ctx->Lexer_Tokens, 0 }; return result; } #undef move_forward #undef skip_whitespace #undef end_line - -GEN_NS_PARSER_END diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 0332d70..820fea0 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -5,42 +5,25 @@ #include "lexer.cpp" #endif -GEN_NS_PARSER_BEGIN - // TODO(Ed) : Rename ETok_Capture_Start, ETok_Capture_End to Open_Parenthesis adn Close_Parenthesis constexpr bool lex_dont_skip_formatting = false; constexpr bool lex_skip_formatting = true; -struct StackNode -{ - StackNode* Prev; - - Token Start; - Token Name; // The name of the AST node (if parsed) - Str ProcName; // The name of the procedure -}; - -struct ParseContext -{ - TokArray Tokens; - StackNode* Scope; -}; - void parser_push( ParseContext* ctx, StackNode* node ) { node->Prev = ctx->Scope; ctx->Scope = node; #if 0 && GEN_BUILD_DEBUG - log_fmt("\tEntering parser_ctx: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); + log_fmt("\tEntering _ctx->parser: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); #endif } void parser_pop(ParseContext* ctx) { #if 0 && GEN_BUILD_DEBUG - log_fmt("\tPopping parser_ctx: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); + log_fmt("\tPopping _ctx->parser: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); #endif ctx->Scope = ctx->Scope->Prev; } @@ -49,7 +32,7 @@ StrBuilder parser_to_strbuilder(ParseContext ctx) { StrBuilder result = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(4) ); - Token scope_start = ctx.Scope->Start; + Token scope_start = * ctx.Scope->Start; Token last_valid = ctx.Tokens.Idx >= array_num(ctx.Tokens.Arr) ? ctx.Tokens.Arr[array_num(ctx.Tokens.Arr) -1] : (* lex_current(& ctx.Tokens, true)); sptr length = scope_start.Text.Len; @@ -80,9 +63,9 @@ StrBuilder parser_to_strbuilder(ParseContext ctx) s32 level = 0; do { - if ( tok_is_valid(curr_scope->Name) ) + if ( curr_scope->Name.Ptr ) { - strbuilder_append_fmt(& result, "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Text.Len, curr_scope->Name.Text.Ptr ); + strbuilder_append_fmt(& result, "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Len, curr_scope->Name.Ptr ); } else { @@ -96,13 +79,11 @@ StrBuilder parser_to_strbuilder(ParseContext ctx) return result; } -global ParseContext parser_ctx; - bool lex__eat(TokArray* self, TokType type ) { if ( array_num(self->Arr) - self->Idx <= 0 ) { - log_failure( "No tokens left.\n%s", parser_to_strbuilder(parser_ctx) ); + log_failure( "No tokens left.\n%s", parser_to_strbuilder(_ctx->parser) ); return false; } @@ -122,7 +103,7 @@ bool lex__eat(TokArray* self, TokType type ) , at_idx.Text.Len, at_idx.Text.Ptr , tok.Line , tok.Column - , parser_to_strbuilder(parser_ctx) + , parser_to_strbuilder(_ctx->parser) ); GEN_DEBUG_TRAP(); return false; @@ -139,19 +120,18 @@ bool lex__eat(TokArray* self, TokType type ) internal void parser_init() { - Lexer_Tokens = array_init_reserve(Token, arena_allocator_info( & _ctx->LexArena) + _ctx->Lexer_Tokens = array_init_reserve(Token, arena_allocator_info( & _ctx->LexArena) , ( _ctx->InitSize_LexArena - sizeof( ArrayHeader ) ) / sizeof(Token) ); - fixed_arena_init(& Lexer_defines_map_arena); - Lexer_defines = hashtable_init_reserve(Str, fixed_arena_allocator_info( & Lexer_defines_map_arena), 256 ); + _ctx->Lexer_defines = hashtable_init_reserve(Str, _ctx->Allocator_DyanmicContainers, 256 ); } internal void parser_deinit() { Array(Token) null_array = { nullptr }; - Lexer_Tokens = null_array; + _ctx->Lexer_Tokens = null_array; } #pragma region Helper Macros @@ -162,26 +142,26 @@ bool _check_parse_args( Str def, char const* func_name ) if ( def.Len <= 0 ) { log_failure( c_str_fmt_buf("gen::%s: length must greater than 0", func_name) ); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return false; } if ( def.Ptr == nullptr ) { log_failure( c_str_fmt_buf("gen::%s: def was null", func_name) ); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return false; } return true; } -# define currtok_noskip (* lex_current( & parser_ctx.Tokens, lex_dont_skip_formatting )) -# define currtok (* lex_current( & parser_ctx.Tokens, lex_skip_formatting )) -# define peektok (* lex_peek(parser_ctx.Tokens, lex_skip_formatting)) -# define prevtok (* lex_previous( parser_ctx.Tokens, lex_dont_skip_formatting)) -# define nexttok (* lex_next( parser_ctx.Tokens, lex_skip_formatting )) -# define nexttok_noskip (* lex_next( parser_ctx.Tokens, lex_dont_skip_formatting)) -# define eat( Type_ ) lex__eat( & parser_ctx.Tokens, Type_ ) -# define left ( array_num(parser_ctx.Tokens.Arr) - parser_ctx.Tokens.Idx ) +# define currtok_noskip (* lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting )) +# define currtok (* lex_current( & _ctx->parser.Tokens, lex_skip_formatting )) +# define peektok (* lex_peek(_ctx->parser.Tokens, lex_skip_formatting)) +# define prevtok (* lex_previous( _ctx->parser.Tokens, lex_dont_skip_formatting)) +# define nexttok (* lex_next( _ctx->parser.Tokens, lex_skip_formatting )) +# define nexttok_noskip (* lex_next( _ctx->parser.Tokens, lex_dont_skip_formatting)) +# define eat( Type_ ) lex__eat( & _ctx->parser.Tokens, Type_ ) +# define left ( array_num(_ctx->parser.Tokens.Arr) - _ctx->parser.Tokens.Idx ) #if GEN_COMPILER_CPP # define def_assign( ... ) { __VA_ARGS__ } @@ -199,9 +179,10 @@ bool _check_parse_args( Str def, char const* func_name ) # define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ ) # define check( Type_ ) ( left && currtok.Type == Type_ ) -# define push_scope() \ - GEN_NS_PARSER StackNode scope = { nullptr, currtok_noskip, GEN_NS_PARSER NullToken, txt( __func__ ) }; \ - parser_push( & GEN_NS_PARSER parser_ctx, & scope ) +# define push_scope() \ + Str null_name = {}; \ + StackNode scope = { nullptr, lex_current( & _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ) }; \ + parser_push( & _ctx->parser, & scope ) #pragma endregion Helper Macros @@ -519,7 +500,7 @@ Code parse_array_decl() eat( Tok_Operator ); // [] - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return array_expr; } @@ -530,15 +511,15 @@ Code parse_array_decl() if ( left == 0 ) { - log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } if ( currtok.Type == Tok_BraceSquare_Close ) { - log_failure( "Error, empty array expression in definition\n%s", parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Error, empty array expression in definition\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -556,15 +537,15 @@ Code parse_array_decl() if ( left == 0 ) { - log_failure( "Error, unexpected end of array declaration, expected ]\n%s", parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Error, unexpected end of array declaration, expected ]\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } if ( currtok.Type != Tok_BraceSquare_Close ) { - log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -580,11 +561,11 @@ Code parse_array_decl() array_expr->Next = adjacent_arr_expr; } - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return array_expr; } - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return NullCode; } @@ -682,7 +663,7 @@ CodeAttributes parse_attributes() if ( len > 0 ) { Str attribute_txt = { start.Text.Ptr, len }; - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); StrBuilder name_stripped = parser_strip_formatting( attribute_txt, parser_strip_formatting_dont_preserve_newlines ); @@ -695,7 +676,7 @@ CodeAttributes parse_attributes() return ( CodeAttributes )result; } - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return NullCode; } @@ -704,11 +685,11 @@ Code parse_class_struct( TokType which, bool inplace_def ) { if ( which != Tok_Decl_Class && which != Tok_Decl_Struct ) { - log_failure( "Error, expected class or struct, not %s\n%s", toktype_to_str( which ), parser_to_strbuilder(parser_ctx) ); + log_failure( "Error, expected class or struct, not %s\n%s", toktype_to_str( which ), parser_to_strbuilder(_ctx->parser) ); return InvalidCode; } - Token name = { nullptr, 0, Tok_Invalid }; + Token name = NullToken; AccessSpec access = AccessSpec_Default; CodeTypename parent = { nullptr }; @@ -734,7 +715,7 @@ Code parse_class_struct( TokType which, bool inplace_def ) if ( check( Tok_Identifier ) ) { name = parse_identifier(nullptr); - parser_ctx.Scope->Name = name; + _ctx->parser.Scope->Name = name.Text; } // @@ -834,7 +815,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) bool expects_function = false; - // parser_ctx.Scope->Start = currtok_noskip; + // _ctx->parser.Scope->Start = currtok_noskip; if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); @@ -928,7 +909,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) case Tok_Operator: //if ( currtok.Text[0] != '~' ) //{ - // log_failure( "Operator token found in global body but not destructor unary negation\n%s", to_strbuilder(parser_ctx) ); + // log_failure( "Operator token found in global body but not destructor unary negation\n%s", to_strbuilder(_ctx->parser) ); // return InvalidCode; //} @@ -1042,8 +1023,8 @@ CodeBody parse_class_struct_body( TokType which, Token name ) break; default: - log_failure( "Invalid specifier %S for variable\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(parser_ctx)) ); - parser_pop(& parser_ctx); + log_failure( "Invalid specifier %S for variable\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -1128,8 +1109,8 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( member == Code_Invalid ) { - log_failure( "Failed to parse member\n%s", parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Failed to parse member\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } continue; @@ -1162,8 +1143,8 @@ CodeBody parse_class_struct_body( TokType which, Token name ) if ( member == Code_Invalid ) { - log_failure( "Failed to parse member\n%s", parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Failed to parse member\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -1172,7 +1153,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) eat( Tok_BraceCurly_Close ); // { } - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } @@ -1188,7 +1169,7 @@ CodeComment parse_comment() // result->Token = currtok_noskip; eat( Tok_Comment ); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } @@ -1199,7 +1180,7 @@ Code parse_complicated_definition( TokType which ) bool is_inplace = false; - TokArray tokens = parser_ctx.Tokens; + TokArray tokens = _ctx->parser.Tokens; s32 idx = tokens.Idx; s32 level = 0; @@ -1220,7 +1201,7 @@ Code parse_complicated_definition( TokType which ) // Its a forward declaration only Code result = parse_forward_or_definition( which, is_inplace ); // ; - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } @@ -1244,12 +1225,12 @@ Code parse_complicated_definition( TokType which ) Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); // , or Name> ... - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } if ( tok.Type == Tok_Identifier ) @@ -1281,7 +1262,7 @@ Code parse_complicated_definition( TokType which ) // : ; ok_to_parse = true; Code result = cast(Code, parser_parse_enum( ! parser_inplace_def)); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } else if ( is_indirection ) @@ -1293,14 +1274,14 @@ Code parse_complicated_definition( TokType which ) if ( ! ok_to_parse ) { - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); // , or Name> ... - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } else if ( tok.Type >= Tok_Type_Unsigned && tok.Type <= Tok_Type_MS_W64 ) @@ -1312,8 +1293,8 @@ Code parse_complicated_definition( TokType which ) && ( tokens.Arr[idx - 4].Type != which)) ) { - log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -1321,7 +1302,7 @@ Code parse_complicated_definition( TokType which ) // : ; // : ; Code result = cast(Code, parser_parse_enum( ! parser_inplace_def)); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } else if ( tok.Type == Tok_BraceCurly_Close ) @@ -1329,7 +1310,7 @@ Code parse_complicated_definition( TokType which ) // Its a definition Code result = parse_forward_or_definition( which, is_inplace ); // { ... }; - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } else if ( tok.Type == Tok_BraceSquare_Close ) @@ -1337,13 +1318,13 @@ Code parse_complicated_definition( TokType which ) // Its an array definition Code result = parse_operator_function_or_variable( false, NullCode, NullCode ); // [ ... ]; - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } else { - log_failure( "Unsupported or bad member definition after %s declaration\n%SB", toktype_to_str(which).Ptr, parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Unsupported or bad member definition after %s declaration\n%SB", toktype_to_str(which).Ptr, parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } } @@ -1361,20 +1342,20 @@ CodeDefine parse_define() if ( ! check( Tok_Identifier ) ) { - log_failure( "Error, expected identifier after #define\n%s", parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Error, expected identifier after #define\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } - parser_ctx.Scope->Name = currtok; + _ctx->parser.Scope->Name = currtok.Text; define->Name = cache_str( tok_to_str(currtok) ); eat( Tok_Identifier ); // #define if ( ! check( Tok_Preprocess_Content )) { - log_failure( "Error, expected content after #define %s\n%s", define->Name, parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Error, expected content after #define %s\n%s", define->Name, parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -1384,7 +1365,7 @@ CodeDefine parse_define() eat( Tok_Preprocess_Content ); // #define - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return define; } @@ -1392,7 +1373,7 @@ CodeDefine parse_define() eat( Tok_Preprocess_Content ); // #define - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return define; } @@ -1408,8 +1389,8 @@ Code parse_assignment_expression() if ( currtok.Type == Tok_Statement_End && currtok.Type != Tok_Comma ) { - log_failure( "Expected expression after assignment operator\n%s", parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Expected expression after assignment operator\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -1460,7 +1441,7 @@ Code parse_forward_or_definition( TokType which, bool is_inplace ) default: log_failure( "Error, wrong token type given to parse_complicated_definition " "(only supports class, enum, struct, union) \n%s" - , parser_to_strbuilder(parser_ctx) ); + , parser_to_strbuilder(_ctx->parser) ); return InvalidCode; } @@ -1502,7 +1483,7 @@ CodeFn parse_function_after_name( body = cast(CodeBody, parse_function_body()); if ( cast(Code, body) == Code_Invalid ) { - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return InvalidCode; } // ( ) { } @@ -1551,8 +1532,8 @@ CodeFn parse_function_after_name( default: { - log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", code_debug_str(body), parser_to_strbuilder(parser_ctx)); - parser_pop(& parser_ctx); + log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", code_debug_str(body), parser_to_strbuilder(_ctx->parser)); + parser_pop(& _ctx->parser); return InvalidCode; } } @@ -1579,7 +1560,7 @@ CodeFn parse_function_after_name( if ( inline_cmt ) result->InlineCmt = inline_cmt; - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } @@ -1621,7 +1602,7 @@ Code parse_function_body() eat( Tok_BraceCurly_Close ); - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return cast(Code, result); } @@ -1649,7 +1630,7 @@ CodeBody parse_global_nspace( CodeType which ) bool expects_function = false; - // parser_ctx.Scope->Start = currtok_noskip; + // _ctx->parser.Scope->Start = currtok_noskip; if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); @@ -1660,8 +1641,8 @@ CodeBody parse_global_nspace( CodeType which ) { case Tok_Comma: { - log_failure("Dangling comma found: %SB\nContext:\n%SB", tok_to_strbuilder(currtok), parser_to_strbuilder(parser_ctx)); - parser_pop( & parser_ctx); + log_failure("Dangling comma found: %SB\nContext:\n%SB", tok_to_strbuilder(currtok), parser_to_strbuilder(_ctx->parser)); + parser_pop( & _ctx->parser); return InvalidCode; } break; @@ -1694,7 +1675,7 @@ CodeBody parse_global_nspace( CodeType which ) case Tok_Decl_Extern_Linkage: if ( which == CT_Extern_Linkage_Body ) - log_failure( "Nested extern linkage\n%s", parser_to_strbuilder(parser_ctx) ); + log_failure( "Nested extern linkage\n%s", parser_to_strbuilder(_ctx->parser) ); member = cast(Code, parser_parse_extern_link()); // extern "..." { ... } @@ -1787,7 +1768,7 @@ CodeBody parse_global_nspace( CodeType which ) case Tok_Module_Export: { if ( which == CT_Export_Body ) - log_failure( "Nested export declaration\n%s", parser_to_strbuilder(parser_ctx) ); + log_failure( "Nested export declaration\n%s", parser_to_strbuilder(_ctx->parser) ); member = cast(Code, parser_parse_export_body()); // export { ... } @@ -1857,8 +1838,8 @@ CodeBody parse_global_nspace( CodeType which ) default: Str spec_str = spec_to_str(spec); - log_failure( "Invalid specifier %S for variable\n%S", spec_str, strbuilder_to_str( parser_to_strbuilder(parser_ctx)) ); - parser_pop(& parser_ctx); + log_failure( "Invalid specifier %S for variable\n%S", spec_str, strbuilder_to_str( parser_to_strbuilder(_ctx->parser)) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -1899,16 +1880,16 @@ CodeBody parse_global_nspace( CodeType which ) } bool found_operator_cast_outside_class_implmentation = false; - s32 idx = parser_ctx.Tokens.Idx; + s32 idx = _ctx->parser.Tokens.Idx; - for ( ; idx < array_num(parser_ctx.Tokens.Arr); idx++ ) + for ( ; idx < array_num(_ctx->parser.Tokens.Arr); idx++ ) { - Token tok = parser_ctx.Tokens.Arr[ idx ]; + Token tok = _ctx->parser.Tokens.Arr[ idx ]; if ( tok.Type == Tok_Identifier ) { idx++; - tok = parser_ctx.Tokens.Arr[ idx ]; + tok = _ctx->parser.Tokens.Arr[ idx ]; if ( tok.Type == Tok_Access_StaticSymbol ) continue; @@ -1940,8 +1921,8 @@ CodeBody parse_global_nspace( CodeType which ) if ( member == Code_Invalid ) { - log_failure( "Failed to parse member\n%s", parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Failed to parse member\n%s", parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } goto Member_Resolved_To_Lone_Macro; @@ -1959,8 +1940,8 @@ CodeBody parse_global_nspace( CodeType which ) Member_Resolved_To_Lone_Macro: if ( member == Code_Invalid ) { - log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(currtok_noskip), parser_to_strbuilder(parser_ctx) ); - parser_pop(& parser_ctx); + log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(currtok_noskip), parser_to_strbuilder(_ctx->parser) ); + parser_pop(& _ctx->parser); return InvalidCode; } @@ -1972,7 +1953,7 @@ CodeBody parse_global_nspace( CodeType which ) eat( Tok_BraceCurly_Close ); // { } - parser_pop(& parser_ctx); + parser_pop(& _ctx->parser); return result; } @@ -1993,7 +1974,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) TODO(Ed): We could fix this by attempting to parse a type, but we would have to have a way to have it soft fail and rollback. */ - TokArray tokens = parser_ctx.Tokens; + TokArray tokens = _ctx->parser.Tokens; s32 idx = tokens.Idx; Token nav = tokens.Arr[ idx ]; @@ -2113,7 +2094,7 @@ Token parse_identifier( bool* possible_member_function ) push_scope(); Token name = currtok; - parser_ctx.Scope->Name = name; + _ctx->parser.Scope->Name = name.Text; eat( Tok_Identifier ); // @@ -2125,26 +2106,26 @@ Token parse_identifier( bool* possible_member_function ) eat( Tok_Access_StaticSymbol ); //