From 772db608be1180280b04bdfa699a3f45771aba74 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 21 Nov 2023 20:09:14 -0500 Subject: [PATCH] Added support for predefining preprocessor defines before parsing strings of code. This prevents issues for preprocessor defines not getting treated properly for specific circumstances (such as macro wrappers for specifiers). --- project/components/header_end.hpp | 5 +++++ project/components/interface.cpp | 27 ++++++++++++++++----------- project/components/lexer.cpp | 29 +++++++++++++++++++++++------ project/components/static_data.cpp | 2 ++ project/components/types.hpp | 1 + project/dependencies/containers.hpp | 11 ++++++++++- 6 files changed, 57 insertions(+), 18 deletions(-) diff --git a/project/components/header_end.hpp b/project/components/header_end.hpp index d00e31d..238da02 100644 --- a/project/components/header_end.hpp +++ b/project/components/header_end.hpp @@ -155,6 +155,11 @@ extern CodeType t_typename; #pragma endregion Macros +// Used by the lexer to persistently treat all these identifiers as preprocessor defines. +// Populate with strings via gen::get_cached_string. +// Functional defines must have format: id( ;at minimum to indicate that the define is only valid with arguments. +extern Array< StringCached > PreprocessorDefines; + #ifdef GEN_EXPOSE_BACKEND // Global allocator used for data with process lifetime. diff --git a/project/components/interface.cpp b/project/components/interface.cpp index ae87295..48de1b0 100644 --- a/project/components/interface.cpp +++ b/project/components/interface.cpp @@ -289,6 +289,9 @@ void init() GEN_FATAL( "gen::init: Failed to initialize the StringCache"); } + // Preprocessor Defines + PreprocessorDefines = Array::init_reserve( GlobalAllocator, kilobytes(1) ); + define_constants(); parser::init(); } @@ -332,6 +335,8 @@ void deinit() } while ( left--, left ); + PreprocessorDefines.free(); + Global_AllocatorBuckets.free(); parser::deinit(); } @@ -418,18 +423,18 @@ Code make_code() } Code result { rcast( AST*, alloc( * allocator, sizeof(AST) )) }; - // mem_set( result.ast, 0, sizeof(AST) ); - result->Type = ECode::Invalid; + mem_set( result.ast, 0, sizeof(AST) ); + // result->Type = ECode::Invalid; - result->Content = { nullptr }; - result->Prev = { nullptr }; - result->Next = { nullptr }; - result->Token = nullptr; - result->Parent = { nullptr }; - result->Name = { nullptr }; - result->Type = ECode::Invalid; - result->ModuleFlags = ModuleFlag::Invalid; - result->NumEntries = 0; + // result->Content = { nullptr }; + // result->Prev = { nullptr }; + // result->Next = { nullptr }; + // result->Token = nullptr; + // result->Parent = { nullptr }; + // result->Name = { nullptr }; + // result->Type = ECode::Invalid; + // result->ModuleFlags = ModuleFlag::Invalid; + // result->NumEntries = 0; return result; } diff --git a/project/components/lexer.cpp b/project/components/lexer.cpp index afc0a59..cbe5d87 100644 --- a/project/components/lexer.cpp +++ b/project/components/lexer.cpp @@ -12,7 +12,6 @@ enum TokFlags : u32 TF_Assign = bit(1), TF_Preprocess = bit(2), TF_Preprocess_Cond = bit(3), - TF_Comment = bit(4), TF_Attribute = bit(6), TF_AccessSpecifier = bit(7), TF_Specifier = bit(8), @@ -154,7 +153,9 @@ struct TokArray } }; -global Array Tokens; +global Arena_64KB defines_map_arena; +global HashTable defines; +global Array Tokens; #define current ( * scanner ) @@ -534,6 +535,7 @@ void lex_found_token( StrC& content Tokens.append( token ); } + neverinline TokArray lex( StrC content ) { @@ -553,8 +555,23 @@ TokArray lex( StrC content ) return { { nullptr }, 0 }; } - local_persist Arena_64KB defines_map_arena = Arena_64KB::init(); - HashTable defines = HashTable::init( defines_map_arena ); + for ( StringCached entry : PreprocessorDefines ) + { + s32 length = 0; + char const* scanner = entry.Data; + while ( entry.length() > length && char_is_alphanumeric( *scanner ) || *scanner == '_' ) + { + scanner++; + length ++; + } + if ( scanner[1] == '(' ) + { + length++; + } + + u64 key = crc32( entry.Data, length ); + defines.set( key, entry ); + } Tokens.clear(); @@ -1036,7 +1053,7 @@ TokArray lex( StrC content ) { token.Type = TokType::Comment; token.Length = 2; - token.Flags = TF_Comment; + token.Flags = TF_Null; move_forward(); while ( left && current != '\n' && current != '\r' ) @@ -1062,7 +1079,7 @@ TokArray lex( StrC content ) { token.Type = TokType::Comment; token.Length = 2; - token.Flags = TF_Comment; + token.Flags = TF_Null; move_forward(); bool star = current == '*'; diff --git a/project/components/static_data.cpp b/project/components/static_data.cpp index cf08c6c..b73a03f 100644 --- a/project/components/static_data.cpp +++ b/project/components/static_data.cpp @@ -81,6 +81,8 @@ global CodeType t_wchar_t; global CodeType t_class; global CodeType t_typename; +global Array< StringCached > PreprocessorDefines; + #ifdef GEN_DEFINE_LIBRARY_CODE_CONSTANTS global CodeType t_b32; diff --git a/project/components/types.hpp b/project/components/types.hpp index b9d5437..142f3d3 100644 --- a/project/components/types.hpp +++ b/project/components/types.hpp @@ -44,6 +44,7 @@ char const* to_str( AccessSpec type ) enum CodeFlag : u32 { + None = 0, FunctionType = bit(0), ParamPack = bit(1), Module_Export = bit(2), diff --git a/project/dependencies/containers.hpp b/project/dependencies/containers.hpp index 391df5e..eab4a5c 100644 --- a/project/dependencies/containers.hpp +++ b/project/dependencies/containers.hpp @@ -5,6 +5,14 @@ #pragma region Containers +template struct RemoveConst { typedef TType Type; }; +template struct RemoveConst { typedef TType Type; }; +template struct RemoveConst { typedef TType Type[]; }; +template struct RemoveConst { typedef TType Type[Size]; }; + +template +using TRemoveConst = typename RemoveConst::Type; + template struct Array { @@ -167,7 +175,8 @@ struct Array Header* get_header( void ) { - return rcast( Header*, Data ) - 1 ; + using NonConstType = TRemoveConst< Type >; + return rcast( Header*, const_cast(Data) ) - 1 ; } bool grow( uw min_capacity )