Added support for predefining preprocessor defines before parsing strings of code.

This prevents issues for preprocessor defines not getting treated properly for specific circumstances (such as macro wrappers for specifiers).
This commit is contained in:
Edward R. Gonzalez 2023-11-21 20:09:14 -05:00
parent be023325a9
commit 772db608be
6 changed files with 57 additions and 18 deletions

View File

@ -155,6 +155,11 @@ extern CodeType t_typename;
#pragma endregion Macros
// Used by the lexer to persistently treat all these identifiers as preprocessor defines.
// Populate with strings via gen::get_cached_string.
// Functional defines must have format: id( ;at minimum to indicate that the define is only valid with arguments.
extern Array< StringCached > PreprocessorDefines;
#ifdef GEN_EXPOSE_BACKEND
// Global allocator used for data with process lifetime.

View File

@ -289,6 +289,9 @@ void init()
GEN_FATAL( "gen::init: Failed to initialize the StringCache");
}
// Preprocessor Defines
PreprocessorDefines = Array<StringCached>::init_reserve( GlobalAllocator, kilobytes(1) );
define_constants();
parser::init();
}
@ -332,6 +335,8 @@ void deinit()
}
while ( left--, left );
PreprocessorDefines.free();
Global_AllocatorBuckets.free();
parser::deinit();
}
@ -418,18 +423,18 @@ Code make_code()
}
Code result { rcast( AST*, alloc( * allocator, sizeof(AST) )) };
// mem_set( result.ast, 0, sizeof(AST) );
result->Type = ECode::Invalid;
mem_set( result.ast, 0, sizeof(AST) );
// result->Type = ECode::Invalid;
result->Content = { nullptr };
result->Prev = { nullptr };
result->Next = { nullptr };
result->Token = nullptr;
result->Parent = { nullptr };
result->Name = { nullptr };
result->Type = ECode::Invalid;
result->ModuleFlags = ModuleFlag::Invalid;
result->NumEntries = 0;
// result->Content = { nullptr };
// result->Prev = { nullptr };
// result->Next = { nullptr };
// result->Token = nullptr;
// result->Parent = { nullptr };
// result->Name = { nullptr };
// result->Type = ECode::Invalid;
// result->ModuleFlags = ModuleFlag::Invalid;
// result->NumEntries = 0;
return result;
}

View File

@ -12,7 +12,6 @@ enum TokFlags : u32
TF_Assign = bit(1),
TF_Preprocess = bit(2),
TF_Preprocess_Cond = bit(3),
TF_Comment = bit(4),
TF_Attribute = bit(6),
TF_AccessSpecifier = bit(7),
TF_Specifier = bit(8),
@ -154,7 +153,9 @@ struct TokArray
}
};
global Array<Token> Tokens;
global Arena_64KB defines_map_arena;
global HashTable<StrC> defines;
global Array<Token> Tokens;
#define current ( * scanner )
@ -534,6 +535,7 @@ void lex_found_token( StrC& content
Tokens.append( token );
}
neverinline
TokArray lex( StrC content )
{
@ -553,8 +555,23 @@ TokArray lex( StrC content )
return { { nullptr }, 0 };
}
local_persist Arena_64KB defines_map_arena = Arena_64KB::init();
HashTable<StrC> defines = HashTable<StrC>::init( defines_map_arena );
for ( StringCached entry : PreprocessorDefines )
{
s32 length = 0;
char const* scanner = entry.Data;
while ( entry.length() > length && char_is_alphanumeric( *scanner ) || *scanner == '_' )
{
scanner++;
length ++;
}
if ( scanner[1] == '(' )
{
length++;
}
u64 key = crc32( entry.Data, length );
defines.set( key, entry );
}
Tokens.clear();
@ -1036,7 +1053,7 @@ TokArray lex( StrC content )
{
token.Type = TokType::Comment;
token.Length = 2;
token.Flags = TF_Comment;
token.Flags = TF_Null;
move_forward();
while ( left && current != '\n' && current != '\r' )
@ -1062,7 +1079,7 @@ TokArray lex( StrC content )
{
token.Type = TokType::Comment;
token.Length = 2;
token.Flags = TF_Comment;
token.Flags = TF_Null;
move_forward();
bool star = current == '*';

View File

@ -81,6 +81,8 @@ global CodeType t_wchar_t;
global CodeType t_class;
global CodeType t_typename;
global Array< StringCached > PreprocessorDefines;
#ifdef GEN_DEFINE_LIBRARY_CODE_CONSTANTS
global CodeType t_b32;

View File

@ -44,6 +44,7 @@ char const* to_str( AccessSpec type )
enum CodeFlag : u32
{
None = 0,
FunctionType = bit(0),
ParamPack = bit(1),
Module_Export = bit(2),

View File

@ -5,6 +5,14 @@
#pragma region Containers
template<class TType> struct RemoveConst { typedef TType Type; };
template<class TType> struct RemoveConst<const TType> { typedef TType Type; };
template<class TType> struct RemoveConst<const TType[]> { typedef TType Type[]; };
template<class TType, uw Size> struct RemoveConst<const TType[Size]> { typedef TType Type[Size]; };
template<class TType>
using TRemoveConst = typename RemoveConst<TType>::Type;
template<class Type>
struct Array
{
@ -167,7 +175,8 @@ struct Array
Header* get_header( void )
{
return rcast( Header*, Data ) - 1 ;
using NonConstType = TRemoveConst< Type >;
return rcast( Header*, const_cast<NonConstType*>(Data) ) - 1 ;
}
bool grow( uw min_capacity )