Added support for predefining preprocessor defines before parsing strings of code.

This prevents issues for preprocessor defines not getting treated properly for specific circumstances (such as macro wrappers for specifiers).

project/components/header_end.hpp

@@ -155,6 +155,11 @@ extern CodeType t_typename;
 
 #pragma endregion Macros
 
+// Used by the lexer to persistently treat all these identifiers as preprocessor defines.
+// Populate with strings via gen::get_cached_string.
+// Functional defines must have format: id( ;at minimum to indicate that the define is only valid with arguments.
+extern Array< StringCached > PreprocessorDefines;
+
 #ifdef GEN_EXPOSE_BACKEND
 
 	// Global allocator used for data with process lifetime.

project/components/interface.cpp

@@ -289,6 +289,9 @@ void init()
 			GEN_FATAL( "gen::init: Failed to initialize the StringCache");
 	}
 
+	// Preprocessor Defines
+	PreprocessorDefines = Array<StringCached>::init_reserve( GlobalAllocator, kilobytes(1) );
+
 	define_constants();
 	parser::init();
 }
@@ -332,6 +335,8 @@ void deinit()
 	}
 	while ( left--, left );
 
+	PreprocessorDefines.free();
+
 	Global_AllocatorBuckets.free();
 	parser::deinit();
 }
@@ -418,18 +423,18 @@ Code make_code()
 	}
 
 	Code result { rcast( AST*, alloc( * allocator, sizeof(AST) )) };
-	// mem_set( result.ast, 0, sizeof(AST) );
-	result->Type = ECode::Invalid;
+	mem_set( result.ast, 0, sizeof(AST) );
+	// result->Type = ECode::Invalid;
 
-	result->Content     = { nullptr };
-	result->Prev        = { nullptr };
-	result->Next        = { nullptr };
-	result->Token		= nullptr;
-	result->Parent      = { nullptr };
-	result->Name        = { nullptr };
-	result->Type        = ECode::Invalid;
-	result->ModuleFlags = ModuleFlag::Invalid;
-	result->NumEntries  = 0;
+	// result->Content     = { nullptr };
+	// result->Prev        = { nullptr };
+	// result->Next        = { nullptr };
+	// result->Token		= nullptr;
+	// result->Parent      = { nullptr };
+	// result->Name        = { nullptr };
+	// result->Type        = ECode::Invalid;
+	// result->ModuleFlags = ModuleFlag::Invalid;
+	// result->NumEntries  = 0;
 
 	return result;
 }

project/components/lexer.cpp

@@ -12,7 +12,6 @@ enum TokFlags : u32
 	TF_Assign          = bit(1),
 	TF_Preprocess      = bit(2),
 	TF_Preprocess_Cond = bit(3),
-	TF_Comment         = bit(4),
 	TF_Attribute       = bit(6),
 	TF_AccessSpecifier = bit(7),
 	TF_Specifier 	   = bit(8),
@@ -154,7 +153,9 @@ struct TokArray
 	}
 };
 
-global Array<Token> Tokens;
+global Arena_64KB      defines_map_arena;
+global HashTable<StrC> defines;
+global Array<Token>    Tokens;
 
 #define current ( * scanner )
 
@@ -534,6 +535,7 @@ void lex_found_token( StrC& content
 	Tokens.append( token );
 }
 
+
 neverinline
 TokArray lex( StrC content )
 {
@@ -553,8 +555,23 @@ TokArray lex( StrC content )
 		return { { nullptr }, 0 };
 	}
 
-	local_persist Arena_64KB defines_map_arena = Arena_64KB::init();
-	HashTable<StrC> defines = HashTable<StrC>::init( defines_map_arena );
+	for ( StringCached entry : PreprocessorDefines )
+	{
+		s32         length  = 0;
+		char const* scanner = entry.Data;
+		while ( entry.length() > length && char_is_alphanumeric( *scanner ) || *scanner == '_' )
+		{
+			scanner++;
+			length ++;
+		}
+		if ( scanner[1] == '(' )
+		{
+			length++;
+		}
+
+		u64 key = crc32( entry.Data, length );
+		defines.set( key, entry );
+	}
 
 	Tokens.clear();
 
@@ -1036,7 +1053,7 @@ TokArray lex( StrC content )
 					{
 						token.Type   = TokType::Comment;
 						token.Length = 2;
-						token.Flags  = TF_Comment;
+						token.Flags  = TF_Null;
 						move_forward();
 
 						while ( left && current != '\n' && current != '\r' )
@@ -1062,7 +1079,7 @@ TokArray lex( StrC content )
 					{
 						token.Type   = TokType::Comment;
 						token.Length = 2;
-						token.Flags  = TF_Comment;
+						token.Flags  = TF_Null;
 						move_forward();
 
 						bool star   = current    == '*';

project/components/static_data.cpp

@@ -81,6 +81,8 @@ global CodeType t_wchar_t;
 global CodeType t_class;
 global CodeType t_typename;
 
+global Array< StringCached > PreprocessorDefines;
+
 #ifdef GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 global CodeType t_b32;
 

project/components/types.hpp

@@ -44,6 +44,7 @@ char const* to_str( AccessSpec type )
 
 enum CodeFlag : u32
 {
+	None          = 0,
 	FunctionType  = bit(0),
 	ParamPack     = bit(1),
 	Module_Export = bit(2),

project/dependencies/containers.hpp

@@ -5,6 +5,14 @@
 
 #pragma region Containers
 
+template<class TType>			struct RemoveConst						{ typedef TType Type; };
+template<class TType>			struct RemoveConst<const TType>			{ typedef TType Type; };
+template<class TType>			struct RemoveConst<const TType[]>		{ typedef TType Type[]; };
+template<class TType, uw Size>	struct RemoveConst<const TType[Size]>	{ typedef TType Type[Size]; };
+
+template<class TType>
+using TRemoveConst = typename RemoveConst<TType>::Type;
+
 template<class Type>
 struct Array
 {
@@ -167,7 +175,8 @@ struct Array
 
 	Header* get_header( void )
 	{
-		return rcast( Header*, Data ) - 1 ;
+		using NonConstType = TRemoveConst< Type >;
+		return rcast( Header*, const_cast<NonConstType*>(Data) ) - 1 ;
 	}
 
 	bool grow( uw min_capacity )