Unreal parsing support: Added MF_Allow_As_Attribute & MF_Allow_As_Definition

base/components/ast_types.hpp

@@ -42,6 +42,7 @@ struct AST_Body
 };
 static_assert( sizeof(AST_Body) == sizeof(AST), "ERROR: AST_Body is not the same size as AST");
 
+// TODO(Ed): Support chaining attributes (Use parameter linkage pattern)
 struct AST_Attributes
 {
 	union {

base/components/gen/especifier.hpp

@@ -39,7 +39,7 @@ enum Specifier : u32
 
 inline Str spec_to_str( Specifier type )
 {
-	local_persist Str lookup[26] = {
+	local_persist Str lookup[] = {
 		{ "INVALID",       sizeof( "INVALID" ) - 1       },
 		{ "consteval",     sizeof( "consteval" ) - 1     },
 		{ "constexpr",     sizeof( "constexpr" ) - 1     },

base/components/header_end.hpp

@@ -6,7 +6,7 @@
 
 #pragma region Constants
 
-extern PreprocessorMacro enum_underlying_macro;
+extern Macro enum_underlying_macro;
 
 extern Code access_public;
 extern Code access_protected;

base/components/interface.cpp

@@ -179,28 +179,28 @@ void define_constants()
 #endif
 
 	spec_const            = def_specifier( Spec_Const);            code_set_global( cast(Code, spec_const ));
-	spec_consteval        = def_specifier( Spec_Consteval);        code_set_global( cast(Code, spec_consteval ));;
-	spec_constexpr        = def_specifier( Spec_Constexpr);        code_set_global( cast(Code, spec_constexpr ));;
-	spec_constinit        = def_specifier( Spec_Constinit);        code_set_global( cast(Code, spec_constinit ));;
-	spec_extern_linkage   = def_specifier( Spec_External_Linkage); code_set_global( cast(Code, spec_extern_linkage ));;
-	spec_final            = def_specifier( Spec_Final);            code_set_global( cast(Code, spec_final ));;
-	spec_forceinline      = def_specifier( Spec_ForceInline);      code_set_global( cast(Code, spec_forceinline ));;
-	spec_global           = def_specifier( Spec_Global);           code_set_global( cast(Code, spec_global ));;
-	spec_inline           = def_specifier( Spec_Inline);           code_set_global( cast(Code, spec_inline ));;
-	spec_internal_linkage = def_specifier( Spec_Internal_Linkage); code_set_global( cast(Code, spec_internal_linkage ));;
-	spec_local_persist    = def_specifier( Spec_Local_Persist);    code_set_global( cast(Code, spec_local_persist ));;
-	spec_mutable          = def_specifier( Spec_Mutable);          code_set_global( cast(Code, spec_mutable ));;
-	spec_neverinline      = def_specifier( Spec_NeverInline);      code_set_global( cast(Code, spec_neverinline ));;
-	spec_noexcept         = def_specifier( Spec_NoExceptions);     code_set_global( cast(Code, spec_noexcept ));;
-	spec_override         = def_specifier( Spec_Override);         code_set_global( cast(Code, spec_override ));;
-	spec_ptr              = def_specifier( Spec_Ptr);              code_set_global( cast(Code, spec_ptr ));;
+	spec_consteval        = def_specifier( Spec_Consteval);        code_set_global( cast(Code, spec_consteval ));
+	spec_constexpr        = def_specifier( Spec_Constexpr);        code_set_global( cast(Code, spec_constexpr ));
+	spec_constinit        = def_specifier( Spec_Constinit);        code_set_global( cast(Code, spec_constinit ));
+	spec_extern_linkage   = def_specifier( Spec_External_Linkage); code_set_global( cast(Code, spec_extern_linkage ));
+	spec_final            = def_specifier( Spec_Final);            code_set_global( cast(Code, spec_final ));
+	spec_forceinline      = def_specifier( Spec_ForceInline);      code_set_global( cast(Code, spec_forceinline ));
+	spec_global           = def_specifier( Spec_Global);           code_set_global( cast(Code, spec_global ));
+	spec_inline           = def_specifier( Spec_Inline);           code_set_global( cast(Code, spec_inline ));
+	spec_internal_linkage = def_specifier( Spec_Internal_Linkage); code_set_global( cast(Code, spec_internal_linkage ));
+	spec_local_persist    = def_specifier( Spec_Local_Persist);    code_set_global( cast(Code, spec_local_persist ));
+	spec_mutable          = def_specifier( Spec_Mutable);          code_set_global( cast(Code, spec_mutable ));
+	spec_neverinline      = def_specifier( Spec_NeverInline);      code_set_global( cast(Code, spec_neverinline ));
+	spec_noexcept         = def_specifier( Spec_NoExceptions);     code_set_global( cast(Code, spec_noexcept ));
+	spec_override         = def_specifier( Spec_Override);         code_set_global( cast(Code, spec_override ));
+	spec_ptr              = def_specifier( Spec_Ptr);              code_set_global( cast(Code, spec_ptr ));
 	spec_pure             = def_specifier( Spec_Pure);             code_set_global( cast(Code, spec_pure ));
-	spec_ref              = def_specifier( Spec_Ref);              code_set_global( cast(Code, spec_ref ));;
-	spec_register         = def_specifier( Spec_Register);         code_set_global( cast(Code, spec_register ));;
-	spec_rvalue           = def_specifier( Spec_RValue);           code_set_global( cast(Code, spec_rvalue ));;
-	spec_static_member    = def_specifier( Spec_Static);           code_set_global( cast(Code, spec_static_member ));;
-	spec_thread_local     = def_specifier( Spec_Thread_Local);     code_set_global( cast(Code, spec_thread_local ));;
-	spec_virtual          = def_specifier( Spec_Virtual);          code_set_global( cast(Code, spec_virtual ));;
+	spec_ref              = def_specifier( Spec_Ref);              code_set_global( cast(Code, spec_ref ));
+	spec_register         = def_specifier( Spec_Register);         code_set_global( cast(Code, spec_register ));
+	spec_rvalue           = def_specifier( Spec_RValue);           code_set_global( cast(Code, spec_rvalue ));
+	spec_static_member    = def_specifier( Spec_Static);           code_set_global( cast(Code, spec_static_member ));
+	spec_thread_local     = def_specifier( Spec_Thread_Local);     code_set_global( cast(Code, spec_thread_local ));
+	spec_virtual          = def_specifier( Spec_Virtual);          code_set_global( cast(Code, spec_virtual ));
 	spec_volatile         = def_specifier( Spec_Volatile);         code_set_global( cast(Code, spec_volatile ));
 
 	spec_local_persist = def_specifiers( 1, Spec_Local_Persist );
@@ -211,7 +211,7 @@ void define_constants()
 		enum_underlying_macro.Type  = MT_Expression;
 		enum_underlying_macro.Flags = MF_Functional;
 	}
-	register_preprocess_macro(enum_underlying_macro);
+	register_macro(enum_underlying_macro);
 }
 
 void init(Context* ctx)
@@ -315,8 +315,8 @@ void init(Context* ctx)
 		if ( ctx->StrCache.Entries == nullptr )
 			GEN_FATAL( "gen::init: Failed to initialize the StringCache");
 
-		ctx->PreprocessorMacros = hashtable_init(PreprocessorMacro, ctx->Allocator_DyanmicContainers);
-		if (ctx->PreprocessorMacros.Hashes == nullptr || ctx->PreprocessorMacros.Entries == nullptr) {
+		ctx->Macros = hashtable_init(Macro, ctx->Allocator_DyanmicContainers);
+		if (ctx->Macros.Hashes == nullptr || ctx->Macros.Entries == nullptr) {
 			GEN_FATAL( "gen::init: Failed to initialize the PreprocessMacros table" );
 		}
 	}
@@ -356,7 +356,7 @@ void deinit(Context* ctx)
 	array_free( ctx->CodePools);
 	array_free( ctx->StringArenas);
 
-	hashtable_destroy(ctx->PreprocessorMacros);
+	hashtable_destroy(ctx->Macros);
 
 	left  = array_num( ctx->Fallback_AllocatorBuckets);
 	if (left)
@@ -401,7 +401,7 @@ void reset(Context* ctx)
 	while ( left--, left );
 
 	hashtable_clear(ctx->StrCache);
-	hashtable_clear(ctx->PreprocessorMacros);
+	hashtable_clear(ctx->Macros);
 	define_constants();
 }
 
@@ -464,47 +464,50 @@ Code make_code()
 	return result;
 }
 
-PreprocessorMacro* lookup_preprocess_macro( Str name ) {
+Macro* lookup_macro( Str name ) {
 	u32 key = crc32( name.Ptr, name.Len );
-	return hashtable_get( _ctx->PreprocessorMacros, key );
+	return hashtable_get( _ctx->Macros, key );
 }
 
-void register_preprocess_macro( PreprocessorMacro macro ) {
+void register_macro( Macro macro ) {
 	GEN_ASSERT_NOT_NULL(macro.Name.Ptr);
 	GEN_ASSERT(macro.Name.Len > 0);
 	u32 key = crc32( macro.Name.Ptr, macro.Name.Len );
-	hashtable_set( _ctx->PreprocessorMacros, key, macro );
+	macro.Name = cache_str(macro.Name);
+	hashtable_set( _ctx->Macros, key, macro );
 }
 
-void register_preprocess_macros( s32 num, ... )
+void register_macros( s32 num, ... )
 {
 	GEN_ASSERT(num > 0);
 	va_list va;
 	va_start(va, num);
 	do
 	{
-		PreprocessorMacro macro = va_arg(va, PreprocessorMacro);
+		Macro macro = va_arg(va, Macro);
 		GEN_ASSERT_NOT_NULL(macro.Name.Ptr);
 		GEN_ASSERT(macro.Name.Len > 0);
+		macro.Name = cache_str(macro.Name);
 
 		u32 key = crc32( macro.Name.Ptr, macro.Name.Len );
-		hashtable_set( _ctx->PreprocessorMacros, key, macro );
+		hashtable_set( _ctx->Macros, key, macro );
 	}
 	while (num--, num > 0);
 	va_end(va);
 }
 
-void register_preprocess_macros( s32 num,  PreprocessorMacro* macros )
+void register_macros( s32 num,  Macro* macros )
 {
 	GEN_ASSERT(num > 0);
 	do
 	{
-		PreprocessorMacro macro = * macros;
+		Macro macro = * macros;
 		GEN_ASSERT_NOT_NULL(macro.Name.Ptr);
 		GEN_ASSERT(macro.Name.Len > 0);
+		macro.Name = cache_str(macro.Name);
 
 		u32 key = crc32( macro.Name.Ptr, macro.Name.Len );
-		hashtable_set( _ctx->PreprocessorMacros, key, macro );
+		hashtable_set( _ctx->Macros, key, macro );
 		++ macros;
 	}
 	while (num--, num > 0);

base/components/interface.hpp

@@ -71,7 +71,7 @@ struct Context
 	// Used by the lexer to persistently treat all these identifiers as preprocessor defines.
 	// Populate with strings via gen::cache_str.
 	// Functional defines must have format: id( ;at minimum to indicate that the define is only valid with arguments.
-	HashTable(PreprocessorMacro) PreprocessorMacros;
+	MacroTable Macros;
 
 // Backend
 
@@ -107,16 +107,16 @@ GEN_API void reset(Context* ctx);
 GEN_API void set_context(Context* ctx);
 
 // Mostly intended for the parser
-GEN_API PreprocessorMacro* lookup_preprocess_macro( Str Name );
+GEN_API Macro* lookup_macro( Str Name );
 
 // Alternative way to add a preprocess define entry for the lexer & parser to utilize 
 // if the user doesn't want to use def_define
 // Macros are tracked by name so if the name already exists the entry will be overwritten.
-GEN_API void register_preprocess_macro( PreprocessorMacro macro );
+GEN_API void register_macro( Macro macro );
 
 // Ease of use batch registration
-GEN_API void register_preprocess_macros( s32 num, ... );
-GEN_API void register_preprocess_macros( s32 num,  PreprocessorMacro* macros );
+GEN_API void register_macros( s32 num, ... );
+GEN_API void register_macros( s32 num,  Macro* macros );
 
 // Used internally to retrive or make string allocations.
 // Strings are stored in a series of string arenas of fixed size (SizePer_StringArena)

base/components/interface.upfront.cpp

@@ -578,8 +578,8 @@ CodeDefine def_define( Str name, MacroType type, Opts_def_define p )
 
 	b32  register_define = ! p.dont_register_to_preprocess_macros;
 	if ( register_define ) {
-		PreprocessorMacro macro_entry = { result->Name, type, p.flags };
-		register_preprocess_macro(macro_entry);
+		Macro macro_entry = { result->Name, type, p.flags };
+		register_macro(macro_entry);
 	}
 	return result;
 }

base/components/lexer.cpp

@@ -128,8 +128,8 @@ s32 lex_preprocessor_define( LexContext* ctx )
 	b32 not_specifier = spec   == Spec_Invalid;
 	b32 not_attribute = attrib <= Tok___Attributes_Start;
 
-	PreprocessorMacro  macro            = { name.Text, MT_Expression, (MacroFlags)0 };
-	PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text);
+	Macro  macro            = { name.Text, MT_Expression, (MacroFlags)0 };
+	Macro* registered_macro = lookup_macro(name.Text);
 
 	if ( registered_macro == nullptr && not_specifier && not_attribute ) {
 		log_fmt("Warning: '%S' was not registered before the lexer processed its #define directive, it will be registered as a expression macro\n"
@@ -256,7 +256,7 @@ s32 lex_preprocessor_define( LexContext* ctx )
 	}
 
 	if ( registered_macro == nullptr ) {
-		register_preprocess_macro(macro);
+		register_macro(macro);
 	}
 
 	// Define's content handled by lex_preprocessor_directive (the original caller of this)
@@ -525,52 +525,31 @@ void lex_found_token( LexContext* ctx )
 		return;
 	}
 
-	PreprocessorMacro* macro = lookup_preprocess_macro( ctx->token.Text );
+	Macro* macro = lookup_macro( ctx->token.Text );
 	b32 has_args          = ctx->left && (* ctx->scanner) == '(';
 	b32 resolved_to_macro = false;
 	if (macro) {
 		ctx->token.Type   = macrotype_to_toktype(macro->Type);
 		b32 is_functional = macro_is_functional(* macro);
 		resolved_to_macro = has_args ? is_functional : ! is_functional;
+		if ( ! resolved_to_macro ) {
+			log_fmt("Info(%d, %d): %S identified as a macro but usage here does not resolve to one (interpreting as identifier)\n"
+				, ctx->token.Line
+				, ctx->token.Line
+				, macro->Name
+			);
+		}
 	}
 	if ( resolved_to_macro )
 	{
 		// TODO(Ed): When we introduce a macro AST (and expression support), we'll properly lex this section.
 		// Want to ignore any arguments the define may have as they can be execution expressions.
-		if ( has_args )
-		{
+		if ( has_args ) {
 			ctx->token.Flags |= TF_Macro_Functional;
-
-			// move_forward();
-			// ctx->token.Text.Len++;
-
-			// s32 level = 0;
-			// while ( ctx->left && ((* ctx->scanner) != ')' || level > 0) )
-			// {
-			// 	if ( (* ctx->scanner) == '(' )
-			// 		level++;
-
-			// 	else if ( (* ctx->scanner) == ')' && level > 0 )
-			// 		level--;
-
-			// 	move_forward();
-			// 	ctx->token.Text.Len++;
-			// }
-
-			// move_forward();
-			// ctx->token.Text.Len++;
 		}
-
-		//if ( (* ctx->scanner) == '\r' && ctx->scanner[1] == '\n' )
-		//{
-		//	move_forward();
-		//	ctx->token..Text.Length++;
-		//}
-		//else if ( (* ctx->scanner) == '\n' )
-		//{
-		//	move_forward();
-		//	ctx->token..Text.Length++;
-		//}
+		if ( bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Attribute) ) {
+			ctx->token.Flags |= TF_Attribute;
+		}
 	}
 	else
 	{

base/components/parser.cpp

@@ -97,7 +97,7 @@ bool lex__eat(TokArray* self, TokType type )
 	b32 is_identifier = at_idx.Type == Tok_Identifier;
 	if ( not_accepted )
 	{
-		PreprocessorMacro* macro = lookup_preprocess_macro(at_idx.Text);
+		Macro* macro = lookup_macro(at_idx.Text);
 		b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier );
 		not_accepted             = type == Tok_Identifier && accept_as_identifier ? false : true;
 	}
@@ -196,7 +196,6 @@ internal CodeComment        parse_comment                      ();
 internal Code               parse_complicated_definition       ( TokType which );
 internal CodeBody           parse_class_struct_body            ( TokType which, Token name );
 internal Code               parse_class_struct                 ( TokType which, bool inplace_def );
-internal CodeDefine         parser_parse_define                ();
 internal Code               parse_expression                   ();
 internal Code               parse_forward_or_definition        ( TokType which, bool is_inplace );
 internal CodeFn             parse_function_after_name          ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type, Token name );
@@ -205,6 +204,7 @@ internal CodeBody           parse_global_nspace                ( CodeType which
 internal Code               parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers );
 internal Token              parse_identifier                   ( bool* possible_member_function );
 internal CodeInclude        parse_include                      ();
+internal Code               parse_macro_as_definiton           ( CodeAttributes attributes, CodeSpecifiers specifiers );
 internal CodeOperator       parse_operator_after_ret_type      ( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type );
 internal Code               parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers );
 internal CodePragma         parse_pragma                       ();
@@ -218,6 +218,7 @@ internal CodeVar            parse_variable_declaration_list    ();
 
 internal CodeClass       parser_parse_class           ( bool inplace_def );
 internal CodeConstructor parser_parse_constructor     ( CodeSpecifiers specifiers );
+internal CodeDefine      parser_parse_define          ();
 internal CodeDestructor  parser_parse_destructor      ( CodeSpecifiers specifiers );
 internal CodeEnum        parser_parse_enum            ( bool inplace_def );
 internal CodeBody        parser_parse_export_body     ();
@@ -578,7 +579,7 @@ CodeAttributes parse_attributes()
 	s32   len   = 0;
 
 	// There can be more than one attribute. If there is flatten them to a single string.
-	// TODO(Ed): Support keeping an linked list of attributes similar to parameters
+	// TODO(Ed): Support chaining attributes (Use parameter linkage pattern)
 	while ( left && tok_is_attribute(currtok) )
 	{
 		if ( check( Tok_Attribute_Open ) )
@@ -1293,89 +1294,6 @@ Code parse_complicated_definition( TokType which )
 	}
 }
 
-internal inline
-CodeDefine parser_parse_define()
-{
-	push_scope();
-	if ( check(Tok_Preprocess_Hash)) {
-		// If parse_define is called by the user the hash reach here.
-		eat(Tok_Preprocess_Hash);
-	}
-
-	eat( Tok_Preprocess_Define );
-	// #define
-
-	CodeDefine
-	define = (CodeDefine) make_code();
-	define->Type = CT_Preprocess_Define;
-	if ( ! check( Tok_Identifier ) ) {
-		log_failure( "Error, expected identifier after #define\n%s", parser_to_strbuilder(_ctx->parser) );
-		parser_pop(& _ctx->parser);
-		return InvalidCode;
-	}
-	_ctx->parser.Scope->Name = currtok.Text;
-	define->Name = cache_str( tok_to_str(currtok) );
-	eat( Tok_Identifier );
-	// #define <Name>
-
-	PreprocessorMacro* macro = lookup_preprocess_macro(define->Name);
-	if (macro_is_functional(* macro)) {
-		eat( Tok_Capture_Start );
-		// #define <Name> (
-		CodeDefineParams params;
-		if ( left && currtok.Type != Tok_Capture_End ) {
-			params = (CodeDefineParams) make_code();
-			params->Type = CT_Parameters_Define;
-			params->Name = currtok.Text;
-			params->NumEntries ++;
-
-			define->Params = params;
-			eat( Tok_Preprocess_Define_Param );
-			// #define <Name> ( <param>
-		}
-		
-		while( left && currtok.Type != Tok_Capture_End ) {
-			eat( Tok_Comma );
-			// #define <Name> ( <param>, 
-
-			CodeDefineParams next_param = (CodeDefineParams) make_code();
-			next_param->Type = CT_Parameters_Define;
-			next_param->Name = currtok.Text;
-			define_params_append(params, next_param);
-
-			// #define  <Name> (  <param>, <next_param> ...
-			eat( Tok_Preprocess_Define_Param );
-		}
-
-		eat( Tok_Capture_End );
-		// #define <Name> ( <params> )
-	}
-
-	if ( ! check( Tok_Preprocess_Content ))
-	{
-		log_failure( "Error, expected content after #define %s\n%s", define->Name, parser_to_strbuilder(_ctx->parser) );
-		parser_pop(& _ctx->parser);
-		return InvalidCode;
-	}
-
-	if ( currtok.Text.Len == 0 )
-	{
-		define->Body = untyped_str( txt("\n") );
-		eat( Tok_Preprocess_Content );
-		// #define <Name> ( <params> ) <Content>
-
-		parser_pop(& _ctx->parser);
-		return define;
-	}
-
-	define->Body = untyped_str( strbuilder_to_str( parser_strip_formatting( tok_to_str(currtok), parser_strip_formatting_dont_preserve_newlines )) );
-	eat( Tok_Preprocess_Content );
-	// #define <Name> ( <params> ) <Content>
-
-	parser_pop(& _ctx->parser);
-	return define;
-}
-
 internal inline
 Code parse_assignment_expression()
 {
@@ -2517,6 +2435,12 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes
 
 	Code result = InvalidCode;
 
+	Code macro_stmt = parse_macro_as_definiton(attributes, specifiers);
+	if (macro_stmt) {
+		parser_pop(& _ctx->parser);
+		return macro_stmt;
+	}
+
 	CodeTypename type = parser_parse_type( parser_not_from_template, nullptr );
 	// <Attributes> <Specifiers> <ReturnType/ValueType>
 
@@ -2589,6 +2513,36 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes
 	return result;
 }
 
+internal
+Code parse_macro_as_definiton( CodeAttributes attributes, CodeSpecifiers specifiers )
+{
+	push_scope();
+
+	if (currtok.Type != Tok_Preprocess_Macro_Stmt ) {
+		parser_pop(& _ctx->parser);
+		return NullCode;
+	}
+	Macro* macro                     = lookup_macro(currtok.Text);
+	b32    can_resolve_to_definition = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Definition);
+	if ( ! can_resolve_to_definition) {
+		parser_pop(& _ctx->parser);
+		return NullCode;
+	}
+
+	// TODO(Ed): When AST_Macro is made, have it support attributs and specifiers for when its behaving as a declaration/definition.
+	Code code = parse_simple_preprocess( Tok_Preprocess_Macro_Stmt );
+
+	// Attributes and sepcifiers will be collapsed into the macro's serialization.
+	StrBuilder resolved_definition = strbuilder_fmt_buf(_ctx->Allocator_Temp, "%S %S %S"
+		, attributes ? strbuilder_to_str( attributes_to_strbuilder(attributes)) : txt("")
+		, specifiers ? strbuilder_to_str( specifiers_to_strbuilder(specifiers)) : txt("")
+		, code->Content
+	);
+	Code result = untyped_str( strbuilder_to_str(resolved_definition) );
+	parser_pop(& _ctx->parser);
+	return result;
+}
+
 internal
 CodePragma parse_pragma()
 {
@@ -2945,7 +2899,7 @@ Code parse_simple_preprocess( TokType which )
 	eat( which );
 	// <Macro>
 
-	PreprocessorMacro* macro = lookup_preprocess_macro( full_macro.Text );
+	Macro* macro = lookup_macro( full_macro.Text );
 	if ( which != Tok_Preprocess_Unsupported && macro == nullptr ) {
 		log_failure("Expected the macro %S to be registered\n%S", full_macro,  parser_to_strbuilder(_ctx->parser));
 		return NullCode;
@@ -3484,6 +3438,93 @@ CodeConstructor parser_parse_constructor( CodeSpecifiers specifiers )
 	return result;
 }
 
+internal inline
+CodeDefine parser_parse_define()
+{
+	push_scope();
+	if ( check(Tok_Preprocess_Hash)) {
+		// If parse_define is called by the user the hash reach here.
+		eat(Tok_Preprocess_Hash);
+	}
+
+	eat( Tok_Preprocess_Define );
+	// #define
+
+	CodeDefine
+	define = (CodeDefine) make_code();
+	define->Type = CT_Preprocess_Define;
+	if ( ! check( Tok_Identifier ) ) {
+		log_failure( "Error, expected identifier after #define\n%s", parser_to_strbuilder(_ctx->parser) );
+		parser_pop(& _ctx->parser);
+		return InvalidCode;
+	}
+	_ctx->parser.Scope->Name = currtok.Text;
+	define->Name = cache_str( tok_to_str(currtok) );
+	eat( Tok_Identifier );
+	// #define <Name>
+
+	Macro* macro = lookup_macro(define->Name);
+	if (macro_is_functional(* macro)) {
+		eat( Tok_Capture_Start );
+		// #define <Name> (
+
+		// We provide the define params even if empty to make sure '()' are serialized.
+		CodeDefineParams
+		params = (CodeDefineParams) make_code();
+		params->Type = CT_Parameters_Define;
+
+		if ( left && currtok.Type != Tok_Capture_End ) {
+			params->Name = currtok.Text;
+			params->NumEntries ++;
+
+			eat( Tok_Preprocess_Define_Param );
+			// #define <Name> ( <param>
+		}
+		
+		while( left && currtok.Type != Tok_Capture_End ) {
+			eat( Tok_Comma );
+			// #define <Name> ( <param>, 
+
+			CodeDefineParams next_param = (CodeDefineParams) make_code();
+			next_param->Type = CT_Parameters_Define;
+			next_param->Name = currtok.Text;
+			define_params_append(params, next_param);
+
+			// #define  <Name> (  <param>, <next_param> ...
+			eat( Tok_Preprocess_Define_Param );
+		}
+
+		eat( Tok_Capture_End );
+		// #define <Name> ( <params> )
+
+		define->Params = params;
+	}
+
+	if ( ! check( Tok_Preprocess_Content ))
+	{
+		log_failure( "Error, expected content after #define %s\n%s", define->Name, parser_to_strbuilder(_ctx->parser) );
+		parser_pop(& _ctx->parser);
+		return InvalidCode;
+	}
+
+	if ( currtok.Text.Len == 0 )
+	{
+		define->Body = untyped_str( txt("\n") );
+		eat( Tok_Preprocess_Content );
+		// #define <Name> ( <params> ) <Content>
+
+		parser_pop(& _ctx->parser);
+		return define;
+	}
+
+	define->Body = untyped_str( strbuilder_to_str( parser_strip_formatting( tok_to_str(currtok), parser_strip_formatting_dont_preserve_newlines )) );
+	eat( Tok_Preprocess_Content );
+	// #define <Name> ( <params> ) <Content>
+
+	parser_pop(& _ctx->parser);
+	return define;
+}
+
 internal
 CodeDestructor parser_parse_destructor( CodeSpecifiers specifiers )
 {
@@ -3757,8 +3798,11 @@ CodeEnum parser_parse_enum( bool inplace_def )
 
 					// Unreal UMETA macro support
 					if ( currtok.Type == Tok_Preprocess_Macro_Expr ) {
-						eat( Tok_Preprocess_Macro_Expr );
+						Code macro = parse_simple_preprocess( Tok_Preprocess_Macro_Expr );
 						// <Name> = <Expression> <Macro>
+
+						// We're intentially ignoring this code as its going to be serialized as an untyped string with the rest of the enum "entry".
+						// TODO(Ed): We need a CodeEnumEntry, AST_EnumEntry types
 					}
 
 					if ( currtok.Type == Tok_Comma )
@@ -4056,6 +4100,13 @@ CodeFn parser_parse_function()
 	}
 	// <export> <Attributes> <Specifiers>
 
+	// Note(Ed): We're enforcing that using this codepath requires non-macro jank.
+	// Code macro_stmt = parse_macro_as_definiton(attributes, specifiers);
+	// if (macro_stmt) {
+	// 	parser_pop(& _ctx->parser);
+	// 	return macro_stmt;
+	// }
+
 	CodeTypename ret_type = parser_parse_type(parser_not_from_template, nullptr);
 	if ( cast(Code, ret_type) == Code_Invalid ) {
 		parser_pop(& _ctx->parser);
@@ -4960,7 +5011,7 @@ CodeTypedef parser_parse_typedef()
 	valid_macro |= left && currtok.Type == Tok_Preprocess_Macro_Stmt;
 	// if (currtok.Type == Tok_Preprocess_Macro_Stmt)
 	// {
-		// PreprocessMacro* macro = lookup_preprocess_macro(currtok.Text);
+		// PreprocessMacro* macro = lookup_macro(currtok.Text);
 		// valid_macro |= macro && macro_expects_body(* macro));
 	// }
 
@@ -5444,6 +5495,13 @@ CodeVar parser_parse_variable()
 	}
 	// <ModuleFlags> <Attributes> <Specifiers>
 
+	// Note(Ed): We're enforcing that using this codepath requires non-macro jank.
+	// Code macro_stmt = parse_macro_as_definiton(attributes, specifiers);
+	// if (macro_stmt) {
+	// 	parser_pop(& _ctx->parser);
+	// 	return macro_stmt;
+	// }
+
 	CodeTypename type = parser_parse_type(parser_not_from_template, nullptr);
 	// <ModuleFlags> <Attributes> <Specifiers> <ValueType>
 

base/components/parser_types.hpp

@@ -177,15 +177,29 @@ enum EMacroFlags : u16
 {
 	MF_Functional          = bit(0), // Macro has parameters (args expected to be passed)
 	MF_Expects_Body        = bit(1), // Expects to assign a braced scope to its body.
-	MF_Allow_As_Identifier = bit(2), // lex__eat wil treat this macro as an identifier if the parser attempts to consume it as one.
-                                     //  ^^^ This is a sort of kludge because we don't support push/pop macro programs rn. ^^^
+
+	// lex__eat wil treat this macro as an identifier if the parser attempts to consume it as one.
+	//  ^^^ This is a kludge because we don't support push/pop macro pragmas rn.
+	MF_Allow_As_Identifier = bit(2), 
+
+	// lex__eat wil treat this macro as an attribute if the parser attempts to consume it as one.
+	//  ^^^ This a kludge because unreal has a macro that behaves as both a 'statement' and an attribute (UE_DEPRECATED, PRAGMA_ENABLE_DEPRECATION_WARNINGS, etc)
+	// TODO(Ed): We can keep the MF_Allow_As_Attribute flag for macros, however, we need to add the ability of AST_Attributes to chain themselves.
+	// Its thats already a thing in the standard language anyway
+	// & it would allow UE_DEPRECATED, (UE_PROPERTY / UE_FUNCTION) to chain themselves as attributes of a resolved member function/varaible definition
+	MF_Allow_As_Attribute  = bit(3),
+
+	// When a macro is encountered after attributs and specifiers while parsing a function, or variable:
+	// It will consume the macro and treat it as resolving the definition. (Yes this is for Unreal Engine)
+	// (MUST BE OF MT_Statement TYPE)
+	MF_Allow_As_Definition = bit(4),
 
 	MF_Null           = 0,
 	MF_UnderlyingType = GEN_U16_MAX,
 };
 typedef u16 MacroFlags;
 
-struct PreprocessorMacro
+struct Macro
 {
 	StrCached  Name;
 	MacroType  Type;
@@ -193,11 +207,13 @@ struct PreprocessorMacro
 };
 
 forceinline
-b32 macro_is_functional( PreprocessorMacro macro ) {
+b32 macro_is_functional( Macro macro ) {
 	return bitfield_is_set( b16, macro.Flags, MF_Functional );
 }
 
 forceinline
-b32 macro_expects_body( PreprocessorMacro macro ) {
+b32 macro_expects_body( Macro macro ) {
 	return bitfield_is_set( b16, macro.Flags, MF_Expects_Body );
 }
+
+typedef HashTable(Macro) MacroTable;

base/components/static_data.cpp

@@ -9,7 +9,7 @@ global Context* _ctx;
 #pragma region Constants
 global u32 context_counter;
 
-global PreprocessorMacro enum_underlying_macro;
+global Macro enum_underlying_macro;
 
 global Code Code_Global;
 global Code Code_Invalid;

base/dependencies/strings.hpp

@@ -40,9 +40,9 @@ struct Str
 
 #ifndef txt
 #	if GEN_COMPILER_CPP
-#		define txt( text )          Str { ( text ), sizeof( text ) - 1 }
+#		define txt( text )          GEN_NS Str { ( text ), sizeof( text ) - 1 }
 #	else
-#		define txt( text )         (Str){ ( text ), sizeof( text ) - 1 }
+#		define txt( text )         (GEN_NS Str){ ( text ), sizeof( text ) - 1 }
 #	endif
 #endif
 

base/helpers/base_codegen.hpp

@@ -237,7 +237,7 @@ CodeBody gen_especifier( char const* path, bool use_c_definition = false )
 		Str spec_to_str( Specifier type )
 		{
 			local_persist
-			Str lookup[<num>] = {
+			Str lookup[] = {
 				<entries>
 			};
 

gen_c_library/c_library.cpp

@@ -65,122 +65,122 @@ int gen_main()
 	Context ctx {};
 	gen::init(& ctx);
 
-	register_preprocess_macros( args(
-		(PreprocessorMacro { txt("bit"),                          MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("bitfield_is_set"),              MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_C_LIKE_CPP"),               MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("cast"),                         MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("ccast"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("rcast"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("pcast"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("scast"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("stringize_va"),                 MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("stringize"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("do_once"),                      MT_Expression, MF_Functional }), 
-		(PreprocessorMacro { txt("do_once_defer"),                MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("do_once_start"),                MT_Statement,  MF_Null       }), 
-		(PreprocessorMacro { txt("do_once_end"),                  MT_Statement,  MF_Null       }), 
-		(PreprocessorMacro { txt("labeled_scope_start"),          MT_Statement,  MF_Null       }), 
-		(PreprocessorMacro { txt("labeled_scope_end"),            MT_Statement,  MF_Null       }), 
-		(PreprocessorMacro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null       }), 
-		(PreprocessorMacro { txt("num_args_impl"),                MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("num_args"),                     MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("count_of"),                     MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("clamp"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("is_between"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("size_of"),                      MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("min"),                          MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("max"),                          MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("offset_of"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("static_assert"),                MT_Statement,  MF_Functional }),
-		(PreprocessorMacro { txt("typeof"),                       MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("GEN_API_C_BEGIN"),              MT_Statement,  MF_Null       }),
-		(PreprocessorMacro { txt("GEN_API_C_END"),                MT_Statement,  MF_Null       }),
-		(PreprocessorMacro { txt("nullptr"),                      MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("GEN_REMOVE_PTR"),               MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_PARAM_DEFAULT"),            MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("struct_init"),                  MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"),  MT_Statement,  MF_Null       }),
-		(PreprocessorMacro { txt("GEN_OPITMIZE_MAPPINGS_END"),    MT_Statement,  MF_Null       }),
-		(PreprocessorMacro { txt("Array"),                        MT_Typename,   MF_Functional }),
-		(PreprocessorMacro { txt("HashTable"),                    MT_Typename,   MF_Functional }),
-		(PreprocessorMacro { txt("Using_Code"),                   MT_Statement,  MF_Functional }),
-		(PreprocessorMacro { txt("Using_CodeOps"),                MT_Statement,  MF_Functional }),
-		(PreprocessorMacro { txt("kilobytes"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("megabytes"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("gigabytes"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("terabytes"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN__ONES"),                    MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("GEN__HIGHS"),                   MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("GEN__HAS_ZERO"),                MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("zero_item"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("zero_array"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_DEFAULT_MEMORY_ALIGNMENT"), MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("GEN_DEFAULT_ALLOCATOR_FLAGS"),  MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("alloc_item"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("alloc_array"),                  MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("malloc"),                       MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("mfree"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_PRINTF_MAXLEN"),            MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("cast_to_str"),                  MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("current"),                      MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("txt"),                          MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_FILE_OPEN_PROC"),           MT_Statement,  MF_Functional | MF_Expects_Body }),
-		(PreprocessorMacro { txt("GEN_FILE_READ_AT_PROC"),        MT_Statement,  MF_Functional | MF_Expects_Body }),
-		(PreprocessorMacro { txt("GEN_FILE_WRITE_AT_PROC"),       MT_Statement,  MF_Functional | MF_Expects_Body }),
-		(PreprocessorMacro { txt("GEN_FILE_SEEK_PROC"),           MT_Statement,  MF_Functional | MF_Expects_Body }),
-		(PreprocessorMacro { txt("GEN_FILE_CLOSE_PROC"),          MT_Statement,  MF_Functional | MF_Expects_Body }),
-		(PreprocessorMacro { txt("log_failure"),                  MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("operator"),                     MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("InvalidCode"),                  MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("NullCode"),                     MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("Verify_POD"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("gen_main"),                     MT_Statement,  MF_Null       })
+	register_macros( args(
+		(Macro { txt("bit"),                          MT_Expression, MF_Functional }),
+		(Macro { txt("bitfield_is_set"),              MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_C_LIKE_CPP"),               MT_Expression, MF_Null       }),
+		(Macro { txt("cast"),                         MT_Expression, MF_Functional }),
+		(Macro { txt("ccast"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("rcast"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("pcast"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("scast"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("stringize_va"),                 MT_Expression, MF_Functional }),
+		(Macro { txt("stringize"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("do_once"),                      MT_Expression, MF_Functional }), 
+		(Macro { txt("do_once_defer"),                MT_Expression, MF_Functional }),
+		(Macro { txt("do_once_start"),                MT_Statement,  MF_Null       }), 
+		(Macro { txt("do_once_end"),                  MT_Statement,  MF_Null       }), 
+		(Macro { txt("labeled_scope_start"),          MT_Statement,  MF_Null       }), 
+		(Macro { txt("labeled_scope_end"),            MT_Statement,  MF_Null       }), 
+		(Macro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null       }), 
+		(Macro { txt("num_args_impl"),                MT_Expression, MF_Functional }),
+		(Macro { txt("num_args"),                     MT_Expression, MF_Functional }),
+		(Macro { txt("count_of"),                     MT_Expression, MF_Functional }),
+		(Macro { txt("clamp"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("is_between"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("size_of"),                      MT_Expression, MF_Functional }),
+		(Macro { txt("min"),                          MT_Expression, MF_Functional }),
+		(Macro { txt("max"),                          MT_Expression, MF_Functional }),
+		(Macro { txt("offset_of"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("static_assert"),                MT_Statement,  MF_Functional }),
+		(Macro { txt("typeof"),                       MT_Expression, MF_Null       }),
+		(Macro { txt("GEN_API_C_BEGIN"),              MT_Statement,  MF_Null       }),
+		(Macro { txt("GEN_API_C_END"),                MT_Statement,  MF_Null       }),
+		(Macro { txt("nullptr"),                      MT_Expression, MF_Null       }),
+		(Macro { txt("GEN_REMOVE_PTR"),               MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_PARAM_DEFAULT"),            MT_Expression, MF_Null       }),
+		(Macro { txt("struct_init"),                  MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"),  MT_Statement,  MF_Null       }),
+		(Macro { txt("GEN_OPITMIZE_MAPPINGS_END"),    MT_Statement,  MF_Null       }),
+		(Macro { txt("Array"),                        MT_Typename,   MF_Functional }),
+		(Macro { txt("HashTable"),                    MT_Typename,   MF_Functional }),
+		(Macro { txt("Using_Code"),                   MT_Statement,  MF_Functional }),
+		(Macro { txt("Using_CodeOps"),                MT_Statement,  MF_Functional }),
+		(Macro { txt("kilobytes"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("megabytes"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("gigabytes"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("terabytes"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("GEN__ONES"),                    MT_Expression, MF_Null       }),
+		(Macro { txt("GEN__HIGHS"),                   MT_Expression, MF_Null       }),
+		(Macro { txt("GEN__HAS_ZERO"),                MT_Expression, MF_Functional }),
+		(Macro { txt("zero_item"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("zero_array"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_DEFAULT_MEMORY_ALIGNMENT"), MT_Expression, MF_Null       }),
+		(Macro { txt("GEN_DEFAULT_ALLOCATOR_FLAGS"),  MT_Expression, MF_Null       }),
+		(Macro { txt("alloc_item"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("alloc_array"),                  MT_Expression, MF_Functional }),
+		(Macro { txt("malloc"),                       MT_Expression, MF_Functional }),
+		(Macro { txt("mfree"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_PRINTF_MAXLEN"),            MT_Expression, MF_Null       }),
+		(Macro { txt("cast_to_str"),                  MT_Expression, MF_Functional }),
+		(Macro { txt("current"),                      MT_Expression, MF_Null       }),
+		(Macro { txt("txt"),                          MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_FILE_OPEN_PROC"),           MT_Statement,  MF_Functional | MF_Expects_Body }),
+		(Macro { txt("GEN_FILE_READ_AT_PROC"),        MT_Statement,  MF_Functional | MF_Expects_Body }),
+		(Macro { txt("GEN_FILE_WRITE_AT_PROC"),       MT_Statement,  MF_Functional | MF_Expects_Body }),
+		(Macro { txt("GEN_FILE_SEEK_PROC"),           MT_Statement,  MF_Functional | MF_Expects_Body }),
+		(Macro { txt("GEN_FILE_CLOSE_PROC"),          MT_Statement,  MF_Functional | MF_Expects_Body }),
+		(Macro { txt("log_failure"),                  MT_Expression, MF_Null       }),
+		(Macro { txt("operator"),                     MT_Expression, MF_Null       }),
+		(Macro { txt("InvalidCode"),                  MT_Expression, MF_Null       }),
+		(Macro { txt("NullCode"),                     MT_Expression, MF_Null       }),
+		(Macro { txt("Verify_POD"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("gen_main"),                     MT_Statement,  MF_Null       })
 	));
-	register_preprocess_macros( args(
-		(PreprocessorMacro { txt("name"),                         MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("code"),                         MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("args"),                         MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("code_str"),                     MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("code_fmt"),                     MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("parse_fmt"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("token_fmt"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("check_member_val"),             MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("check_member_str"),             MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("check_member_content"),         MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("check_member_ast"),             MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("check_params"),                 MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("check_param_eq_ret"),           MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("specs"),                        MT_Expression, MF_Functional | MF_Allow_As_Identifier }),
-		(PreprocessorMacro { txt("name_check"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("null_check"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("def_body_start"),               MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("def_body_code_array_start"),    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("move_forward"),                 MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("skip_whitespace"),              MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("end_line"),                     MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("check_parse_args"),             MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("currtok_noskip"),               MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("currtok"),                      MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("peektok"),                      MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("prevtok"),                      MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("nexttok"),                      MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("nexttok_noskip"),               MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("eat"),                          MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("left"),                         MT_Expression, MF_Null       | MF_Allow_As_Identifier }),
-		(PreprocessorMacro { txt("def_assign"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("CHECK_WAS_DEFINED"),            MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("check_noskip"),                 MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("check"),                        MT_Expression, MF_Functional | MF_Allow_As_Identifier }),
-		(PreprocessorMacro { txt("push_scope"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("cut_length"),                   MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("cut_ptr"),                      MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("pos"),                          MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("move_fwd"),                     MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("Entry"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("CheckEndParams"),               MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("UseTemplateCapture"),           MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("check_current"),                MT_Expression, MF_Functional })
+	register_macros( args(
+		(Macro { txt("name"),                         MT_Expression, MF_Functional }),
+		(Macro { txt("code"),                         MT_Expression, MF_Functional }),
+		(Macro { txt("args"),                         MT_Expression, MF_Functional }),
+		(Macro { txt("code_str"),                     MT_Expression, MF_Functional }),
+		(Macro { txt("code_fmt"),                     MT_Expression, MF_Functional }),
+		(Macro { txt("parse_fmt"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("token_fmt"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("check_member_val"),             MT_Expression, MF_Functional }),
+		(Macro { txt("check_member_str"),             MT_Expression, MF_Functional }),
+		(Macro { txt("check_member_content"),         MT_Expression, MF_Functional }),
+		(Macro { txt("check_member_ast"),             MT_Expression, MF_Functional }),
+		(Macro { txt("check_params"),                 MT_Expression, MF_Functional }),
+		(Macro { txt("check_param_eq_ret"),           MT_Expression, MF_Functional }),
+		(Macro { txt("specs"),                        MT_Expression, MF_Functional | MF_Allow_As_Identifier }),
+		(Macro { txt("name_check"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("null_check"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("def_body_start"),               MT_Expression, MF_Functional }),
+		(Macro { txt("def_body_code_array_start"),    MT_Expression, MF_Functional }),
+		(Macro { txt("move_forward"),                 MT_Expression, MF_Functional }),
+		(Macro { txt("skip_whitespace"),              MT_Expression, MF_Functional }),
+		(Macro { txt("end_line"),                     MT_Expression, MF_Functional }),
+		(Macro { txt("check_parse_args"),             MT_Expression, MF_Functional }),
+		(Macro { txt("currtok_noskip"),               MT_Expression, MF_Null       }),
+		(Macro { txt("currtok"),                      MT_Expression, MF_Null       }),
+		(Macro { txt("peektok"),                      MT_Expression, MF_Null       }),
+		(Macro { txt("prevtok"),                      MT_Expression, MF_Null       }),
+		(Macro { txt("nexttok"),                      MT_Expression, MF_Null       }),
+		(Macro { txt("nexttok_noskip"),               MT_Expression, MF_Null       }),
+		(Macro { txt("eat"),                          MT_Expression, MF_Functional }),
+		(Macro { txt("left"),                         MT_Expression, MF_Null       | MF_Allow_As_Identifier }),
+		(Macro { txt("def_assign"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("CHECK_WAS_DEFINED"),            MT_Expression, MF_Null       }),
+		(Macro { txt("check_noskip"),                 MT_Expression, MF_Functional }),
+		(Macro { txt("check"),                        MT_Expression, MF_Functional | MF_Allow_As_Identifier }),
+		(Macro { txt("push_scope"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("cut_length"),                   MT_Expression, MF_Null       }),
+		(Macro { txt("cut_ptr"),                      MT_Expression, MF_Null       }),
+		(Macro { txt("pos"),                          MT_Expression, MF_Null       }),
+		(Macro { txt("move_fwd"),                     MT_Expression, MF_Functional }),
+		(Macro { txt("Entry"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("CheckEndParams"),               MT_Expression, MF_Functional }),
+		(Macro { txt("UseTemplateCapture"),           MT_Expression, MF_Null       }),
+		(Macro { txt("check_current"),                MT_Expression, MF_Functional })
 	));
 
 	Code push_ignores           = scan_file( path_base "helpers/push_ignores.inline.hpp" );
@@ -243,7 +243,7 @@ int gen_main()
 			CodeTemplate tmpl = cast(CodeTemplate, entry);
 			if ( tmpl->Declaration->Name.contains(txt("swap")))
 			{
-				register_preprocess_macro({ txt("swap"), MT_Expression, MF_Functional });
+				register_macro({ txt("swap"), MT_Expression, MF_Functional });
 				CodeDefine macro_swap = parse_define( txt(R"(
 #define swap( a, b )        \
 do                          \
@@ -1115,6 +1115,7 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 
 	CodeBody array_arena           = gen_array(txt("gen_Arena"), txt("Array_gen_Arena"));
 	CodeBody array_pool            = gen_array(txt("gen_Pool"),  txt("Array_gen_Pool"));
+	CodeBody ht_preprocessor_macro = gen_hashtable(txt("gen_Macro"), txt("MacroTable"));
 
 	CodeBody parsed_interface = parse_file( path_base "components/interface.hpp" );
 	CodeBody interface        = def_body(CT_Global_Body);
@@ -1354,11 +1355,37 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 	Code src_static_data 	      = scan_file( path_base "components/static_data.cpp" );
 	Code src_ast_case_macros    = scan_file( path_base "components/ast_case_macros.cpp" );
 	Code src_code_serialization = scan_file( path_base "components/code_serialization.cpp" );
-	Code src_interface          = scan_file( path_base "components/interface.cpp" );
+
 	Code src_parsing_interface  = scan_file( path_base "components/interface.parsing.cpp" );
 	Code src_untyped            = scan_file( path_base "components/interface.untyped.cpp" );
 	Code src_parser_case_macros = scan_file( path_base "components/parser_case_macros.cpp" );
 
+	CodeBody parsed_src_interface = parse_file( path_base "components/interface.cpp" );
+	CodeBody src_interface        = def_body(CT_Global_Body);
+	for ( Code entry = parsed_src_interface.begin(); entry != parsed_src_interface.end(); ++ entry ) switch( entry ->Type )
+	{
+		case CT_Function:
+		{
+			CodeFn fn = cast(CodeFn, entry);
+			Code prev = entry->Prev;
+			for ( CodeParams arr_param : fn->Params )
+			{
+				b32 repeat_register_macros = fn->Name.is_equal(txt("register_macros")) && arr_param->Name.is_equal(txt("num")) && ! arr_param->Next->Name.is_equal(txt("..."));
+				if ( repeat_register_macros ) {
+					// rename second definition so there isn't a symbol conflict
+					StrBuilder postfix_arr = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "%S_arr", fn->Name);
+					fn->Name = cache_str(postfix_arr.to_str());
+				}
+			}
+			src_interface.append(fn);
+		}
+		break;
+
+		default:
+			src_interface.append(entry);
+		break;
+	}
+
 	CodeBody parsed_src_ast = parse_file( path_base "components/ast.cpp" );
 	CodeBody src_ast        = def_body(CT_Global_Body);
 	for ( Code entry = parsed_src_ast.begin(); entry != parsed_src_ast.end(); ++ entry ) switch( entry ->Type )
@@ -1418,26 +1445,22 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 			Code prev = entry->Prev;
 
 			for ( CodeParams arr_param : fn->Params )
-				if (	fn->Name.starts_with(txt("def_"))
-					&&	(		(arr_param->ValueType->Name.starts_with(txt("Specifier")) && fn->Params->NumEntries > 1)
-							||	arr_param->ValueType->Name.starts_with(txt("Code")) )
-				)
 			{
+				b32 repeat_def_array = fn->Name.starts_with(txt("def_"))         && arr_param->Name.is_equal(txt("num")) && ! arr_param->Next->Name.is_equal(txt("..."));
+				if ( repeat_def_array ) {
 					// rename second definition so there isn't a symbol conflict
 					StrBuilder postfix_arr = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "%S_arr", fn->Name);
 					fn->Name = cache_str(postfix_arr.to_str());
-					postfix_arr.free();
 				}
-
+			}
 			for ( CodeParams opt_param : fn->Params ) if (opt_param->ValueType->Name.starts_with(txt("Opts_")))
 			{
+				// The frontend names are warapped in macros so we need to give it the intenral symbol name
 				Str prefix      = txt("def_");
 				Str actual_name = { fn->Name.Ptr + prefix.Len, fn->Name.Len  - prefix.Len };
 				Str new_name    = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "def__%S", actual_name ).to_str();
-
 				fn->Name = cache_str(new_name);
 			}
-
 			src_upfront.append(fn);
 		}
 		break;
@@ -1588,6 +1611,7 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 	Code rf_interface = refactor_and_format(interface);
 	Code rf_inlines   = refactor_and_format(inlines);
 
+	Code rf_ht_preprocessor_macro = refactor_and_format(ht_preprocessor_macro);
 	Code rf_array_string_cached   = refactor_and_format(array_string_cached);
 	Code rf_header_end            = refactor_and_format(header_end);
 	Code rf_header_builder        = refactor_and_format(header_builder);
@@ -1613,7 +1637,7 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 	Code r_src_code_serialization = refactor(src_code_serialization);
 	Code r_src_parser_case_macros = refactor(src_parser_case_macros);
 
-	Code r_src_interface        = refactor(src_interface);
+	Code r_src_interface        = refactor_and_format(src_interface);
 	Code r_src_upfront          = refactor_and_format(src_upfront);
 	Code r_src_lexer            = refactor_and_format(src_lexer);
 	Code rf_array_code_typename = refactor_and_format(array_code_typename);
@@ -1696,6 +1720,8 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 		header.print( rf_array_pool);
 		header.print( fmt_newline);
 		header.print( rf_array_string_cached );
+		header.print( fmt_newline);
+		header.print( rf_ht_preprocessor_macro );
 
 		header.print( rf_interface );
 		header.print(fmt_newline);

gen_c_library/c_library.refactor

@@ -310,6 +310,14 @@ word spec_to_str,      gen_spec_to_str
 word spec_is_trailing, gen_spec_is_trailing
 // word str_to_specifier, gen_str_to_specifier
 
+word MacroType,   gen_MacroType
+word EMacroFlags, gen_EMacroFlags
+word MacroFlags,  gen_MacroFlags
+word Macro,       gen_Macro
+
+namespace macro_,    gen_macro_
+namespace macrotype, gen_macrotype_
+
 // AST
 
 word AST, gen_AST

gen_c_library/components/containers.hashtable.hpp

@@ -19,7 +19,7 @@ CodeBody gen_hashtable_base()
 	));
 
 	Code define_type = untyped_str(txt(
-R"(#define HashTable(_type) struct _type
+R"(#define HashTable(_type) struct gen_HashTable_##_type
 )"
 	));
 

gen_c_library/components/memory.fixed_arena.hpp

@@ -89,11 +89,11 @@ CodeBody gen_fixed_arenas()
 	result.append(arena_interface_2mb);
 	result.append(arena_interface_4mb);
 
-	register_preprocess_macros( args(
-		( PreprocessorMacro { txt("fixed_arena_allocator_info"), MT_Expression, MF_Functional }),
-		( PreprocessorMacro { txt("fixed_arena_init"),           MT_Expression, MF_Functional }),
-		( PreprocessorMacro { txt("fixed_arena_free"),           MT_Expression, MF_Functional }),
-		( PreprocessorMacro { txt("fixed_arena_size_remaining"), MT_Expression, MF_Functional })
+	register_macros( args(
+		( Macro { txt("fixed_arena_allocator_info"), MT_Expression, MF_Functional }),
+		( Macro { txt("fixed_arena_init"),           MT_Expression, MF_Functional }),
+		( Macro { txt("fixed_arena_free"),           MT_Expression, MF_Functional }),
+		( Macro { txt("fixed_arena_size_remaining"), MT_Expression, MF_Functional })
 	));
 
 	CodeDefine def = parse_define(txt("#define fixed_arena_allocator_info(fixed_arena) ( (AllocatorInfo) { arena_allocator_proc, & (fixed_arena)->arena } )\n"));

gen_unreal_engine/enums/AttributeTokens.csv

@@ -4,4 +4,3 @@ COREUOBJECT_API,       COREUOBJECT_API
 ENGINE_API,            ENGINE_API
 GAMEPLAYABILITIES_API, GAMEPLAYABILITIES_API
 UMG_API,               UMG_API
-UE_DEPRECATED,         UE_DEPRECATED

gen_unreal_engine/enums/ETokType.csv

@@ -42,6 +42,7 @@ Number,                      "__number__"
 Operator,                    "__operator__"
 Preprocess_Hash,             "#"
 Preprocess_Define,           "define"
+Preprocess_Define_Param,     "__define_param__"
 Preprocess_If,               "if"
 Preprocess_IfDef,            "ifdef"
 Preprocess_IfNotDef,         "ifndef"

gen_unreal_engine/unreal.cpp

@@ -64,43 +64,43 @@ int gen_main()
 	Code ue_forceinline = code_str(FORCEINLINE);
 	// Code
 
-	register_preprocess_macros( args(
-		(PreprocessorMacro { txt("bit"),                          MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("bitfield_is_set"),              MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_C_LIKE_CPP"),               MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("cast"),                         MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("ccast"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("rcast"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("pcast"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("scast"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("stringize_va"),                 MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("stringize"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("do_once"),                      MT_Expression, MF_Functional }), 
-		(PreprocessorMacro { txt("do_once_defer"),                MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("do_once_start"),                MT_Statement,  MF_Null       }), 
-		(PreprocessorMacro { txt("do_once_end"),                  MT_Statement,  MF_Null       }), 
-		(PreprocessorMacro { txt("labeled_scope_start"),          MT_Statement,  MF_Null       }), 
-		(PreprocessorMacro { txt("labeled_scope_end"),            MT_Statement,  MF_Null       }), 
-		(PreprocessorMacro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null       }), 
-		(PreprocessorMacro { txt("num_args_impl"),                MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("num_args"),                     MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("count_of"),                     MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("clamp"),                        MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("is_between"),                   MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("size_of"),                      MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("min"),                          MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("max"),                          MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("offset_of"),                    MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("static_assert"),                MT_Statement,  MF_Functional }),
-		(PreprocessorMacro { txt("typeof"),                       MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("GEN_API_C_BEGIN"),              MT_Statement,  MF_Null       }),
-		(PreprocessorMacro { txt("GEN_API_C_END"),                MT_Statement,  MF_Null       }),
-		(PreprocessorMacro { txt("nullptr"),                      MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("GEN_REMOVE_PTR"),               MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_PARAM_DEFAULT"),            MT_Expression, MF_Null       }),
-		(PreprocessorMacro { txt("struct_init"),                  MT_Expression, MF_Functional }),
-		(PreprocessorMacro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"),  MT_Statement,  MF_Null       }),
-		(PreprocessorMacro { txt("GEN_OPITMIZE_MAPPINGS_END"),    MT_Statement,  MF_Null       })
+	register_macros( args(
+		(Macro { txt("bit"),                          MT_Expression, MF_Functional }),
+		(Macro { txt("bitfield_is_set"),              MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_C_LIKE_CPP"),               MT_Expression, MF_Null       }),
+		(Macro { txt("cast"),                         MT_Expression, MF_Functional }),
+		(Macro { txt("ccast"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("rcast"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("pcast"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("scast"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("stringize_va"),                 MT_Expression, MF_Functional }),
+		(Macro { txt("stringize"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("do_once"),                      MT_Expression, MF_Functional }), 
+		(Macro { txt("do_once_defer"),                MT_Expression, MF_Functional }),
+		(Macro { txt("do_once_start"),                MT_Statement,  MF_Null       }), 
+		(Macro { txt("do_once_end"),                  MT_Statement,  MF_Null       }), 
+		(Macro { txt("labeled_scope_start"),          MT_Statement,  MF_Null       }), 
+		(Macro { txt("labeled_scope_end"),            MT_Statement,  MF_Null       }), 
+		(Macro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null       }), 
+		(Macro { txt("num_args_impl"),                MT_Expression, MF_Functional }),
+		(Macro { txt("num_args"),                     MT_Expression, MF_Functional }),
+		(Macro { txt("count_of"),                     MT_Expression, MF_Functional }),
+		(Macro { txt("clamp"),                        MT_Expression, MF_Functional }),
+		(Macro { txt("is_between"),                   MT_Expression, MF_Functional }),
+		(Macro { txt("size_of"),                      MT_Expression, MF_Functional }),
+		(Macro { txt("min"),                          MT_Expression, MF_Functional }),
+		(Macro { txt("max"),                          MT_Expression, MF_Functional }),
+		(Macro { txt("offset_of"),                    MT_Expression, MF_Functional }),
+		(Macro { txt("static_assert"),                MT_Statement,  MF_Functional }),
+		(Macro { txt("typeof"),                       MT_Expression, MF_Null       }),
+		(Macro { txt("GEN_API_C_BEGIN"),              MT_Statement,  MF_Null       }),
+		(Macro { txt("GEN_API_C_END"),                MT_Statement,  MF_Null       }),
+		(Macro { txt("nullptr"),                      MT_Expression, MF_Null       }),
+		(Macro { txt("GEN_REMOVE_PTR"),               MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_PARAM_DEFAULT"),            MT_Expression, MF_Null       }),
+		(Macro { txt("struct_init"),                  MT_Expression, MF_Functional }),
+		(Macro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"),  MT_Statement,  MF_Null       }),
+		(Macro { txt("GEN_OPITMIZE_MAPPINGS_END"),    MT_Statement,  MF_Null       })
 	));
 
 	// gen_dep.hpp