Unreal variant generates, doing c_library corrections next

base/components/code_types.hpp

@@ -202,7 +202,7 @@ struct CodeParams
 {
 #if ! GEN_C_LIKE_CPP
 	Using_Code( CodeParams );
-	forceinline void          append( CodeParams other )          { return params_append(* this, other) }
+	forceinline void          append( CodeParams other )          { return params_append(* this, other); }
 	forceinline CodeParams    get( s32 idx )                      { return params_get( * this, idx); }
 	forceinline bool          has_entries()                       { return params_has_entries(* this); }
 	forceinline StrBuilder    to_strbuilder()                     { return params_to_strbuilder(* this); }
@@ -225,8 +225,8 @@ struct CodeDefineParams
 {
 #if ! GEN_C_LIKE_CPP
 	Using_Code( CodeDefineParams );
-	forceinline void             append( CodeDefineParams other )    { return params_append( cast(CodeParams, * this), other) }
+	forceinline void             append( CodeDefineParams other )    { return params_append( cast(CodeParams, * this), cast(CodeParams, other)); }
-	forceinline CodeDefineParams get( s32 idx )                      { return params_get( cast(CodeParams, * this), idx); }
+	forceinline CodeDefineParams get( s32 idx )                      { return (CodeDefineParams) (Code) params_get( cast(CodeParams, * this), idx); }
 	forceinline bool             has_entries()                       { return params_has_entries( cast(CodeParams, * this)); }
 	forceinline StrBuilder       to_strbuilder()                     { return define_params_to_strbuilder(* this); }
 	forceinline void             to_strbuilder( StrBuilder& result ) { return define_params_to_strbuilder_ref(* this, & result); }
@@ -240,7 +240,7 @@ struct CodeDefineParams
 		GEN_ASSERT(ast);
 		return ast;
 	}
-	forceinline CodeDefineParams& operator++() { return (CodeDefineParams) (Code) cast(CodeParams, * this).operator ++(); };
+	forceinline CodeDefineParams& operator++();
 	AST_DefineParams* ast;
 };
 
@@ -1058,9 +1058,9 @@ forceinline StrBuilder to_strbuilder    ( CodeClass self )
 forceinline void       to_strbuilder_def( CodeClass self, StrBuilder& result )     { return class_to_strbuilder_def(self, & result); }
 forceinline void       to_strbuilder_fwd( CodeClass self, StrBuilder& result )     { return class_to_strbuilder_fwd(self, & result); }
 
-forceinline void             append       (CodeDefineParams appendee, CodeDefineParams other ) { return params_append(cast(CodeParam, appendee), other); }
+forceinline void             append       (CodeDefineParams appendee, CodeDefineParams other ) {        params_append(cast(CodeParams, appendee), cast(CodeParams, other)); }
-forceinline CodeDefineParams get          (CodeDefineParams params, s32 idx)                   { return params_get(cast(CodeParam, params), idx); }
+forceinline CodeDefineParams get          (CodeDefineParams params, s32 idx)                   { return (CodeDefineParams) (Code) params_get(cast(CodeParams, params), idx); }
-forceinline bool             has_entries  (CodeDefineParams params )                           { return params_has_entries(cast(CodeParam, params)); }
+forceinline bool             has_entries  (CodeDefineParams params )                           { return params_has_entries(cast(CodeParams, params)); }
 forceinline StrBuilder       to_strbuilder(CodeDefineParams params )                           { return define_params_to_strbuilder(params); }
 forceinline void             to_strbuilder(CodeDefineParams params, StrBuilder& result )       { return define_params_to_strbuilder_ref(params, & result); }
 

base/components/inlines.hpp

@@ -265,6 +265,15 @@ forceinline bool             define_params_has_entries(CodeDefineParams self)
 CodeDefineParams begin_CodeDefineParams(CodeDefineParams params)                              { return (CodeDefineParams) (Code) begin_CodeParams( cast(CodeParams, (Code)params)); }
 CodeDefineParams end_CodeDefineParams  (CodeDefineParams params)                              { return (CodeDefineParams) (Code) end_CodeParams  ( cast(CodeParams, (Code)params)); }
 CodeDefineParams next_CodeDefineParams (CodeDefineParams params, CodeDefineParams entry_iter) { return (CodeDefineParams) (Code) next_CodeParams ( cast(CodeParams, (Code)params), cast(CodeParams, (Code)entry_iter)); }
+
+#if GEN_COMPILER_CPP
+forceinline
+CodeDefineParams& CodeDefineParams::operator ++()
+{
+	* this = ast->Next;
+	return * this;
+}
+#endif
 #pragma endregion CodeDefineParams
 
 #pragma region CodeSpecifiers

base/components/interface.cpp

@@ -207,7 +207,7 @@ void define_constants()
 	code_set_global(cast(Code, spec_local_persist));
 
 	if (enum_underlying_macro.Name.Len == 0) {
-		enum_underlying_macro.Name  = txt("enum_underlying(");
+		enum_underlying_macro.Name  = txt("enum_underlying");
 		enum_underlying_macro.Type  = MT_Expression;
 		enum_underlying_macro.Flags = MF_Functional;
 	}
@@ -470,6 +470,42 @@ PreprocessorMacro* lookup_preprocess_macro( Str name ) {
 }
 
 void register_preprocess_macro( PreprocessorMacro macro ) {
+	GEN_ASSERT_NOT_NULL(macro.Name.Ptr);
+	GEN_ASSERT(macro.Name.Len > 0);
 	u32 key = crc32( macro.Name.Ptr, macro.Name.Len );
 	hashtable_set( _ctx->PreprocessorMacros, key, macro );
 }
+
+void register_preprocess_macros( s32 num, ... )
+{
+	GEN_ASSERT(num > 0);
+	va_list va;
+	va_start(va, num);
+	do
+	{
+		PreprocessorMacro macro = va_arg(va, PreprocessorMacro);
+		GEN_ASSERT_NOT_NULL(macro.Name.Ptr);
+		GEN_ASSERT(macro.Name.Len > 0);
+
+		u32 key = crc32( macro.Name.Ptr, macro.Name.Len );
+		hashtable_set( _ctx->PreprocessorMacros, key, macro );
+	}
+	while (num--, num > 0);
+	va_end(va);
+}
+
+void register_preprocess_macros( s32 num,  PreprocessorMacro* macros )
+{
+	GEN_ASSERT(num > 0);
+	do
+	{
+		PreprocessorMacro macro = * macros;
+		GEN_ASSERT_NOT_NULL(macro.Name.Ptr);
+		GEN_ASSERT(macro.Name.Len > 0);
+
+		u32 key = crc32( macro.Name.Ptr, macro.Name.Len );
+		hashtable_set( _ctx->PreprocessorMacros, key, macro );
+		++ macros;
+	}
+	while (num--, num > 0);
+}

base/components/interface.hpp

@@ -114,6 +114,10 @@ GEN_API PreprocessorMacro* lookup_preprocess_macro( Str Name );
 // Macros are tracked by name so if the name already exists the entry will be overwritten.
 GEN_API void register_preprocess_macro( PreprocessorMacro macro );
 
+// Ease of use batch registration
+GEN_API void register_preprocess_macros( s32 num, ... );
+GEN_API void register_preprocess_macros( s32 num,  PreprocessorMacro* macros );
+
 // Used internally to retrive or make string allocations.
 // Strings are stored in a series of string arenas of fixed size (SizePer_StringArena)
 GEN_API StrCached cache_str( Str str );

base/components/lexer.cpp

@@ -118,25 +118,34 @@ s32 lex_preprocessor_define( LexContext* ctx )
 	Token name = { { ctx->scanner, 1 }, Tok_Identifier, ctx->line, ctx->column, TF_Preprocess };
 	move_forward();
 
-	PreprocessorMacro  macro            = { name.Text, MT_Statement, (MacroFlags)0 };
-	PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text);
-	if ( registered_macro == nullptr ) {
-		log_fmt("Warning: %S is was not registered before the lexer processed its #define directive, it will be registered as a statement macro"
-			, name.Text 
-		);
-	}
 	while ( ctx->left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) {
 		move_forward();
 		name.Text.Len++;
 	}
+
+	Specifier spec   = str_to_specifier( name.Text );
+	TokType   attrib = str_to_toktype( name.Text );
+	b32 not_specifier = spec   == Spec_Invalid;
+	b32 not_attribute = attrib <= Tok___Attributes_Start;
+
+	PreprocessorMacro  macro            = { name.Text, MT_Statement, (MacroFlags)0 };
+	PreprocessorMacro* registered_macro = lookup_preprocess_macro(name.Text);
+
+	if ( registered_macro == nullptr && not_specifier && not_attribute ) {
+		log_fmt("Warning: '%S' was not registered before the lexer processed its #define directive, it will be registered as a statement macro\n"
+			, name.Text 
+		);
+		GEN_DEBUG_TRAP();
+	}
 	array_append( _ctx->Lexer_Tokens, name );
 
 	if ( ctx->left && (* ctx->scanner) == '(' )
 	{
 		if (registered_macro && ! macro_is_functional(* registered_macro)) {
-			log_fmt("Warning: %S registered macro is not flagged as functional yet the definition detects opening parenthesis '(' for arguments"
+			log_fmt("Warning: %S registered macro is not flagged as functional yet the definition detects opening parenthesis '(' for arguments\n"
 				, name.Text
 			);
+			GEN_DEBUG_TRAP();
 		}
 		else {
 			macro.Flags |= MF_Functional;
@@ -151,7 +160,39 @@ s32 lex_preprocessor_define( LexContext* ctx )
 		while( ctx->left && * ctx->scanner != ')')
 		{
 			skip_whitespace();
-			if ( char_is_alpha( (* ctx->scanner) ) || (* ctx->scanner) == '_' )
+			
+			Str possible_varadic = { ctx->scanner, 3 };
+			if ( ctx->left > 3 && str_are_equal( txt("..."), possible_varadic ) ) {
+				Token parameter = { { ctx->scanner, 3 }, Tok_Preprocess_Define_Param, ctx->line, ctx->column, TF_Preprocess };
+				move_forward();
+				move_forward();
+				move_forward();
+
+				array_append(_ctx->Lexer_Tokens, parameter);
+				skip_whitespace();
+				last_parameter = parameter;
+
+				while ( (* ctx->scanner) == '\\' ) {
+					move_forward();
+					skip_whitespace();
+				}
+				if (* ctx->scanner != ')' )
+				{
+					log_failure("lex_preprocessor_define(%d, %d): Expected a ')' after '...' (varaidc macro param) %S\n"
+						, ctx->line
+						, ctx->column
+						, name.Text
+					);
+					return Lex_ReturnNull;
+				}
+				break;
+			}
+			else if ( (* ctx->scanner) == '\\' ) {
+				move_forward();
+				skip_whitespace();
+				continue;
+			}
+			else if ( char_is_alpha( (* ctx->scanner) ) || (* ctx->scanner) == '_' )
 			{
 				Token parameter = { { ctx->scanner, 1 }, Tok_Preprocess_Define_Param, ctx->line, ctx->column, TF_Preprocess };
 				move_forward();
@@ -173,6 +214,10 @@ s32 lex_preprocessor_define( LexContext* ctx )
 				);
 				return Lex_ReturnNull;
 			}
+
+			if (* ctx->scanner == ')' )
+				break;
+
 			// There should be a comma
 			if ( * ctx->scanner != ',' ) {
 				log_failure("lex_preprocessor_define(%d, %d): Expected a comma after parameter %S for %S\n"
@@ -201,6 +246,15 @@ s32 lex_preprocessor_define( LexContext* ctx )
 		array_append(_ctx->Lexer_Tokens, closing_paren);
 		move_forward();
 	}
+	else if ( registered_macro && macro_is_functional( * registered_macro) ) {
+		if (registered_macro && ! macro_is_functional(* registered_macro)) {
+			log_fmt("Warning: %S registered macro is flagged as functional yet the definition detects no opening parenthesis '(' for arguments\n"
+				, name.Text
+			);
+			GEN_DEBUG_TRAP();
+		}
+	}
+
 	if ( registered_macro == nullptr ) {
 		register_preprocess_macro(macro);
 	}

base/components/parser.cpp

@@ -1318,10 +1318,13 @@ CodeDefine parse_define()
 			define->Params = params;
 
 			eat( Tok_Preprocess_Define_Param );
-			// #define <Name> ( <param> )
+			// #define <Name> ( <param>
 		}
 		
 		while( left && currtok.Type != Tok_Capture_End ) {
+			eat( Tok_Comma );
+			// #define <Name> ( <param>, 
+
 			CodeDefineParams next_param = (CodeDefineParams) make_code();
 			next_param->Type = CT_Parameters_Define;
 			next_param->Name = currtok.Text;
@@ -1344,7 +1347,7 @@ CodeDefine parse_define()
 
 	if ( currtok.Text.Len == 0 )
 	{
-		define->Body = untyped_str( tok_to_str(currtok) );
+		define->Body = untyped_str( txt("\n") );
 		eat( Tok_Preprocess_Content );
 		// #define <Name> ( <params> ) <Content>
 
@@ -1882,7 +1885,6 @@ CodeBody parse_global_nspace( CodeType which )
 			}
 		}
 
-	Member_Resolved_To_Lone_Macro:
 		if ( member == Code_Invalid )
 		{
 			log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(currtok_noskip), parser_to_strbuilder(_ctx->parser) );
@@ -2930,9 +2932,9 @@ Code parse_simple_preprocess( TokType which )
 	eat( which );
 	// <Macro>
 
-	PreprocessorMacro macro = * lookup_preprocess_macro( full_macro.Text );
+	PreprocessorMacro* macro = lookup_preprocess_macro( full_macro.Text );
 
-	if ( macro_expects_body(macro) && peektok.Type == Tok_BraceCurly_Open )
+	if ( macro && macro_expects_body(* macro) && peektok.Type == Tok_BraceCurly_Open )
 	{
 		// Eat the block scope right after the macro. Were assuming the macro defines a function definition's signature
 		eat( Tok_BraceCurly_Open );
@@ -2955,7 +2957,7 @@ Code parse_simple_preprocess( TokType which )
 
 		// TODO(Ed): Review this?
 		Str prev_proc = _ctx->parser.Scope->Prev->ProcName;
-		if ( macro.Type == MT_Typename && c_str_compare_len( prev_proc.Ptr, "parser_parse_typedef", prev_proc.Len ) != 0 )
+		if ( macro->Type == MT_Typename && c_str_compare_len( prev_proc.Ptr, "parser_parse_typedef", prev_proc.Len ) != 0 )
 		{
 			if ( check( Tok_Statement_End ))
 			{
@@ -2982,7 +2984,7 @@ Code parse_simple_preprocess( TokType which )
 			// Do nothing
 			goto Leave_Scope_Early;
 		}
-		else if (macro.Type == MT_Typename && str_contains(_ctx->parser.Scope->Prev->ProcName, txt("parser_parse_typedef")))
+		else if (macro && macro->Type == MT_Typename && str_contains(_ctx->parser.Scope->Prev->ProcName, txt("parser_parse_typedef")))
 		{
 			if ( peektok.Type == Tok_Statement_End )
 			{

base/components/parser_types.hpp

@@ -132,6 +132,8 @@ enum MacroType : u16
 	MT_Statement,      // A macro is assumed to be a statement if not resolved.
 	MT_Expression,
 	MT_Typename,
+	MT_Attribute,      // More of a note to the parser than anythign else (attributes should be defined in the user attribues def).
+	MT_Specifier,      // More of a note to the parser than anythign else (specifiers should be defined in the user attribues def).
 	MT_Block_Start,    // Not Supported yet
 	MT_Block_End,      // Not Supported yet
 	MT_Case_Statement, // Not Supported yet
@@ -157,6 +159,8 @@ Str macrotype_to_str( MacroType type )
 		{ "Statement",        sizeof("Statement")        - 1 },
 		{ "Expression",       sizeof("Expression")       - 1 },
 		{ "Typename",         sizeof("Typename")         - 1 },
+		{ "Attribute(Macro)", sizeof("Attribute(Macro)") - 1 },
+		{ "Specifier(Macro)", sizeof("Specifier(Macro)") - 1 },
 		{ "Block_Start",      sizeof("Block_Start")      - 1 },
 		{ "Block_End",        sizeof("Block_End")        - 1 },
 		{ "Case_Statement",   sizeof("Case_Statement")   - 1 },

base/dependencies/macros.hpp

@@ -248,9 +248,9 @@
 #	if ! GEN_COMPILER_C
 #		define typeof decltype
 #	elif defined(_MSC_VER)
-#		define typeof(x) __typeof__(x)
+#		define typeof __typeof__
 #	elif defined(__GNUC__) || defined(__clang__)
-#		define typeof(x) __typeof__(x)
+#		define typeof __typeof__
 #	else
 #		error "Compiler not supported"
 #	endif

gen_unreal_engine/enums/ETokType.csv

@@ -51,7 +51,9 @@ Preprocess_EndIf,            "endif"
 Preprocess_Include,          "include"
 Preprocess_Pragma,           "pragma"
 Preprocess_Content,          "__macro_content__"
-Preprocess_Macro,            "__macro__"
+Preprocess_Macro_Expr,      "__macro_expression__"
+Preprocess_Macro_Stmt,      "__macro_statment__"
+Preprocess_Macro_Typename,  "__macro_typename__"
 Preprocess_Unsupported,      "__unsupported__"
 Spec_Alignas,                "alignas"
 Spec_Const,                  "const"

gen_unreal_engine/unreal.cpp

@@ -64,6 +64,45 @@ int gen_main()
 	Code ue_forceinline = code_str(FORCEINLINE);
 	// Code
 
+	register_preprocess_macros( args(
+		(PreprocessorMacro { txt("bit"),                          MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("bitfield_is_set"),              MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("GEN_C_LIKE_CPP"),               MT_Expression, MF_Null       }),
+		(PreprocessorMacro { txt("cast"),                         MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("ccast"),                        MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("rcast"),                        MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("pcast"),                        MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("scast"),                        MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("stringize_va"),                 MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("stringize"),                    MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("do_once"),                      MT_Expression, MF_Functional }), 
+		(PreprocessorMacro { txt("do_once_defer"),                MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("do_once_start"),                MT_Statement,  MF_Null       }), 
+		(PreprocessorMacro { txt("do_once_end"),                  MT_Statement,  MF_Null       }), 
+		(PreprocessorMacro { txt("labeled_scope_start"),          MT_Statement,  MF_Null       }), 
+		(PreprocessorMacro { txt("labeled_scope_end"),            MT_Statement,  MF_Null       }), 
+		(PreprocessorMacro { txt("compiler_decorated_func_name"), MT_Expression, MF_Null       }), 
+		(PreprocessorMacro { txt("num_args_impl"),                MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("num_args"),                     MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("count_of"),                     MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("clamp"),                        MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("is_between"),                   MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("size_of"),                      MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("min"),                          MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("max"),                          MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("offset_of"),                    MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("static_assert"),                MT_Statement,  MF_Functional }),
+		(PreprocessorMacro { txt("typeof"),                       MT_Expression, MF_Null       }),
+		(PreprocessorMacro { txt("GEN_API_C_BEGIN"),              MT_Statement,  MF_Null       }),
+		(PreprocessorMacro { txt("GEN_API_C_END"),                MT_Statement,  MF_Null       }),
+		(PreprocessorMacro { txt("nullptr"),                      MT_Expression, MF_Null       }),
+		(PreprocessorMacro { txt("GEN_REMOVE_PTR"),               MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("GEN_PARAM_DEFAULT"),            MT_Expression, MF_Null       }),
+		(PreprocessorMacro { txt("struct_init"),                  MT_Expression, MF_Functional }),
+		(PreprocessorMacro { txt("GEN_OPTIMIZE_MAPPINGS_BEGIN"),  MT_Statement,  MF_Null       }),
+		(PreprocessorMacro { txt("GEN_OPITMIZE_MAPPINGS_END"),    MT_Statement,  MF_Null       })
+	));
+
 	// gen_dep.hpp
 	{
 		CodeBody macros = def_body( CT_Global_Body );