Updated gencpp with latest fixes used in other projects.

This variant can support parsing some Unreal Engine files!!

project/components/ast.hpp

@@ -333,12 +333,13 @@ struct AST
 				AST*  ValueType;       // Parameter, Variable
 			};
 			union {
+				AST*  Macro;           // Parameter
 				AST*  BitfieldSize;    // Variable (Class/Struct Data Member)
 				AST*  Params;          // Constructor, Function, Operator, Template, Typename
 			};
 			union {
 				AST*  ArrExpr;          // Typename
-				AST*  Body;             // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union
+				AST*  Body;             // Class, Constructr, Destructor, Enum, Friend, Function, Namespace, Struct, Union
 				AST*  Declaration;      // Friend, Template
 				AST*  Value;            // Parameter, Variable
 			};
@@ -393,12 +394,13 @@ struct AST_POD
 				AST*  ValueType;       // Parameter, Variable
 			};
 			union {
+				AST*  Macro;           // Parameter
 				AST*  BitfieldSize;    // Variable (Class/Struct Data Member)
 				AST*  Params;          // Constructor, Function, Operator, Template, Typename
 			};
 			union {
 				AST*  ArrExpr;          // Typename
-				AST*  Body;             // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union
+				AST*  Body;             // Class, Constructr, Destructor, Enum, Friend, Function, Namespace, Struct, Union
 				AST*  Declaration;      // Friend, Template
 				AST*  Value;            // Parameter, Variable
 			};

project/components/ast_types.hpp

@@ -118,7 +118,7 @@ struct AST_Constructor
 	Code              Next;
 	parser::Token*    Tok;
 	Code              Parent;
-	char              _PAD_NAME_[ sizeof(StringCached) ];
+	StringCached      Name;
 	CodeT             Type;
 	char              _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
 };
@@ -158,7 +158,7 @@ struct AST_Destructor
 	Code                   Next;
 	parser::Token*         Tok;
 	Code                   Parent;
-	char                   _PAD_NAME_[ sizeof(StringCached) ];
+	StringCached           Name;
 	CodeT                  Type;
 	char                   _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
 };
@@ -642,7 +642,7 @@ struct AST_Param
 		{
 			char 	  _PAD_PROPERTIES_2_[ sizeof(AST*) * 3 ];
 			CodeType  ValueType;
-			char 	  _PAD_PROPERTIES_[ sizeof(AST*) ];
+			Code      Macro;
 			Code      Value;
 			char 	  _PAD_PROPERTIES_3_[ sizeof(AST*) ];
 		};

project/components/code_serialization.cpp

@@ -96,12 +96,18 @@ String CodeConstructor::to_string()
 
 void CodeConstructor::to_string_def( String& result )
 {
-	result.append( ast->Parent->Name );
+	AST* ClassStructParent = ast->Parent->Parent;
+	if (ClassStructParent) {
+		result.append( ClassStructParent->Name );
+	}
+	else {
+		result.append( ast->Name );
+	}
 
 	if ( ast->Params )
 		result.append_fmt( "( %S )", ast->Params.to_string() );
 	else
-		result.append( "(void)" );
+		result.append( "()" );
 
 	if ( ast->InitializerList )
 		result.append_fmt( " : %S", ast->InitializerList.to_string() );
@@ -114,17 +120,26 @@ void CodeConstructor::to_string_def( String& result )
 
 void CodeConstructor::to_string_fwd( String& result )
 {
-	result.append( ast->Parent->Name );
+	AST* ClassStructParent = ast->Parent->Parent;
+	if (ClassStructParent) {
+		result.append( ClassStructParent->Name );
+	}
+	else {
+		result.append( ast->Name );
+	}
 
 	if ( ast->Params )
 		result.append_fmt( "( %S )", ast->Params.to_string() );
 	else
-	{
+		result.append_fmt("()");
+
+	if (ast->Body)
+		result.append_fmt( " = %S", ast->Body.to_string() );
+
 	if ( ast->InlineCmt )
-			result.append_fmt( "(void); // %S\n", ast->InlineCmt->Content );
+		result.append_fmt( "; // %S\n", ast->InlineCmt->Content );
 	else
-			result.append( "(void);\n" );
+		result.append( ";" );
-	}
 }
 
 String CodeClass::to_string()
@@ -159,7 +174,7 @@ void CodeClass::to_string_def( String& result )
 	{
 		char const* access_level = to_str( ast->ParentAccess );
 
-		result.append_fmt( "%S : %s %S", ast->Name, access_level, ast->ParentType );
+		result.append_fmt( "%S : %s %S", ast->Name, access_level, ast->ParentType.to_string() );
 
 		CodeType interface = ast->ParentType->Next->cast< CodeType >();
 		if ( interface )
@@ -235,7 +250,11 @@ String CodeDestructor::to_string()
 
 void CodeDestructor::to_string_def( String& result )
 {
-	if ( ast->Specs )
+	if ( ast->Name )
+	{
+		result.append_fmt( "%S()", ast->Name );
+	}
+	else if ( ast->Specs )
 	{
 		if ( ast->Specs.has( ESpecifier::Virtual ) )
 			result.append_fmt( "virtual ~%S()", ast->Parent->Name );
@@ -259,6 +278,8 @@ void CodeDestructor::to_string_fwd( String& result )
 
 		if ( ast->Specs.has( ESpecifier::Pure ) )
 			result.append( " = 0;" );
+		else if (ast->Body)
+			result.append_fmt( " = %S;", ast->Body.to_string() );
 	}
 	else
 		result.append_fmt( "~%S();", ast->Parent->Name );
@@ -424,7 +445,7 @@ void CodeFriend::to_string( String& result )
 {
 	result.append_fmt( "friend %S", ast->Declaration->to_string() );
 
-	if ( result[ result.length() -1 ] != ';' )
+	if ( ast->Declaration->Type != ECode::Function && result[ result.length() - 1 ] != ';' )
 	{
 		result.append( ";" );
 	}
@@ -513,7 +534,7 @@ void CodeFn::to_string_fwd( String& result )
 	{
 		for ( SpecifierT spec : ast->Specs )
 		{
-			if ( ! ESpecifier::is_trailing( spec ) )
+			if ( ESpecifier::is_trailing( spec ) && ! (spec != ESpecifier::Pure) )
 			{
 				StrC spec_str = ESpecifier::to_str( spec );
 				result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr );
@@ -550,6 +571,11 @@ void CodeFn::to_string_fwd( String& result )
 		}
 	}
 
+	if ( ast->Specs.has( ESpecifier::Pure ) >= 0 )
+		result.append( " = 0;" );
+	else if (ast->Body)
+		result.append_fmt( " = %S;", ast->Body.to_string() );
+
 	if ( ast->InlineCmt )
 		result.append_fmt( ";  %S", ast->InlineCmt->Content );
 	else
@@ -797,16 +823,22 @@ String CodeParam::to_string()
 
 void CodeParam::to_string( String& result )
 {
-	if ( ast->ValueType.ast == nullptr )
+	if ( ast->Macro )
 	{
-		result.append_fmt( "%S", ast->Name );
+		// Related to parsing: ( <macro>, ... )
-		return;
+		result.append( ast->Macro.ast->Content );
+		// Could also be: ( <macro> <type <name>, ... )
 	}
 
 	if ( ast->Name )
+	{
+		if ( ast->ValueType.ast == nullptr )
+			result.append_fmt( " %S", ast->Name );
+		else
 			result.append_fmt( " %S %S", ast->ValueType.to_string(), ast->Name );
 
-	else
+	}
+	else if ( ast->ValueType )
 		result.append_fmt( " %S", ast->ValueType.to_string() );
 
 	if ( ast->Value )
@@ -942,7 +974,7 @@ void CodeStruct::to_string_def( String& result )
 	{
 		char const* access_level = to_str( ast->ParentAccess );
 
-		result.append_fmt( "%S : %s %S", ast->Name, access_level, ast->ParentType );
+		result.append_fmt( "%S : %s %S", ast->Name, access_level, ast->ParentType.to_string() );
 
 		CodeType interface = ast->ParentType->Next->cast< CodeType >();
 		if ( interface )

project/components/code_types.hpp

@@ -159,7 +159,7 @@ struct CodeSpecifiers
 	{
 		for ( s32 idx = 0; idx < raw()->NumEntries; idx++ )
 		{
-			if ( raw()->ArrSpecs[ raw()->NumEntries ] == spec )
+			if ( raw()->ArrSpecs[ idx ] == spec )
 				return idx;
 		}
 

project/components/gen/eoperator.hpp

@@ -52,6 +52,10 @@ namespace EOperator
 		PtrToMemOfPtr,
 		FunctionCall,
 		Comma,
+		New,
+		NewArray,
+		Delete,
+		DeleteArray,
 		NumOps
 	};
 
@@ -101,6 +105,10 @@ namespace EOperator
 			{ sizeof( "->*" ),      "->*"      },
 			{ sizeof( "()" ),       "()"       },
 			{ sizeof( "," ),        ","        },
+			{ sizeof( "new" ),      "new"      },
+			{ sizeof( "new[]" ),    "new[]"    },
+			{ sizeof( "delete" ),   "delete"   },
+			{ sizeof( "delete[]" ), "delete[]" },
 		};
 		return lookup[op];
 	}

project/components/gen/etoktype.cpp

@@ -9,7 +9,7 @@ namespace parser
 {
 	namespace ETokType
 	{
-#define GEN_DEFINE_ATTRIBUTE_TOKENS Entry( API_Export, GEN_API_Export_Code ) Entry( API_Import, GEN_API_Import_Code )
+#define GEN_DEFINE_ATTRIBUTE_TOKENS Entry( Attribute_API_Export, "GEN_API_Export_Code" ) Entry( Attribute_API_Import, "GEN_API_Import_Code" )
 
 		enum Type : u32
 		{
@@ -108,8 +108,8 @@ namespace parser
 			Type_MS_W64,
 			Varadic_Argument,
 			__Attributes_Start,
-			API_Export,
+			Attribute_API_Export,
-			API_Import,
+			Attribute_API_Import,
 			NumTokens
 		};
 

project/components/interface.parsing.cpp

@@ -33,8 +33,54 @@ CodeConstructor parse_constructor( StrC def )
 	if ( toks.Arr == nullptr )
 		return CodeInvalid;
 
+	// TODO(Ed): Constructors can have prefix attributes
+
+	CodeSpecifiers specifiers;
+	SpecifierT     specs_found[ 16 ] { ESpecifier::NumSpecifiers };
+	s32            NumSpecifiers = 0;
+
+	while ( left && currtok.is_specifier() )
+	{
+		SpecifierT spec = ESpecifier::to_type( currtok );
+
+		b32 ignore_spec = false;
+
+		switch ( spec )
+		{
+			case ESpecifier::Constexpr :
+			case ESpecifier::Explicit:
+			case ESpecifier::Inline :
+			case ESpecifier::ForceInline :
+			case ESpecifier::NeverInline :
+				break;
+
+			case ESpecifier::Const :
+				ignore_spec = true;
+				break;
+
+			default :
+				log_failure( "Invalid specifier %s for variable\n%s", ESpecifier::to_str( spec ), Context.to_string() );
+				Context.pop();
+				return CodeInvalid;
+		}
+
+		// Every specifier after would be considered part of the type type signature
+		if (ignore_spec)
+			break;
+
+		specs_found[ NumSpecifiers ] = spec;
+		NumSpecifiers++;
+		eat( currtok.Type );
+	}
+
+	if ( NumSpecifiers )
+	{
+		specifiers = def_specifiers( NumSpecifiers, specs_found );
+		// <specifiers> ...
+	}
+
 	Context.Tokens         = toks;
-	CodeConstructor result = parse_constructor();
+	CodeConstructor result = parse_constructor( specifiers );
 	return result;
 }
 
@@ -47,6 +93,9 @@ CodeDestructor parse_destructor( StrC def )
 	if ( toks.Arr == nullptr )
 		return CodeInvalid;
 
+	// TODO(Ed): Destructors can have prefix attributes
+	// TODO(Ed): Destructors can have virtual
+
 	Context.Tokens        = toks;
 	CodeDestructor result = parse_destructor();
 	return result;

project/components/interface.upfront.cpp

@@ -159,7 +159,6 @@ OpValidateResult operator__validate( OperatorT op, CodeParam params_code, CodeTy
 
 		case Unary_Plus:
 		case Unary_Minus:
-		case BNot:
 			if ( ! params_code )
 				is_member_symbol = true;
 
@@ -194,6 +193,41 @@ OpValidateResult operator__validate( OperatorT op, CodeParam params_code, CodeTy
 			}
 			break;
 
+		case BNot:
+		{
+			// Some compilers let you do this...
+		#if 0
+			if ( ! ret_type.is_equal( t_bool) )
+			{
+				log_failure( "gen::def_operator: return type is not a boolean - %s", params_code.debug_str() );
+				return OpValidateResult::Fail;
+			}
+		#endif
+
+			if ( ! params_code )
+				is_member_symbol = true;
+
+			else
+			{
+				if ( params_code->Type != ECode::Parameters )
+				{
+					log_failure( "gen::def_operator: params is not of Parameters type - %s", params_code.debug_str() );
+					return OpValidateResult::Fail;
+				}
+
+				if ( params_code->NumEntries > 1 )
+				{
+					log_failure(
+					    "gen::def_operator: operator%s may not have more than one parameter - param count: %d",
+					    to_str( op ),
+					    params_code->NumEntries
+					);
+					return OpValidateResult::Fail;
+				}
+			}
+			break;
+		}
+
 		case Add:
 		case Subtract:
 		case Multiply:
@@ -325,6 +359,11 @@ OpValidateResult operator__validate( OperatorT op, CodeParam params_code, CodeTy
 		case Comma:
 			check_params();
 			break;
+
+		case New:
+		case Delete:
+			// This library doesn't support validating new and delete yet.
+			break;
 #	undef specs
 	}
 
@@ -520,7 +559,7 @@ CodeClass def_class( StrC name
 		return CodeInvalid;
 	}
 
-	if ( parent && (parent->Type != Class || parent->Type != Struct || parent->Type != Typename || parent->Type != Untyped) )
+	if ( parent && ( parent->Type != Class && parent->Type != Struct && parent->Type != Typename && parent->Type != Untyped ) )
 	{
 		log_failure( "gen::def_class: parent provided is not type 'Class', 'Struct', 'Typeanme', or 'Untyped': %s", parent.debug_str() );
 		return CodeInvalid;
@@ -546,6 +585,7 @@ CodeClass def_class( StrC name
 
 		result->Type = Class;
 		result->Body = body;
+		result->Body->Parent = result; // TODO(Ed): Review this?
 	}
 	else
 	{
@@ -578,16 +618,24 @@ CodeDefine def_define( StrC name, StrC content )
 
 	name_check( def_define, name );
 
+	// Defines can be empty definitions
+#if 0
 	if ( content.Len <= 0 || content.Ptr == nullptr )
 	{
 		log_failure( "gen::def_define: Invalid value provided" );
 		return CodeInvalid;
 	}
+#endif
 
 	CodeDefine
 	result          = (CodeDefine) make_code();
 	result->Type    = Preprocess_Define;
 	result->Name    = get_cached_string( name );
+	if ( content.Len <= 0 || content.Ptr == nullptr )
+	{
+		result->Content = get_cached_string( txt("") );
+	}
+	else
 		result->Content = get_cached_string( content );
 
 	return result;

project/components/lexer.cpp

@@ -13,11 +13,12 @@ enum TokFlags : u32
 	TF_Preprocess      = bit(2),
 	TF_Preprocess_Cond = bit(3),
 	TF_Attribute       = bit(6),
-	TF_AccessSpecifier = bit(7),
+	TF_AccessOperator  = bit( 7 ),
-	TF_Specifier 	   = bit(8),
+	TF_AccessSpecifier = bit( 8 ),
-	TF_EndDefinition   = bit(9), // Either ; or }
+	TF_Specifier       = bit( 9 ),
-	TF_Formatting      = bit(10),
+	TF_EndDefinition   = bit( 10 ),    // Either ; or }
-	TF_Literal         = bit(11),
+	TF_Formatting      = bit( 11 ),
+	TF_Literal         = bit( 12 ),
 
 	TF_Null = 0,
 };
@@ -41,6 +42,11 @@ struct Token
 		return { Length, Text };
 	}
 
+	bool is_access_operator()
+	{
+		return bitfield_is_equal( u32, Flags, TF_AccessOperator );
+	}
+
 	bool is_access_specifier()
 	{
 		return bitfield_is_equal( u32, Flags, TF_AccessSpecifier );
@@ -141,7 +147,7 @@ struct TokArray
 			while ( Arr[idx].Type == TokType::NewLine )
 				idx++;
 
-			return Arr[idx];
+			return Arr[idx + 1];
 		}
 
 		return Arr[idx + 1];
@@ -153,7 +159,7 @@ struct TokArray
 	}
 };
 
-global Arena_64KB      defines_map_arena;
+global Arena_128KB     defines_map_arena;
 global HashTable<StrC> defines;
 global Array<Token>    Tokens;
 
@@ -451,6 +457,16 @@ void lex_found_token( StrC& content
 
 	TokType type = ETokType::to_type( token );
 
+	if (type <= TokType::Access_Public && type >= TokType::Access_Private )
+	{
+		token.Flags |= TF_AccessSpecifier;
+	}
+
+	if ( type > TokType::__Attributes_Start )
+	{
+		token.Flags |= TF_Attribute;
+	}
+
 	if ( type == ETokType::Decl_Extern_Linkage )
 	{
 		SkipWhitespace();
@@ -565,7 +581,7 @@ TokArray lex( StrC content )
 			scanner++;
 			length ++;
 		}
-		if ( scanner[1] == '(' )
+		if ( scanner[0] == '(' )
 		{
 			length++;
 		}
@@ -634,7 +650,7 @@ TokArray lex( StrC content )
 				token.Text   = scanner;
 				token.Length = 1;
 				token.Type   = TokType::Access_MemberSymbol;
-				token.Flags  = TF_AccessSpecifier;
+				token.Flags  = TF_AccessOperator;
 
 				if (left) {
 					move_forward();
@@ -1003,7 +1019,7 @@ TokArray lex( StrC content )
 					{
 						token.Length++;
 //						token.Type = TokType::Access_PointerToMemberSymbol;
-						token.Flags |= TF_AccessSpecifier;
+						token.Flags |= TF_AccessOperator;
 						move_forward();
 
 						if ( current == '*' )

project/components/types.hpp

@@ -16,9 +16,9 @@ using LogFailType = sw(*)(char const*, ...);
 enum class AccessSpec : u32
 {
 	Default,
-	Public,
-	Protected,
 	Private,
+	Protected,
+	Public,
 
 	Num_AccessSpec,
 	Invalid,
@@ -30,9 +30,9 @@ char const* to_str( AccessSpec type )
 	local_persist
 	char const* lookup[ (u32)AccessSpec::Num_AccessSpec ] = {
 		"",
-		"public",
-		"protected",
 		"private",
+		"protected",
+		"public",
 	};
 
 	if ( type > AccessSpec::Public )

project/enums/EOperator.csv

@@ -41,3 +41,7 @@ MemberOfPointer, "->"
 PtrToMemOfPtr,   "->*"
 FunctionCall,    "()"
 Comma,           ","
+New,             "new"
+NewArray,        "new[]"
+Delete,          "delete"
+DeleteArray,     "delete[]"

project/helpers/helper.hpp

@@ -250,9 +250,9 @@ CodeBody gen_etoktype( char const* etok_path, char const* attr_path )
 		char const* attribute_str = attribute_strs[idx].string;
 		char const* entry_to_str  = attribute_str_strs [idx].string;
 
-		attribute_entries.append_fmt( "%s,\n", attribute_str );
+		attribute_entries.append_fmt( "Attribute_%s,\n", attribute_str );
 		to_str_attributes.append_fmt( "{ sizeof(\"%s\"), \"%s\" },\n", entry_to_str, entry_to_str);
-		attribute_define_entries.append_fmt( "Entry( %s, %s )", attribute_str, entry_to_str );
+		attribute_define_entries.append_fmt( "Entry( Attribute_%s, \"%s\" )", attribute_str, entry_to_str );
 
 		if ( idx < attribute_strs.num() - 1 )
 			attribute_define_entries.append( " \\\n");
@@ -265,6 +265,7 @@ CodeBody gen_etoktype( char const* etok_path, char const* attr_path )
 	CodeDefine attribute_entires_def = def_define( name(GEN_DEFINE_ATTRIBUTE_TOKENS), attribute_define_entries  );
 #pragma pop_macro("GEN_DEFINE_ATTRIBUTE_TOKENS")
 
+	// We cannot parse this enum, it has Attribute names as enums
 	CodeEnum enum_code = parse_enum(token_fmt("entries", (StrC)enum_entries, "attribute_toks", (StrC)attribute_entries, stringize(
 		enum Type : u32
 		{

scripts/.clang-format

@@ -151,17 +151,17 @@ SpaceBeforeRangeBasedForLoopColon: true
 SpaceBeforeSquareBrackets: false
 SpacesBeforeTrailingComments: 4
 
-SpaceInEmptyBlock: true
+SpaceInEmptyBlock: false
 SpaceInEmptyParentheses: false
-SpacesInAngles: true
+SpacesInAngles: false
-SpacesInCStyleCastParentheses: true
+SpacesInCStyleCastParentheses: false
-SpacesInConditionalStatement: true
+SpacesInConditionalStatement: false
-SpacesInContainerLiterals: true
+SpacesInContainerLiterals: false
 SpacesInLineCommentPrefix:
   Minimum: 1
   Maximum: 20
 SpacesInParentheses: true
-SpacesInSquareBrackets: true
+SpacesInSquareBrackets: false
 
 Standard: c++17