WIP: It can parse to around ~2k lines. Need to improve its ability to detect when a forward declare of a class/enum/struct/union..

This language truly is a mess.

project/components/ast.cpp

@@ -107,13 +107,16 @@ String AST::to_string()
 				}
 				else
 				{
-					result.append_fmt( "%s \n{\n%s\n};", Name, Body->to_string() );
+					result.append_fmt( "%s \n{\n%s\n}", Name, Body->to_string() );
 				}
 			}
 			else
 			{
-				result.append_fmt( "class %s\n{\n%s\n};", Name, Body->to_string() );
+				result.append_fmt( "class %s\n{\n%s\n}", Name, Body->to_string() );
 			}
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 
@@ -123,9 +126,12 @@ String AST::to_string()
 				result.append( "export " );
 
 			if ( Attributes )
-				result.append_fmt( "class %s %s;", Attributes->to_string(), Name );
+				result.append_fmt( "class %s %s", Attributes->to_string(), Name );
 
-			else result.append_fmt( "class %s;", Name );
+			else result.append_fmt( "class %s", Name );
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 
@@ -142,21 +148,24 @@ String AST::to_string()
 					result.append_fmt( "%s ", Attributes->to_string() );
 
 				if ( UnderlyingType )
-					result.append_fmt( "%s : %s\n{\n%s\n};"
+					result.append_fmt( "%s : %s\n{\n%s\n}"
 						, Name
 						, UnderlyingType->to_string()
 						, Body->to_string()
 					);
 
-				else result.append_fmt( "%s\n{\n%s\n};"
+				else result.append_fmt( "%s\n{\n%s\n}"
 					, Name
 					, Body->to_string()
 				);
 			}
-			else result.append_fmt( "enum %s\n{\n%s\n};"
+			else result.append_fmt( "enum %s\n{\n%s\n}"
 				, Name
 				, Body->to_string()
 			);
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 
@@ -168,7 +177,10 @@ String AST::to_string()
 			if ( Attributes )
 				result.append_fmt( "%s ", Attributes->to_string() );
 
-			result.append_fmt( "enum %s : %s;", Name, UnderlyingType->to_string() );
+			result.append_fmt( "enum %s : %s", Name, UnderlyingType->to_string() );
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 
@@ -188,7 +200,7 @@ String AST::to_string()
 
 				if ( UnderlyingType )
 				{
-					result.append_fmt( "%s : %s\n{\n%s\n};"
+					result.append_fmt( "%s : %s\n{\n%s\n}"
 						, Name
 						, UnderlyingType->to_string()
 						, Body->to_string()
@@ -196,7 +208,7 @@ String AST::to_string()
 				}
 				else
 				{
-					result.append_fmt( "%s\n{\n%s\n};"
+					result.append_fmt( "%s\n{\n%s\n}"
 						, Name
 						, Body->to_string()
 					);
@@ -204,10 +216,13 @@ String AST::to_string()
 			}
 			else
 			{
-				result.append_fmt( "enum class %s\n{\n%s\n};"
+				result.append_fmt( "enum class %s\n{\n%s\n}"
 					, Body->to_string()
 				);
 			}
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 
@@ -221,7 +236,10 @@ String AST::to_string()
 			if ( Attributes )
 				result.append_fmt( "%s ", Attributes->to_string() );
 
-			result.append_fmt( "%s : %s;", Name, UnderlyingType->to_string() );
+			result.append_fmt( "%s : %s", Name, UnderlyingType->to_string() );
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 
@@ -587,13 +605,16 @@ String AST::to_string()
 				{
 					if ( Name )
 
-					result.append_fmt( "%s \n{\n%s\n};", Name, Body->to_string() );
+					result.append_fmt( "%s \n{\n%s\n}", Name, Body->to_string() );
 				}
 			}
 			else
 			{
-				result.append_fmt( "struct %s\n{\n%s\n};", Name, Body->to_string() );
+				result.append_fmt( "struct %s\n{\n%s\n}", Name, Body->to_string() );
 			}
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 
@@ -603,9 +624,12 @@ String AST::to_string()
 				result.append( "export " );
 
 			if ( Attributes )
-				result.append_fmt( "struct %s %s;", Attributes->to_string(), Name );
+				result.append_fmt( "struct %s %s", Attributes->to_string(), Name );
 
-			else result.append_fmt( "struct %s;", Name );
+			else result.append_fmt( "struct %s", Name );
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 
@@ -670,7 +694,7 @@ String AST::to_string()
 
 			if ( Name )
 			{
-				result.append_fmt( "%s\n{\n%s\n};"
+				result.append_fmt( "%s\n{\n%s\n}"
 					, Name
 					, Body->to_string()
 				);
@@ -678,10 +702,13 @@ String AST::to_string()
 			else
 			{
 				// Anonymous union
-				result.append_fmt( "\n{\n%s\n};"
+				result.append_fmt( "\n{\n%s\n}"
 					, Body->to_string()
 				);
 			}
+
+			if ( Parent && Parent->Type != ECode::Typedef )
+				result.append(";");
 		}
 		break;
 

project/components/data_structures.hpp

@@ -489,6 +489,12 @@ struct CodeSpecifiers
 
 	bool append( SpecifierT spec )
 	{
+		if ( ast == nullptr )
+		{
+			log_failure("CodeSpecifiers: Attempted to append to a null specifiers AST!");
+			return false;
+		}
+
 		if ( raw()->NumEntries == AST::ArrSpecs_Cap )
 		{
 			log_failure("CodeSpecifiers: Attempted to append over %d specifiers to a specifiers AST!", AST::ArrSpecs_Cap );

project/components/etoktype.cpp

@@ -6,7 +6,7 @@ namespace Parser
 
 	For the sake of scanning files, it can scan preprocessor directives
 
-	Attributes_Start is only used to indicate the start of the user_defined attribute list.
+	__Attributes_Start is only used to indicate the start of the user_defined attribute list.
 */
 
 #ifndef GEN_DEFINE_ATTRIBUTE_TOKENS
@@ -33,8 +33,8 @@ namespace Parser
 	Entry( BraceSquare_Close,      "]" )                \
 	Entry( Capture_Start,          "(" )                \
 	Entry( Capture_End,            ")" )                \
-	Entry( Comment,                "comment" )          \
+	Entry( Comment,                "__comment__" )      \
-	Entry( Char,                   "character" )        \
+	Entry( Char,                   "__character__" )    \
 	Entry( Comma,                  "," )                \
 	Entry( Decl_Class,             "class" )            \
 	Entry( Decl_GNU_Attribute,     "__attribute__" )    \
@@ -50,11 +50,11 @@ namespace Parser
 	Entry( Decl_Typedef,           "typedef" )          \
 	Entry( Decl_Using,             "using" )            \
 	Entry( Decl_Union,             "union" )            \
-	Entry( Identifier,             "identifier" )       \
+	Entry( Identifier,             "__identifier__" )   \
 	Entry( Module_Import,          "import" )           \
 	Entry( Module_Export,          "export" )           \
-	Entry( Number,                 "number" )           \
+	Entry( Number,                 "__number__" )       \
-	Entry( Operator,               "operator" )         \
+	Entry( Operator,               "__operator__" )     \
 	Entry( Preprocess_Define,      "define")            \
 	Entry( Preprocess_If,          "if")                \
 	Entry( Preprocess_IfDef,       "ifdef")             \
@@ -64,8 +64,9 @@ namespace Parser
 	Entry( Preprocess_EndIf,       "endif")             \
 	Entry( Preprocess_Include,     "include" )          \
 	Entry( Preprocess_Pragma, 	   "pragma")            \
-	Entry( Preprocess_Content, 	   "macro content")     \
+	Entry( Preprocess_Content, 	   "__macro_content__") \
-	Entry( Preprocess_Macro,       "macro")             \
+	Entry( Preprocess_Macro,       "__macro__")         \
+	Entry( Preprocess_Unsupported, "__unsupported__" )  \
 	Entry( Spec_Alignas,           "alignas" )          \
 	Entry( Spec_Const,             "const" )            \
 	Entry( Spec_Consteval,         "consteval" )        \
@@ -85,7 +86,8 @@ namespace Parser
 	Entry( Spec_Volatile,          "volatile")          \
 	Entry( Star,                   "*" )                \
 	Entry( Statement_End,          ";" )                \
-	Entry( String,                 "string" )           \
+	Entry( StaticAssert, 		   "static_assert" )    \
+	Entry( String,                 "__string__" )       \
 	Entry( Type_Unsigned, 	       "unsigned" )         \
 	Entry( Type_Signed,            "signed" )           \
 	Entry( Type_Short,             "short" )            \
@@ -93,8 +95,13 @@ namespace Parser
 	Entry( Type_char, 			   "char" )             \
 	Entry( Type_int, 			   "int" )              \
 	Entry( Type_double, 		   "double" )           \
+	Entry( Type_MS_int8, 		   "__int8" )           \
+	Entry( Type_MS_int16, 		   "__int16" )          \
+	Entry( Type_MS_int32, 		   "__int32" )          \
+	Entry( Type_MS_int64, 		   "__int64" )          \
+	Entry( Type_MS_W64, 		   "_W64" )             \
 	Entry( Varadic_Argument,       "..." )              \
-	Entry( Attributes_Start,       "__attrib_start__" )
+	Entry( __Attributes_Start,     "__attrib_start__" )
 
 	namespace ETokType
 	{

project/enums/ETokType.csv

@@ -15,8 +15,8 @@ BraceSquare_Open,       "["
 BraceSquare_Close,      "]"
 Capture_Start,          "("
 Capture_End,            ")"
-Comment,                "comemnt"
+Comment,                "__comemnt__"
-Char,                   "character"
+Char,                   "__character__"
 Comma,                  ","
 Decl_Class,             "class"
 Decl_GNU_Attribute,     "__attribute__"
@@ -26,18 +26,18 @@ Decl_Extern_Linkage,    "extern"
 Decl_Friend,            "friend"
 Decl_Module,            "module"
 Decl_Namespace,         "namespace"
-Decl_Operator,          "operator"
+Decl_Operator,          "__operator__"
 Decl_Struct,            "struct"
 Decl_Template,          "template"
 Decl_Typedef,           "typedef"
 Decl_Using,             "using"
 Decl_Union,             "union"
-Identifier,             "identifier"
+Identifier,             "__identifier__"
 Module_Import,          "import"
 Module_Export,          "export"
-Number,                 "number"
+Number,                 "__number__"
-Operator,               "operator"
+Operator,               "__operator__"
-Preprocess_Define,      "#define"
+Preprocess_Define,      "define"
 Preprocess_If,          "if"
 Preprocess_IfDef, 	    "ifdef"
 Preprocess_IfNotDef, 	"ifndef"
@@ -46,7 +46,8 @@ Preprocess_Else,        "else"
 Preprocess_EndIf,       "endif"
 Preprocess_Include,     "include"
 Preprocess_Pragma,      "pragma"
-Preprocess_Macro,       "macro"
+Preprocess_Macro,       "__macro__"
+Preprocess_Unsupported, "__unsupported__"
 Spec_Alignas,           "alignas"
 Spec_Const,             "const"
 Spec_Consteval,         "consteval"
@@ -66,7 +67,8 @@ Spec_ThreadLocal,       "thread_local"
 Spec_Volatile,          "volatile"
 Star,                   "*"
 Statement_End,          ";"
-String,                 "string"
+StaticAssert,           "static_assert"
+String,                 "__string__"
 Type_Unsigned,          "unsigned"
 Type_Signed,            "signed"
 Type_Short,             "short"
@@ -74,5 +76,10 @@ Type_Long,              "long"
 Type_char,              "char"
 Type_int,               "int"
 Type_double,            "double"
+Type_MS_int8, 		    "__int8"
+Type_MS_int16, 		    "__int16"
+Type_MS_int32, 		    "__int32"
+Type_MS_int64, 		    "__int64"
+Type_MS_W64, 		    "_W64"
 Varadic_Argument,       "..."
-Attributes_Start,       "__attrib_start__"
+__Attributes_Start,     "__attrib_start__"