Lexer improvement prep, segmentation of lexer and parser.

project/bootstrap.cpp

@@ -188,7 +188,9 @@ int gen_main()
 		Code        code_serialization = scan_file( "components/code_serialization.cpp" );
 		Code        interface	       = scan_file( "components/interface.cpp" );
 		Code        upfront 	       = scan_file( "components/interface.upfront.cpp" );
-		Code 	    parsing 	       = scan_file( "components/interface.parsing.cpp" );
+		Code        lexer              = scan_file( "components/lexer.cpp" );
+		Code        parser             = scan_file( "components/parser.cpp" );
+		Code 	    parsing_interface  = scan_file( "components/interface.parsing.cpp" );
 		Code        untyped 	       = scan_file( "components/interface.untyped.cpp" );
 
 		CodeBody etoktype         = gen_etoktype( "enums/ETokType.csv", "enums/AttributeTokens.csv" );
@@ -206,7 +208,7 @@ int gen_main()
 		src.print_fmt( "\n#pragma region AST\n\n" );
 		src.print( ast_case_macros );
 		src.print( ast );
-		src.print( code );
+		src.print( code_serialization );
 		src.print_fmt( "\n#pragma endregion AST\n" );
 
 		src.print_fmt( "\n#pragma region Interface\n" );
@@ -214,7 +216,9 @@ int gen_main()
 		src.print( upfront );
 		src.print_fmt( "\n#pragma region Parsing\n\n" );
 		src.print( nspaced_etoktype );
-		src.print( parsing );
+		src.print( lexer );
+		src.print( parser );
+		src.print( parsing_interface );
 		src.print( untyped );
 		src.print_fmt( "\n#pragma endregion Parsing\n\n" );
 		src.print_fmt( "#pragma endregion Interface\n\n" );

project/components/ast.cpp

@@ -381,7 +381,7 @@ void AST::to_string( String& result )
 {
 	local_persist thread_local
 	char SerializationLevel = 0;
-	
+
 	switch ( Type )
 	{
 		using namespace ECode;
@@ -410,7 +410,7 @@ void AST::to_string( String& result )
 		case Access_Public:
 			result.append( Name );
 		break;
-		
+
 		case Class:
 			cast<CodeClass>().to_string_def( result );
 		break;
@@ -500,7 +500,7 @@ void AST::to_string( String& result )
 		case Parameters:
 			cast<CodeParam>().to_string( result );
 		break;
-		
+
 		case Preprocess_Define:
 			cast<CodeDefine>().to_string( result );
 		break;

project/components/ast.hpp

@@ -113,13 +113,19 @@ struct Code
 		return ast;
 	}
 	Code& operator ++();
-	
-	
+
+
 	// TODO(Ed) : Remove this overload.
-//	auto& operator*()
-//	{
-//		return *this;
-//	}
+	auto& operator*()
+	{
+		local_persist thread_local
+		Code NullRef = { nullptr };
+
+		if ( ast == nullptr )
+			return NullRef;
+
+		return *this;
+	}
 
 	AST* ast;
 
@@ -182,7 +188,10 @@ struct AST
 	char const* type_str();
 	bool        validate_body();
 
-	neverinline String to_string();
+	String to_string();
+
+	neverinline
+	void to_string( String& result );
 
 	template< class Type >
 	forceinline Type cast()
@@ -282,6 +291,7 @@ struct AST
 	AST*              Parent;
 	StringCached      Name;
 	CodeT             Type;
+//	CodeFlag          CodeFlags;
 	ModuleFlag        ModuleFlags;
 	union {
 		b32           IsFunction;  // Used by typedef to not serialize the name field.
@@ -341,6 +351,7 @@ struct AST_POD
 	AST*              Parent;
 	StringCached      Name;
 	CodeT             Type;
+	CodeFlag          CodeFlags;
 	ModuleFlag        ModuleFlags;
 	union {
 		b32           IsFunction;  // Used by typedef to not serialize the name field.
@@ -369,6 +380,14 @@ static_assert( sizeof(AST_POD) == AST_POD_Size,    "ERROR: AST POD is not size o
 
 #pragma region Code Types
 
+// struct CodeIterator
+// {
+// 	Code begin()
+// 	{
+
+// 	}
+// };
+
 struct CodeBody
 {
 	Using_Code( CodeBody );
@@ -424,7 +443,7 @@ struct CodeClass
 	Using_Code( CodeClass );
 
 	void add_interface( CodeType interface );
-	
+
 	void to_string_def( String& result );
 	void to_string_fwd( String& result );
 
@@ -569,7 +588,7 @@ struct CodeStruct
 	Using_Code( CodeStruct );
 
 	void add_interface( CodeType interface );
-	
+
 	void to_string_def( String& result );
 	void to_string_fwd( String& result );
 
@@ -609,10 +628,10 @@ Define_CodeType( Comment );
 struct CodeConstructor
 {
 	Using_Code( CodeConstructor );
-	
+
 	void to_string_def( String& result );
 	void to_string_fwd( String& result );
-	
+
 	AST*             raw();
 	operator         Code();
 	AST_Constructor* operator->();
@@ -622,9 +641,9 @@ struct CodeConstructor
 struct CodeDefine
 {
 	Using_Code( CodeDefine );
-	
+
 	void to_string( String& result );
-	
+
 	AST*        raw();
 	operator    Code();
 	AST_Define* operator->();
@@ -634,10 +653,10 @@ struct CodeDefine
 struct CodeDestructor
 {
 	Using_Code( CodeDestructor );
-	
+
 	void to_string_def( String& result );
 	void to_string_fwd( String& result );
-	
+
 	AST*             raw();
 	operator         Code();
 	AST_Destructor* operator->();
@@ -647,10 +666,12 @@ struct CodeDestructor
 struct CodeEnum
 {
 	Using_Code( CodeEnum );
-	
+
 	void to_string_def( String& result );
 	void to_string_fwd( String& result );
-	
+	void to_string_class_def( String& result );
+	void to_string_class_fwd( String& result );
+
 	AST*      raw();
 	operator  Code();
 	AST_Enum* operator->();
@@ -662,9 +683,9 @@ Define_CodeType( Exec );
 struct CodeExtern
 {
 	Using_Code( CodeExtern );
-	
+
 	void to_string( String& result );
-	
+
 	AST*        raw();
 	operator    Code();
 	AST_Extern* operator->();
@@ -674,9 +695,9 @@ struct CodeExtern
 struct CodeInclude
 {
 	Using_Code( CodeInclude );
-	
+
 	void to_string( String& result );
-	
+
 	AST*         raw();
 	operator     Code();
 	AST_Include* operator->();
@@ -686,9 +707,9 @@ struct CodeInclude
 struct CodeFriend
 {
 	Using_Code( CodeFriend );
-	
+
 	void to_string( String& result );
-	
+
 	AST*        raw();
 	operator    Code();
 	AST_Friend* operator->();
@@ -697,11 +718,11 @@ struct CodeFriend
 
 struct CodeFn
 {
-	Using_Code( CodeFriend );
-	
+	Using_Code( CodeFn );
+
 	void to_string_def( String& result );
 	void to_string_fwd( String& result );
-	
+
 	AST*     raw();
 	operator Code();
 	AST_Fn*  operator->();
@@ -711,9 +732,9 @@ struct CodeFn
 struct CodeModule
 {
 	Using_Code( CodeModule );
-	
+
 	void to_string( String& result );
-	
+
 	AST*        raw();
 	operator    Code();
 	AST_Module* operator->();
@@ -723,9 +744,9 @@ struct CodeModule
 struct CodeNS
 {
 	Using_Code( CodeNS );
-	
+
 	void to_string( String& result );
-	
+
 	AST*     raw();
 	operator Code();
 	AST_NS*  operator->();
@@ -735,10 +756,10 @@ struct CodeNS
 struct CodeOperator
 {
 	Using_Code( CodeOperator );
-	
+
 	void to_string_def( String& result );
 	void to_string_fwd( String& result );
-	
+
 	AST*          raw();
 	operator      Code();
 	AST_Operator* operator->();
@@ -748,10 +769,10 @@ struct CodeOperator
 struct CodeOpCast
 {
 	Using_Code( CodeOpCast );
-	
+
 	void to_string_def( String& result );
 	void to_string_fwd( String& result );
-	
+
 	AST*        raw();
 	operator    Code();
 	AST_OpCast* operator->();
@@ -761,9 +782,9 @@ struct CodeOpCast
 struct CodePragma
 {
 	Using_Code( CodePragma );
-	
+
 	void to_string( String& result );
-	
+
 	AST*        raw();
 	operator    Code();
 	AST_Pragma* operator->();
@@ -773,14 +794,14 @@ struct CodePragma
 struct CodePreprocessCond
 {
 	Using_Code( CodePreprocessCond );
-	
+
 	void to_string_if( String& result );
 	void to_string_ifdef( String& result );
 	void to_string_ifndef( String& result );
 	void to_string_elif( String& result );
 	void to_string_else( String& result );
 	void to_string_endif( String& result );
-	
+
 	AST*                raw();
 	operator            Code();
 	AST_PreprocessCond* operator->();
@@ -790,9 +811,9 @@ struct CodePreprocessCond
 struct CodeTemplate
 {
 	Using_Code( CodeTemplate );
-	
+
 	void to_string( String& result );
-	
+
 	AST*          raw();
 	operator      Code();
 	AST_Template* operator->();
@@ -802,9 +823,9 @@ struct CodeTemplate
 struct CodeType
 {
 	Using_Code( CodeType );
-	
+
 	void to_string( String& result );
-	
+
 	AST*      raw();
 	operator  Code();
 	AST_Type* operator->();
@@ -814,9 +835,9 @@ struct CodeType
 struct CodeTypedef
 {
 	Using_Code( CodeTypedef );
-	
+
 	void to_string( String& result );
-	
+
 	AST*         raw();
 	operator     Code();
 	AST_Typedef* operator->();
@@ -826,9 +847,9 @@ struct CodeTypedef
 struct CodeUnion
 {
 	Using_Code( CodeUnion );
-	
+
 	void to_string( String& result );
-	
+
 	AST*         raw();
 	operator     Code();
 	AST_Union* operator->();
@@ -838,10 +859,10 @@ struct CodeUnion
 struct CodeUsing
 {
 	Using_Code( CodeUsing );
-	
+
 	void to_string( String& result );
 	void to_string_ns( String& result );
-	
+
 	AST*       raw();
 	operator   Code();
 	AST_Using* operator->();
@@ -851,9 +872,9 @@ struct CodeUsing
 struct CodeVar
 {
 	Using_Code( CodeVar );
-	
+
 	void to_string( String& result );
-	
+
 	AST*     raw();
 	operator Code();
 	AST_Var* operator->();

project/components/code_serialization.cpp

@@ -1,4 +1,4 @@
-#if GEN_INTELLISENSE_DIRECTIVES
+#ifdef GEN_INTELLISENSE_DIRECTIVES
 #pragma once
 #include "ast.cpp"
 #endif
@@ -24,6 +24,11 @@ String CodeBody::to_string()
 	switch ( ast->Type )
 	{
 		using namespace ECode;
+		case Untyped:
+		case Execution:
+			result.append( raw()->Content );
+		break;
+
 		case Enum_Body:
 		case Class_Body:
 		case Extern_Linkage_Body:
@@ -34,7 +39,7 @@ String CodeBody::to_string()
 		case Union_Body:
 			to_string( result );
 		break;
-			
+
 		case Export_Body:
 			to_string_export( result );
 		break;
@@ -57,7 +62,7 @@ void CodeBody::to_string_export( String& result )
 {
 	result.append_fmt( "export\n{\n" );
 
-	Code curr = { this };
+	Code curr = *this;
 	s32  left = ast->NumEntries;
 	while ( left-- )
 	{
@@ -91,26 +96,26 @@ String CodeConstructor::to_string()
 
 void CodeConstructor::to_string_def( String& result )
 {
-	result.append( ast->Parent->name );
-	
+	result.append( ast->Parent->Name );
+
 	if ( ast->Params )
 		result.append_fmt( "( %S )", ast->Params.to_string() );
 	else
 		result.append( "(void)" );
-	
+
 	if ( ast->InitializerList )
 		result.append_fmt( " : %S", ast->InitializerList.to_string() );
-	
+
 	if ( ast->InlineCmt )
 		result.append_fmt( " // %S", ast->InlineCmt->Content );
-	
+
 	result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() );
 }
 
 void CodeConstructor::to_string_fwd( String& result )
 {
 	result.append( ast->Parent->Name );
-	
+
 	if ( ast->Params )
 		result.append_fmt( "( %S )", ast->Params.to_string() );
 	else
@@ -178,7 +183,7 @@ void CodeClass::to_string_def( String& result )
 
 	result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() );
 
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 		result.append(";\n");
 }
 
@@ -193,7 +198,7 @@ void CodeClass::to_string_fwd( String& result )
 	else result.append_fmt( "class %S", ast->Name );
 
 	// Check if it can have an end-statement
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 	{
 		if ( ast->InlineCmt )
 			result.append_fmt( "; // %S\n", ast->InlineCmt->Content );
@@ -209,7 +214,7 @@ String CodeDefine::to_string()
 
 void CodeDefine::to_string( String& result )
 {
-	return result.append_fmt( "#define %S %S\n", ast->Name, ast->Content );
+	result.append_fmt( "#define %S %S\n", ast->Name, ast->Content );
 }
 
 String CodeDestructor::to_string()
@@ -239,7 +244,7 @@ void CodeDestructor::to_string_def( String& result )
 	}
 	else
 		result.append_fmt( "~%S()", ast->Parent->Name );
-	
+
 	result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() );
 }
 
@@ -290,26 +295,26 @@ void CodeEnum::to_string_def( String& result )
 {
 	if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ))
 		result.append( "export " );
-	
+
 	if ( ast->Attributes || ast->UnderlyingType )
 	{
 		result.append( "enum " );
-	
+
 		if ( ast->Attributes )
 			result.append_fmt( "%S ", ast->Attributes.to_string() );
-	
-		if ( UnderlyingType )
+
+		if ( ast->UnderlyingType )
 			result.append_fmt( "%S : %S\n{\n%S\n}"
 				, ast->Name
 				, ast->UnderlyingType.to_string()
 				, ast->Body.to_string()
 			);
-	
+
 		else result.append_fmt( "%S\n{\n%S\n}", ast->Name, ast->Body.to_string() );
 	}
 	else result.append_fmt( "enum %S\n{\n%S\n}", ast->Name, ast->Body.to_string() );
-	
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 		result.append(";\n");
 }
 
@@ -323,7 +328,7 @@ void CodeEnum::to_string_fwd( String& result )
 
 	result.append_fmt( "enum %S : %S", ast->Name, ast->UnderlyingType.to_string() );
 
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 	{
 		if ( ast->InlineCmt )
 			result.append_fmt(";  %S", ast->InlineCmt->Content );
@@ -336,16 +341,16 @@ void CodeEnum::to_string_class_def( String& result )
 {
 	if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ))
 		result.append( "export " );
-	
+
 	if ( ast->Attributes || ast->UnderlyingType )
 	{
 		result.append( "enum class " );
-	
+
 		if ( ast->Attributes )
 		{
 			result.append_fmt( "%S ", ast->Attributes.to_string() );
 		}
-	
+
 		if ( ast->UnderlyingType )
 		{
 			result.append_fmt( "%S : %S\n{\n%S\n}", ast->Name, ast->UnderlyingType.to_string(), ast->Body.to_string() );
@@ -359,8 +364,8 @@ void CodeEnum::to_string_class_def( String& result )
 	{
 		result.append_fmt( "enum class %S\n{\n%S\n}", ast->Body.to_string() );
 	}
-	
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 		result.append(";\n");
 }
 
@@ -376,7 +381,7 @@ void CodeEnum::to_string_class_fwd( String& result )
 
 	result.append_fmt( "%S : %S", ast->Name, ast->UnderlyingType.to_string() );
 
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 	{
 		if ( ast->InlineCmt )
 			result.append_fmt(";  %S", ast->InlineCmt->Content );
@@ -390,13 +395,6 @@ String CodeExec::to_string()
 	return ast->Content.duplicate( GlobalAllocator );
 }
 
-String CodeExtern::to_string()
-{
-	String result = String::make( GlobalAllocator, "" );
-	to_string( result );
-	return result;
-}
-
 void CodeExtern::to_string( String& result )
 {
 	if ( ast->Body )
@@ -407,7 +405,7 @@ void CodeExtern::to_string( String& result )
 
 String CodeInclude::to_string()
 {
-	return String::fmt( GlobalAllocator, "#include %S\n", ast->Content );
+	return String::fmt_buf( GlobalAllocator, "#include %S\n", ast->Content );
 }
 
 void CodeInclude::to_string( String& result )
@@ -415,7 +413,7 @@ void CodeInclude::to_string( String& result )
 	result.append_fmt( "#include %S\n", ast->Content );
 }
 
-String CodeFriend::to_string( String& result )
+String CodeFriend::to_string()
 {
 	String result = String::make( GlobalAllocator, "" );
 	to_string( result );
@@ -437,7 +435,7 @@ void CodeFriend::to_string( String& result )
 		result.append("\n");
 }
 
-String CodeFn::to_string( String& result )
+String CodeFn::to_string()
 {
 	String result = String::make( GlobalAllocator, "" );
 	switch ( ast->Type )
@@ -482,7 +480,7 @@ void CodeFn::to_string_def( String& result )
 	else
 		result.append_fmt( "%S(", ast->Name );
 
-	if ( Params )
+	if ( ast->Params )
 		result.append_fmt( "%S)", ast->Params.to_string() );
 
 	else
@@ -511,9 +509,9 @@ void CodeFn::to_string_fwd( String& result )
 	if ( ast->Attributes )
 		result.append_fmt( "%S ", ast->Attributes.to_string() );
 
-	if ( Specs )
+	if ( ast->Specs )
 	{
-		for ( SpecifierT spec : Specs )
+		for ( SpecifierT spec : ast->Specs )
 		{
 			if ( ! ESpecifier::is_trailing( spec ) )
 			{
@@ -542,7 +540,7 @@ void CodeFn::to_string_fwd( String& result )
 
 	if ( ast->Specs )
 	{
-		for ( SpecifierT spec : Specs )
+		for ( SpecifierT spec : ast->Specs )
 		{
 			if ( ESpecifier::is_trailing( spec ) )
 			{
@@ -673,7 +671,7 @@ void CodeOperator::to_string_fwd( String& result )
 
 	if ( ast->Specs )
 	{
-		for ( SpecifierT spec : Specs )
+		for ( SpecifierT spec : ast->Specs )
 		{
 			if ( ! ESpecifier::is_trailing( spec ) )
 			{
@@ -698,7 +696,7 @@ void CodeOperator::to_string_fwd( String& result )
 
 	if ( ast->Specs )
 	{
-		for ( SpecifierT spec : Specs )
+		for ( SpecifierT spec : ast->Specs )
 		{
 			if ( ESpecifier::is_trailing( spec ) )
 			{
@@ -741,7 +739,7 @@ void CodeOpCast::to_string_def( String& result )
 		else
 			result.append_fmt( "operator %S()", ast->ValueType.to_string() );
 
-		for ( SpecifierT spec : Specs )
+		for ( SpecifierT spec : ast->Specs )
 		{
 			if ( ESpecifier::is_trailing( spec ) )
 			{
@@ -751,7 +749,7 @@ void CodeOpCast::to_string_def( String& result )
 		}
 
 		result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() );
-		break;
+		return;
 	}
 
 	if ( ast->Name && ast->Name.length() )
@@ -765,10 +763,10 @@ void CodeOpCast::to_string_fwd( String& result )
 	if ( ast->Specs )
 	{
 		// TODO : Add support for specifies before the operator keyword
-	
+
 		result.append_fmt( "operator %S()", ast->ValueType.to_string() );
-	
-		for ( SpecifierT spec : Specs )
+
+		for ( SpecifierT spec : ast->Specs )
 		{
 			if ( ESpecifier::is_trailing( spec ) )
 			{
@@ -776,14 +774,14 @@ void CodeOpCast::to_string_fwd( String& result )
 				result.append_fmt( " %*s", spec_str.Len, spec_str.Ptr );
 			}
 		}
-	
+
 		if ( ast->InlineCmt )
 			result.append_fmt( ";  %S", ast->InlineCmt->Content );
 		else
 			result.append( ";\n" );
-		break;
+		return;
 	}
-	
+
 	if ( ast->InlineCmt )
 		result.append_fmt("operator %S();  %S", ast->ValueType.to_string() );
 	else
@@ -799,10 +797,10 @@ String CodeParam::to_string()
 
 void CodeParam::to_string( String& result )
 {
-	if ( ast->ValueType == nullptr )
+	if ( ast->ValueType.ast == nullptr )
 	{
 		result.append_fmt( "%S", ast->Name );
-		break;
+		return;
 	}
 
 	if ( ast->Name )
@@ -835,7 +833,7 @@ String CodePreprocessCond::to_string()
 		case Preprocess_IfDef:
 			to_string_ifdef( result );
 		break;
-		case Preprocess_IfNotDef
+		case Preprocess_IfNotDef:
 			to_string_ifndef( result );
 		break;
 		case Preprocess_ElIf:
@@ -968,7 +966,7 @@ void CodeStruct::to_string_def( String& result )
 
 	result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() );
 
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 		result.append(";\n");
 }
 
@@ -982,7 +980,7 @@ void CodeStruct::to_string_fwd( String& result )
 
 	else result.append_fmt( "struct %S", ast->Name );
 
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 	{
 		if ( ast->InlineCmt )
 			result.append_fmt(";  %S", ast->InlineCmt->Content );
@@ -1029,14 +1027,14 @@ void CodeTypedef::to_string( String& result )
 	else
 		result.append_fmt( "%S %S", ast->UnderlyingType.to_string(), ast->Name );
 
-	if ( ast->UnderlyingType->Type == Typename && ast->UnderlyingType->ArrExpr )
+	if ( ast->UnderlyingType->Type == ECode::Typename && ast->UnderlyingType->ArrExpr )
 	{
 		result.append_fmt( "[ %S ];", ast->UnderlyingType->ArrExpr->to_string() );
 
 		AST* next_arr_expr = ast->UnderlyingType->ArrExpr->Next;
 		while ( next_arr_expr )
 		{
-			result.append_fmt( "[ %S ];", next_arr_expr.to_string() );
+			result.append_fmt( "[ %S ];", next_arr_expr->to_string() );
 			next_arr_expr = next_arr_expr->Next;
 		}
 	}
@@ -1082,13 +1080,13 @@ void CodeType::to_string( String& result )
 				result.append_fmt( "%S ", ast->Attributes.to_string() );
 			else
 			{
-				if ( Specs )
+				if ( ast->Specs )
 					result.append_fmt( "%S %S ( %S ) %S", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string(), ast->Specs.to_string() );
 				else
 					result.append_fmt( "%S %S ( %S )", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string() );
 			}
 
-			break;
+			return;
 		}
 	#endif
 
@@ -1136,7 +1134,7 @@ void CodeUnion::to_string( String& result )
 		);
 	}
 
-	if ( ast->Parent == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
+	if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) )
 		result.append(";\n");
 }
 
@@ -1236,7 +1234,7 @@ void CodeVar::to_string( String& result )
 		if ( ast->NextVar )
 			result.append_fmt( ", %S", ast->NextVar.to_string() );
 
-		break;
+		return;
 	}
 
 	if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ))
@@ -1278,13 +1276,13 @@ void CodeVar::to_string( String& result )
 		else
 			result.append( ";\n" );
 
-		break;
+		return;
 	}
 
 	if ( ast->BitfieldSize )
 		result.append_fmt( "%S %S : %S", ast->ValueType.to_string(), ast->Name, ast->BitfieldSize.to_string() );
 
-	else if ( ValueType->ArrExpr )
+	else if ( ast->ValueType->ArrExpr )
 	{
 		result.append_fmt( "%S %S[ %S ]", ast->ValueType.to_string(), ast->Name, ast->ValueType->ArrExpr.to_string() );
 
@@ -1299,15 +1297,15 @@ void CodeVar::to_string( String& result )
 	else
 		result.append_fmt( "%S %S", ast->ValueType.to_string(), ast->Name );
 
-	if ( Value )
+	if ( ast->Value )
 		result.append_fmt( " = %S", ast->Value.to_string() );
 
-	if ( NextVar )
+	if ( ast->NextVar )
 		result.append_fmt( ", %S", ast->NextVar.to_string() );
 
 	result.append( ";" );
 
-	if ( InlineCmt )
+	if ( ast->InlineCmt )
 		result.append_fmt("  %S", ast->InlineCmt->Content);
 	else
 		result.append("\n");

project/components/gen/ast_inlines.hpp

@@ -869,16 +869,6 @@ void CodeExtern::set_global()
 	rcast( AST*, ast )->Parent = Code::Global.ast;
 }
 
-String CodeExtern::to_string()
-{
-	if ( ast == nullptr )
-	{
-		log_failure( "Code::to_string: Cannot convert code to string, AST is null!" );
-		return { nullptr };
-	}
-	return rcast( AST*, ast )->to_string();
-}
-
 CodeExtern& CodeExtern::operator=( Code other )
 {
 	if ( other.ast && other->Parent )

project/components/gen/etoktype.cpp

@@ -5,7 +5,7 @@
 
 // This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp)
 
-namespace Parser
+namespace parser
 {
 	namespace ETokType
 	{
@@ -234,4 +234,4 @@ namespace Parser
 
 	using TokType = ETokType::Type;
 
-}    // namespace Parser
+}    // namespace parser

project/components/interface.cpp

@@ -3,8 +3,10 @@
 #include "code_serialization.cpp"
 #endif
 
-internal void init_parser();
-internal void deinit_parser();
+namespace parser {
+internal void init();
+internal void deinit();
+}
 
 internal
 void* Global_Allocator_Proc( void* allocator_data, AllocType type, sw size, sw alignment, void* old_memory, sw old_size, u64 flags )
@@ -288,7 +290,7 @@ void init()
 	}
 
 	define_constants();
-	init_parser();
+	parser::init();
 }
 
 void deinit()
@@ -331,7 +333,7 @@ void deinit()
 	while ( left--, left );
 
 	Global_AllocatorBuckets.free();
-	deinit_parser();
+	parser::deinit();
 }
 
 void reset()

project/dependencies/strings.hpp

@@ -169,7 +169,7 @@ struct String
 		get_header().Length = 0;
 	}
 
-	String duplicate( AllocatorInfo allocator )
+	String duplicate( AllocatorInfo allocator ) const
 	{
 		return make_length( allocator, Data, length() );
 	}

project/enums/ETokType_New.csv

@@ -0,0 +1,175 @@
+Invalid,                "__invalid__"
+Access_Private,         "private"
+Access_Protected,       "protected"
+Access_Public,          "public"
+Access_MemberSymbol,    "."
+Access_StaticSymbol,    "::"
+Ampersand,              "&"
+Ampersand_DBL,          "&&"
+Assign_Classifer,       ":"
+Attribute_Open,         "[["
+Attribute_Close,        "]]"
+BraceCurly_Open,        "{"
+BraceCurly_Close,       "}"
+BraceSquare_Open,       "["
+BraceSquare_Close,      "]"
+Capture_Start,          "("
+Capture_End,            ")"
+Comment,                "__comment__"
+Comment_End,            "__comment_end__"
+Comment_Start,          "__comment_start__"
+Char,                   "__character__"
+Comma,                  ","
+
+Decl_Class,             "class"
+Decl_GNU_Attribute,     "__attribute__"
+Decl_MSVC_Attribute,    "__declspec"
+Decl_Enum,              "enum"
+Decl_Extern_Linkage,    "extern"
+Decl_Friend,            "friend"
+Decl_Module,            "module"
+Decl_Namespace,         "namespace"
+Decl_Operator,          "operator"
+Decl_Struct,            "struct"
+Decl_Template,          "template"
+
+Decl_Type,              "decltype"
+
+Decl_Typedef,           "typedef"
+Decl_Using,             "using"
+Decl_Union,             "union"
+
+Identifier,             "__identifier__"
+
+Module_Import,          "import"
+Module_Export,          "export"
+
+NewLine,                "__new_line__"
+
+Number,                 "__number__"
+
+Operator,               "__operator__"
+
+Op_Assign,                    "="
+Op_Assign_Add,                "+="
+Op_Assign_Subtract,           "-="
+Op_Assign_Multiply,           "*="
+Op_Assign_Divide,             "/="
+Op_Assign_Modulo,             "%="
+Op_Assign_Bitwise_And,        "&="
+Op_Assign_Bitwise_Or,         "|="
+Op_Assign_Bitwise_XOr,        "^="
+Op_Assign_Bitwise_LeftShift,  "<<="
+Op_Assign_Bitwise_RightShift, ">>="
+
+Op_Increment, "++"
+Op_Decrement, "--"
+
+Op_Add,                 "+"
+Op_Subtract,            "-"
+Op_Multiply,            "*"
+Op_Divide,              "/"
+Op_Modulo,              "%"
+
+Op_Bitwise_And,         "&"
+Op_Bitwise_Or,          "|"
+Op_Bitwise_XOr,         "^"
+Op_Bitwise_LeftShitf,   "<<"
+Op_Bitwise_RightShift,  ">>"
+
+Op_UnaryAdd,            "+"
+Op_UnaryMinus,          "-"
+Op_UnaryNot,            "~"
+
+Op_Logical_Not,         "!"
+Op_Logical_And,         "&&"
+Op_Logical_Or,          "||"
+
+Op_Equal,               "=="
+Op_NotEqual,            "!="
+Op_Lesser,              "<"
+Op_Greater,             ">"
+Op_LesserEqual,         "<="
+Op_GreaterEqual",       ">=
+
+Op_Subscript,                "[]"
+Op_Indirection,              "*"
+Op_Address_Of,               "&"
+Op_MemberOfObject,           "."
+Op_MemberOfPointer",         "->"
+Op_PointerToMemberOfObject,  ".*"
+Op_PointerToMemberOfPointer, "->*"
+
+Op_Comma,                    ","
+Op_Ternary,                  "?"
+
+Preprocess_Hash,        "#"
+Preprocess_Define,      "define"
+Preprocess_If,          "if"
+Preprocess_IfDef,       "ifdef"
+Preprocess_IfNotDef,    "ifndef"
+Preprocess_ElIf,        "elif"
+Preprocess_Else,        "else"
+Preprocess_EndIf,       "endif"
+Preprocess_Include,     "include"
+Preprocess_Pragma,      "pragma"
+Preprocess_Content,	    "__macro_content__"
+Preprocess_Macro,       "__macro__"
+Preprocess_Unsupported, "__unsupported__"
+
+Spec_Alignof,           "alignof"
+Spec_Const,             "const"
+Spec_Consteval,         "consteval"
+Spec_Constexpr,         "constexpr"
+Spec_Constinit,         "constinit"
+Spec_Explicit,          "explicit"
+Spec_Extern,            "extern"
+Spec_Final,             "final"
+Spec_ForceInline,	    "forceinline"
+Spec_Global,            "global"
+Spec_Inline,            "inline"
+Spec_Internal_Linkage,  "internal"
+Spec_LocalPersist,      "local_persist"
+Spec_Mutable,           "mutable"
+Spec_NeverInline,       "neverinline"
+Spec_Override,          "override"
+Spec_Static,            "static"
+Spec_ThreadLocal,       "thread_local"
+Spec_Volatile,          "volatile"
+Spec_Virtual,           "virtual"
+
+Star,                   "*"
+
+Stmt_Break,             "break"
+Stmt_Case,              "case"
+Stmt_Continue,          "continue"
+Stmt_Do,                "do"
+Stmt_Else,              "else"
+Stmt_End,               ";"
+Stmt_If,                "if"
+Stmt_For,               "for"
+Stmt_Return,            "return"
+Stmt_Switch,            "switch"
+Stmt_While,             "while"
+
+StaticAssert,           "static_assert"
+String,                 "__string__"
+
+Type_Unsigned,          "unsigned"
+Type_Signed,            "signed"
+Type_Short,             "short"
+Type_Long,              "long"
+Type_bool,              "bool"
+Type_char,              "char"
+Type_int,               "int"
+Type_float,             "float"
+Type_double,            "double"
+Type_MS_int8,           "__int8"
+Type_MS_int16,          "__int16"
+Type_MS_int32,          "__int32"
+Type_MS_int64,          "__int64"
+Type_MS_W64,            "_W64"
+
+Varadic_Argument,       "..."
+
+__Attributes_Start,     "__attrib_start__"

project/gen.cpp

@@ -25,6 +25,8 @@ GEN_NS_BEGIN
 #include "components/interface.cpp"
 #include "components/interface.upfront.cpp"
 #include "components/gen/etoktype.cpp"
+#include "components/lexer.cpp"
+#include "components/parser.cpp"
 #include "components/interface.parsing.cpp"
 #include "components/interface.untyped.cpp"
 

project/helpers/helper.hpp

@@ -379,16 +379,6 @@ CodeBody gen_ast_inlines()
 
 			rcast(AST*, ast)->Parent = Code::Global.ast;
 		}
-//		String <typename>::to_string()
-//		{
-//			if ( ast == nullptr )
-//			{
-//			log_failure("Code::to_string: Cannot convert code to string, AST is null!");
-//			return { nullptr };
-//			}
-//
-//			return rcast(AST*, ast)->to_string();
-//		}
 		<typename>& <typename>::operator =( Code other )
 		{
 			if ( other.ast && other->Parent )

singleheader/singleheader.cpp

@@ -187,7 +187,7 @@ int gen_main()
 			if ( generate_scanner )
 			{
 				header.print_fmt( "\n#pragma region Parsing\n" );
-			header.print( scan_file( project_dir "dependencies/parsing.cpp" ) );
+				header.print( scan_file( project_dir "dependencies/parsing.cpp" ) );
 				header.print_fmt( "#pragma endregion Parsing\n\n" );
 			}
 
@@ -195,14 +195,16 @@ int gen_main()
 			header.print_fmt( roll_own_dependencies_guard_end );
 		}
 
-		Code static_data 	 = scan_file( project_dir "components/static_data.cpp" );
-		Code ast_case_macros = scan_file( project_dir "components/ast_case_macros.cpp" );
-		Code ast             = scan_file( project_dir "components/ast.cpp" );
-		Code code            = scan_file( project_dir "components/code_serialization.cpp" );
-		Code interface       = scan_file( project_dir "components/interface.cpp" );
-		Code upfront         = scan_file( project_dir "components/interface.upfront.cpp" );
-		Code parsing         = scan_file( project_dir "components/interface.parsing.cpp" );
-		Code untyped         = scan_file( project_dir "components/interface.untyped.cpp" );
+		Code static_data 	   = scan_file( project_dir "components/static_data.cpp" );
+		Code ast_case_macros   = scan_file( project_dir "components/ast_case_macros.cpp" );
+		Code ast               = scan_file( project_dir "components/ast.cpp" );
+		Code code              = scan_file( project_dir "components/code_serialization.cpp" );
+		Code interface         = scan_file( project_dir "components/interface.cpp" );
+		Code upfront           = scan_file( project_dir "components/interface.upfront.cpp" );
+		Code lexer             = scan_file( project_dir "components/lexer.cpp" );
+		Code parser            = scan_file( project_dir "components/parser.cpp" );
+		Code parsing_interface = scan_file( project_dir "components/interface.parsing.cpp" );
+		Code untyped           = scan_file( project_dir "components/interface.untyped.cpp" );
 
 		CodeBody etoktype      = gen_etoktype( project_dir "enums/ETokType.csv", project_dir "enums/AttributeTokens.csv" );
 		CodeNS   parser_nspace = def_namespace( name(Parser), def_namespace_body( args(etoktype)) );
@@ -221,7 +223,9 @@ int gen_main()
 		header.print( upfront );
 		header.print_fmt( "\n#pragma region Parsing\n\n" );
 		header.print( parser_nspace );
-		header.print( parsing );
+		header.print( lexer );
+		header.print( parser );
+		header.print( parsing_interface );
 		header.print_fmt( "\n#pragma endregion Parsing\n" );
 		header.print( untyped );
 		header.print_fmt( "\n#pragma endregion Interface\n\n");