Library can now construct into AST and serialization itself (singleheader).

Still need to validate if they match.

.gitignore

@@ -6,12 +6,12 @@ build/*
 **/*.gen.*
 **/gen/gen.hpp
 **/gen/gen.cpp
-**/gen/gen_dep.hpp
+**/gen/gen.dep.hpp
-**/gen/gen_dep.cpp
+**/gen/gen.dep.cpp
-**/gen/gen_builder.hpp
+**/gen/gen.builder.hpp
-**/gen/gen_builder.cpp
+**/gen/gen.builder.cpp
-**/gen/gen_scanner.hpp
+**/gen/gen.scanner.hpp
-**/gen/gen_scanner.cpp
+**/gen/gen.scanner.cpp
 
 gencpp.hpp
 gencpp.cpp

.vscode/c_cpp_properties.json

@@ -14,6 +14,7 @@
             "windowsSdkVersion": "10.0.19041.0",
             "compilerPath": "C:/Users/Ed/scoop/apps/llvm/current/bin/clang++.exe",
             "intelliSenseMode": "windows-clang-x64",
+            "compileCommands": "${workspaceFolder}/project/build/compile_commands.json"
         }
     ],
     "version": 4

project/components/ast.cpp

@@ -29,6 +29,10 @@ String AST::to_string()
 			log_failure("Attempted to serialize invalid code! - %s", Parent ? Parent->debug_str() : Name );
 		break;
 
+		case NewLine:
+			result.append("\n");
+		break;
+
 		case Untyped:
 		case Execution:
 			result.append( Content );
@@ -257,7 +261,7 @@ String AST::to_string()
 			s32  left = NumEntries;
 			while ( left-- )
 			{
-				result.append_fmt( "%s\n", curr.to_string() );
+				result.append_fmt( "%s", curr.to_string() );
 				++curr;
 			}
 
@@ -455,9 +459,9 @@ String AST::to_string()
 			if ( Specs )
 			{
 				if ( Name && Name.length() )
-					result.append_fmt( "%.*soperator %s()", Name.length(), Name, EOperator::to_str( Op ));
+					result.append_fmt( "%.*soperator %s()", Name.length(), Name, ValueType->to_string() );
 				else
-					result.append_fmt( "operator %s()", EOperator::to_str( Op ) );
+					result.append_fmt( "operator %s()", ValueType->to_string() );
 
 				CodeSpecifiers specs = cast<CodeSpecifiers>();
 
@@ -524,9 +528,8 @@ String AST::to_string()
 			}
 		}
 		break;
-
 		case Preprocess_Define:
-			result.append_fmt( "#define %s \\\n%s\n", Name, Content );
+			result.append_fmt( "#define %s %s", Name, Content );
 		break;
 
 		case Preprocess_If:
@@ -550,11 +553,11 @@ String AST::to_string()
 		break;
 
 		case Preprocess_Else:
-			result.append_fmt( "#else" );
+			result.append_fmt( "\n#else" );
 		break;
 
 		case Preprocess_EndIf:
-			result.append_fmt( "#endif\n" );
+			result.append_fmt( "#endif" );
 		break;
 
 		case Preprocess_Pragma:
@@ -796,8 +799,21 @@ String AST::to_string()
 		}
 		break;
 
-		case Class_Body:
+#if 0
+		{
+			Code curr = Front->cast<Code>();
+			s32  left = NumEntries;
+			while ( left -- )
+			{
+				result.append_fmt( "%s", curr.to_string() );
+				++curr;
+			}
+		}
+		break;
+#endif
+
 		case Enum_Body:
+		case Class_Body:
 		case Extern_Linkage_Body:
 		case Function_Body:
 		case Global_Body:
@@ -809,7 +825,11 @@ String AST::to_string()
 			s32  left = NumEntries;
 			while ( left -- )
 			{
-				result.append_fmt( "%s\n", curr.to_string() );
+				result.append_fmt( "%s", curr.to_string() );
+
+				if ( curr->Type != ECode::NewLine )
+					result.append( "\n" );
+
 				++curr;
 			}
 		}

project/components/header_end.hpp

@@ -57,6 +57,9 @@ extern CodeAttributes attrib_api_import;
 extern Code module_global_fragment;
 extern Code module_private_fragment;
 
+// Exposed, but this is really used for parsing.
+extern Code fmt_newline;
+
 extern CodePragma pragma_once;
 
 extern CodeParam param_varadic;

project/components/interface.cpp

@@ -112,6 +112,10 @@ void define_constants()
 	module_private_fragment->Content = module_private_fragment->Name;
 	module_private_fragment.set_global();
 
+	fmt_newline = make_code();
+	fmt_newline->Type = ECode::NewLine;
+	fmt_newline.set_global();
+
 	pragma_once          = (CodePragma) make_code();
 	pragma_once->Type    = ECode::Untyped;
 	pragma_once->Name    = get_cached_string( txt_StrC("once") );

project/components/interface.parsing.cpp

@@ -63,20 +63,29 @@ namespace Parser
 
 		bool __eat( TokType type );
 
-		Token& current()
+		Token& current( bool skip_new_lines = true )
 		{
-			while ( Arr[Idx].Type == TokType::Empty_Line )
+			if ( skip_new_lines )
+			{
+				while ( Arr[Idx].Type == TokType::NewLine )
 					Idx++;
+			}
 
 			return Arr[Idx];
 		}
 
-		Token& previous()
+		Token& previous( bool skip_new_lines = false )
 		{
 			s32 idx = this->Idx;
-			while ( Arr[Idx].Type == TokType::Empty_Line )
+
+			if ( skip_new_lines )
+			{
+				while ( Arr[idx].Type == TokType::NewLine )
 					idx--;
 
+				return Arr[idx];
+			}
+
 			return Arr[idx - 1];
 		}
 
@@ -86,6 +95,8 @@ namespace Parser
 		}
 	};
 
+	constexpr bool dont_skip_new_lines = false;
+
 	struct StackNode
 	{
 		StackNode* Prev;
@@ -145,7 +156,7 @@ namespace Parser
 			{
 				if ( curr_scope->Name )
 				{
-					result.append_fmt("\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Length, (StrC)curr_scope->Name );
+					result.append_fmt("\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Length, curr_scope->Name.Text );
 				}
 				else
 				{
@@ -170,19 +181,16 @@ namespace Parser
 			return false;
 		}
 
-		if ( Arr[Idx].Type == TokType::Empty_Line && type != TokType::Empty_Line )
+		if ( Arr[Idx].Type == TokType::NewLine && type != TokType::NewLine )
 		{
 			Idx++;
-			return log_fmt( "Auto-skipping empty line (%d, %d)\n", current().Line, current().Column );
 		}
 
 		if ( Arr[Idx].Type != type )
 		{
-			String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } );
+			log_failure( "Parse Error, TokArray::eat, Expected: ' %s ' not ' %.*s ' (%d, %d)`\n%s"
-
+				, ETokType::to_str(type).Ptr
-			log_failure( "Parse Error, TokArray::eat, Expected: ' %s ' not ' %s ' (%d, %d)`\n%s"
+				, Arr[Idx].Length, Arr[Idx].Text
-				, ETokType::to_str(type)
-				, token_str
 				, current().Line
 				, current().Column
 				, Context.to_string()
@@ -255,27 +263,31 @@ namespace Parser
 
 		while (left )
 		{
-			Token token = { nullptr, 0, TokType::Invalid, line, column, false };
+			Token token = { scanner, 0, TokType::Invalid, line, column, false };
 
 			bool is_define = false;
 
 			if ( column == 1 )
 			{
-				token.Text = scanner;
-
 				if ( current == '\r')
+				{
+					move_forward();
 					token.Length = 1;
+				}
 
 				if ( current == '\n' )
 				{
-					token.Type = TokType::Empty_Line;
+					token.Type = TokType::NewLine;
 					token.Length ++;
+					move_forward();
 
 					Tokens.append( token );
 					continue;
 				}
 			}
 
+			token.Length = 0;
+
 			SkipWhitespace();
 			if ( left <= 0 )
 				break;
@@ -321,27 +333,30 @@ namespace Parser
 								if ( current == '\r' )
 								{
 									move_forward();
-									// token.Length++;
+									token.Length++;
 								}
 
 								if ( current == '\n' )
 								{
 									move_forward();
-									// token.Length++;
+									token.Length++;
 									continue;
 								}
 								else
 								{
-									String directive_str = String::make_length( GlobalAllocator, token.Text, token.Length );
-
 									log_failure( "gen::Parser::lex: Invalid escape sequence '\\%c' (%d, %d)"
-												" in preprocessor directive (%d, %d)\n%s"
+												" in preprocessor directive (%d, %d)\n%.100s"
 										, current, line, column
-										, token.Line, token.Column, directive_str );
+										, token.Line, token.Column, token.Text );
 									break;
 								}
 							}
 
+							if ( current == '\r' )
+							{
+								move_forward();
+							}
+
 							if ( current == '\n' )
 							{
 								move_forward();
@@ -353,8 +368,8 @@ namespace Parser
 							token.Length++;
 						}
 
+						token.Length = token.Length + token.Text - hash;
 						token.Text   = hash;
-						token.Length = (sptr)token.Text + token.Length - (sptr)hash;
 						Tokens.append( token );
 						continue; // Skip found token, its all handled here.
 					}
@@ -817,11 +832,22 @@ namespace Parser
 							token.Text   = scanner;
 							token.Length = 0;
 
-							while ( left && current != '\n' )
+							while ( left && current != '\n' && current != '\r' )
 							{
 								move_forward();
 								token.Length++;
 							}
+
+							if ( current == '\r' )
+							{
+								move_forward();
+							}
+
+							if ( current == '\n' )
+							{
+								move_forward();
+								// token.Length++;
+							}
 						}
 						else if ( current == '*' )
 						{
@@ -986,6 +1012,17 @@ namespace Parser
 					move_forward();
 					token.Length++;
 				}
+
+				if ( current == '\r' )
+				{
+					move_forward();
+					// token.Length++;
+				}
+				if ( current == '\n' )
+				{
+					move_forward();
+					// token.Length++;
+				}
 			}
 			else
 			{
@@ -1042,16 +1079,14 @@ if ( def.Ptr == nullptr )                                                      \
 	return CodeInvalid;                                                        \
 }
 
+#	define currtok_noskip Context.Tokens.current( dont_skip_new_lines )
 #	define currtok        Context.Tokens.current()
 #	define prevtok        Context.Tokens.previous()
 #	define eat( Type_ )   Context.Tokens.__eat( Type_ )
 #	define left           ( Context.Tokens.Arr.num() - Context.Tokens.Idx )
 
-#	define check( Type_ )                     \
+#	define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ )
-	( left                                    \
+#	define check( Type_ )        ( left && currtok.Type == Type_ )
-	&& (currtok.Type == TokType::Empty_Line ? \
-		eat( TokType::Empty_Line) : true)     \
-	&& currtok.Type == Type_ )
 
 #	define push_scope()                                                    \
 	StackNode scope { nullptr, currtok, NullToken, txt_StrC( __func__ ) }; \
@@ -2415,7 +2450,7 @@ CodeBody parse_class_struct_body( Parser::TokType which )
 	else
 		result->Type = Struct_Body;
 
-	while ( left && currtok.Type != TokType::BraceCurly_Close )
+	while ( left && currtok_noskip.Type != TokType::BraceCurly_Close )
 	{
 		Code           member     = Code::Invalid;
 		CodeAttributes attributes = { nullptr };
@@ -2423,12 +2458,13 @@ CodeBody parse_class_struct_body( Parser::TokType which )
 
 		bool expects_function = false;
 
-		switch ( currtok.Type )
+		Context.Scope->Start = currtok_noskip;
+
+		switch ( currtok_noskip.Type )
 		{
-			case TokType::Empty_Line:
+			case TokType::NewLine:
-				// Empty lines are auto skipped by Tokens.current()
+				member = fmt_newline;
-				member = untyped_str( Context.Tokens.Arr[ Context.Tokens.Idx] );
+				eat( TokType::NewLine );
-				eat( TokType::Empty_Line );
 			break;
 
 			case TokType::Comment:
@@ -2773,7 +2809,7 @@ CodeBody parse_global_nspace( CodeT which )
 	result = (CodeBody) make_code();
 	result->Type = which;
 
-	while ( left && currtok.Type != TokType::BraceCurly_Close )
+	while ( left && currtok_noskip.Type != TokType::BraceCurly_Close )
 	{
 		Code           member     = Code::Invalid;
 		CodeAttributes attributes = { nullptr };
@@ -2781,14 +2817,14 @@ CodeBody parse_global_nspace( CodeT which )
 
 		bool expects_function = false;
 
-		Context.Scope->Start = currtok;
+		Context.Scope->Start = currtok_noskip;
 
-		switch ( currtok.Type )
+		switch ( currtok_noskip.Type )
 		{
-			case TokType::Empty_Line:
+			case TokType::NewLine:
 				// Empty lines are auto skipped by Tokens.current()
-				member = untyped_str( Context.Tokens.Arr[ Context.Tokens.Idx] );
+				member = fmt_newline;
-				eat( TokType::Empty_Line );
+				eat( TokType::NewLine );
 			break;
 
 			case TokType::Comment:
@@ -3103,14 +3139,13 @@ CodeEnum parse_enum( bool inplace_def )
 
 		Code member = CodeInvalid;
 
-		while ( currtok.Type != TokType::BraceCurly_Close )
+		while ( left && currtok_noskip.Type != TokType::BraceCurly_Close )
 		{
-			switch ( currtok.Type )
+			switch ( currtok_noskip.Type )
 			{
-				case TokType::Empty_Line:
+				case TokType::NewLine:
-					// Empty lines are auto skipped by Tokens.current()
+					member = untyped_str( currtok_noskip );
-					member = untyped_str( Context.Tokens.Arr[ Context.Tokens.Idx] );
+					eat( TokType::NewLine );
-					eat( TokType::Empty_Line );
 				break;
 
 				case TokType::Comment:
@@ -3160,9 +3195,9 @@ CodeEnum parse_enum( bool inplace_def )
 					{
 						eat( TokType::Operator );
 
-						while ( currtok.Type != TokType::Comma && currtok.Type != TokType::BraceCurly_Close )
+						while ( currtok_noskip.Type != TokType::Comma && currtok_noskip.Type != TokType::BraceCurly_Close )
 						{
-							eat( currtok.Type );
+							eat( currtok_noskip.Type );
 						}
 					}
 
@@ -3653,15 +3688,15 @@ CodeOpCast parse_operator_cast()
 			if ( currtok.Type == TokType::BraceCurly_Open )
 				level++;
 
-			else if ( currtok.Type == TokType::BraceCurly_Close && level > 0 )
+			else if ( currtok.Type == TokType::BraceCurly_Close )
 				level--;
 
 			eat( currtok.Type );
 		}
-		eat( TokType::BraceCurly_Close );
-
 		body_str.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)body_str.Text;
 
+		eat( TokType::BraceCurly_Close );
+
 		body = untyped_str( body_str );
 	}
 	else
@@ -4175,15 +4210,15 @@ CodeUnion parse_union( bool inplace_def )
 	body = make_code();
 	body->Type = ECode::Union_Body;
 
-	while ( ! check( TokType::BraceCurly_Close ) )
+	while ( ! check_noskip( TokType::BraceCurly_Close ) )
 	{
 		Code member = { nullptr };
-		switch ( currtok.Type )
+		switch ( currtok_noskip.Type )
 		{
-			case TokType::Empty_Line:
+			case TokType::NewLine:
 				// Empty lines are auto skipped by Tokens.current()
-				member = untyped_str( Context.Tokens.Arr[ Context.Tokens.Idx] );
+				member = fmt_newline;
-				eat( TokType::Empty_Line );
+				eat( TokType::NewLine );
 			break;
 
 			case TokType::Comment:

project/components/interface.upfront.cpp

@@ -812,8 +812,13 @@ CodeOperator def_operator( OperatorT op, StrC nspace
 		return CodeInvalid;
 	}
 
-	char const* name = str_fmt_buf( "%.*soperator %s", nspace.Len, nspace.Ptr, to_str(op) );
+	char const* name = nullptr;
 
+	StrC op_str = to_str( op );
+	if ( nspace.Len > 0 )
+		name = str_fmt_buf( "%.*soperator %.*s", nspace.Len, nspace.Ptr, op_str.Len, op_str.Ptr );
+	else
+		name = str_fmt_buf( "operator %.*s", op_str.Len, op_str.Ptr );
 	CodeOperator
 	result              = (CodeOperator) make_code();
 	result->Name        = get_cached_string( { str_len(name), name } );

project/components/src_start.cpp

@@ -2,6 +2,8 @@
 #	error Gen.hpp : GEN_TIME not defined
 #endif
 
+#include "gen.hpp"
+
 //! If its desired to roll your own dependencies, define GEN_ROLL_OWN_DEPENDENCIES before including this file.
 //! Dependencies are derived from the c-zpl library: https://github.com/zpl-c/zpl
 #ifndef GEN_ROLL_OWN_DEPENDENCIES

project/components/static_data.cpp

@@ -32,6 +32,8 @@ global CodeAttributes attrib_api_import;
 global Code module_global_fragment;
 global Code module_private_fragment;
 
+global Code fmt_newline;
+
 global CodeParam param_varadic;
 
 global CodePragma pragma_once;

project/components/temp/ecode.hpp

@@ -5,6 +5,7 @@ namespace ECode
 #	define Define_Types           \
 	Entry( Invalid )              \
 	Entry( Untyped )              \
+	Entry( NewLine )              \
 	Entry( Comment )              \
 	Entry( Access_Private )       \
 	Entry( Access_Protected )     \

project/components/temp/eoperator.hpp

@@ -57,14 +57,14 @@ namespace EOperator
 	};
 
 	inline
-	char const* to_str( Type op )
+	StrC to_str( Type op )
 	{
 		local_persist
-		char const* lookup[ Num_Ops ] = {
+		StrC lookup[ Num_Ops ] = {
-		#	define Entry( Type_, Token_ ) stringize(Token_),
+		#	define Entry( Type_, Token_ ) { sizeof(stringize(Token_)), stringize(Token_) },
 			Define_Operators
 		#	undef Entry
-			","
+			txt_StrC(",")
 		};
 
 		return lookup[ op ];

project/components/temp/etoktype.cpp

@@ -50,10 +50,10 @@ namespace Parser
 	Entry( Decl_Typedef,           "typedef" )          \
 	Entry( Decl_Using,             "using" )            \
 	Entry( Decl_Union,             "union" )            \
-	Entry( Empty_Line,             "__empty_line__" )   \
 	Entry( Identifier,             "__identifier__" )   \
 	Entry( Module_Import,          "import" )           \
 	Entry( Module_Export,          "export" )           \
+	Entry( NewLine,                "__NewLine__" )      \
 	Entry( Number,                 "__number__" )       \
 	Entry( Operator,               "__operator__" )     \
 	Entry( Preprocess_Define,      "define")            \
@@ -144,12 +144,12 @@ namespace Parser
 		}
 
 		internal inline
-		char const* to_str( Type type )
+		StrC to_str( Type type )
 		{
 			local_persist
-			char const* lookup[(u32)NumTokens] =
+			StrC lookup[(u32)NumTokens] =
 			{
-			#	define Entry( Name_, Str_ ) Str_,
+			#	define Entry( Name_, Str_ ) { sizeof(Str_), Str_ },
 				Define_TokType
 				GEN_DEFINE_ATTRIBUTE_TOKENS
 			#	undef Entry

project/enums/ECode.csv

@@ -1,5 +1,6 @@
 Invalid
 Untyped
+NewLine
 Comment
 Access_Private
 Access_Protected

project/enums/ETokType.csv

@@ -26,16 +26,16 @@ Decl_Extern_Linkage,    "extern"
 Decl_Friend,            "friend"
 Decl_Module,            "module"
 Decl_Namespace,         "namespace"
-Decl_Operator,          "__operator__"
+Decl_Operator,          "operator"
 Decl_Struct,            "struct"
 Decl_Template,          "template"
 Decl_Typedef,           "typedef"
 Decl_Using,             "using"
 Decl_Union,             "union"
-Empty_Line,             "__empty_line__"
 Identifier,             "__identifier__"
 Module_Import,          "import"
 Module_Export,          "export"
+NewLine,                "__new_line__"
 Number,                 "__number__"
 Operator,               "__operator__"
 Preprocess_Define,      "define"
@@ -47,6 +47,7 @@ Preprocess_Else,        "else"
 Preprocess_EndIf,       "endif"
 Preprocess_Include,     "include"
 Preprocess_Pragma,      "pragma"
+Preprocess_Content,	    "__macro_content__"
 Preprocess_Macro,       "__macro__"
 Preprocess_Unsupported, "__unsupported__"
 Spec_Alignas,           "alignas"

project/gen.bootstrap.cpp

@@ -42,10 +42,9 @@ int gen_main()
 		Code timing        = scan_file( "dependencies/timing.hpp" );
 
 		Builder
-		deps_header = Builder::open("gen/gen_dep.hpp");
+		deps_header = Builder::open("gen/gen.dep.hpp");
 		deps_header.print_fmt( generation_notice );
 		deps_header.print_fmt( "// This file is intended to be included within gen.hpp (There is no pragma diagnostic ignores)\n\n" );
-		deps_header.print_fmt( "#pragma once\n\n" );
 		deps_header.print( header_start );
 		deps_header.print_fmt( "GEN_NS_BEGIN\n\n" );
 
@@ -78,7 +77,7 @@ int gen_main()
 		Code timing        = scan_file( "dependencies/timing.cpp" );
 
 		Builder
-		deps_impl = Builder::open( "gen/gen_dep.cpp" );
+		deps_impl = Builder::open( "gen/gen.dep.cpp" );
 		deps_impl.print_fmt( generation_notice );
 		deps_impl.print_fmt( "// This file is intended to be included within gen.cpp (There is no pragma diagnostic ignores)\n\n" );
 		deps_impl.print( src_start );
@@ -148,7 +147,6 @@ int gen_main()
 	// gen.cpp
 	{
 		Code        src_start       = scan_file( "components/src_start.cpp" );
-		CodeInclude header          = def_include( txt_StrC("gen.hpp") );
 		Code        static_data 	= scan_file( "components/static_data.cpp" );
 		Code        ast_case_macros = scan_file( "components/ast_case_macros.cpp" );
 		Code        ast			    = scan_file( "components/ast.cpp" );
@@ -165,7 +163,6 @@ int gen_main()
 		src.print_fmt( generation_notice );
 		src.print( push_ignores );
 		src.print( src_start );
-		src.print( header );
 		src.print_fmt( "\nGEN_NS_BEGIN\n\n");
 
 		src.print( static_data );
@@ -195,8 +192,9 @@ int gen_main()
 		Code builder = scan_file( "file_processors/builder.hpp" );
 
 		Builder
-		header = Builder::open( "gen/gen_builder.hpp" );
+		header = Builder::open( "gen/gen.builder.hpp" );
 		header.print_fmt( generation_notice );
+		header.print_fmt( "#pragma once\n\n" );
 		header.print( def_include( txt_StrC("gen.hpp") ));
 		header.print_fmt( "\nGEN_NS_BEGIN\n\n" );
 		header.print( builder );
@@ -209,9 +207,9 @@ int gen_main()
 		Code builder = scan_file( "file_processors/builder.cpp" );
 
 		Builder
-		src = Builder::open( "gen/gen_builder.cpp" );
+		src = Builder::open( "gen/gen.builder.cpp" );
 		src.print_fmt( generation_notice );
-		src.print( def_include( txt_StrC("gen_builder.hpp") ) );
+		src.print( def_include( txt_StrC("gen.builder.hpp") ) );
 		src.print_fmt( "\nGEN_NS_BEGIN\n\n" );
 		src.print( builder );
 		src.print_fmt( "\nGEN_NS_END\n\n" );
@@ -224,7 +222,9 @@ int gen_main()
 		Code scanner = scan_file( "file_processors/scanner.hpp" );
 
 		Builder
-		header = Builder::open( "gen/gen_scanner.hpp" );
+		header = Builder::open( "gen/gen.scanner.hpp" );
+		header.print_fmt( generation_notice );
+		header.print_fmt( "#pragma once\n\n" );
 		header.print( def_include( txt_StrC("gen.hpp") ) );
 		header.print_fmt( "\nGEN_NS_BEGIN\n\n" );
 		header.print( parsing );
@@ -239,8 +239,9 @@ int gen_main()
 		// Code scanner = scan_file( "file_processors/scanner.cpp" );
 
 		Builder
-		src = Builder::open( "gen/gen_scanner.cpp" );
+		src = Builder::open( "gen/gen.scanner.cpp" );
-		src.print( def_include( txt_StrC("gen_scanner.hpp") ) );
+		src.print_fmt( generation_notice );
+		src.print( def_include( txt_StrC("gen.scanner.hpp") ) );
 		src.print_fmt( "\nGEN_NS_BEGIN\n\n" );
 		src.print( parsing );
 		// src.print( scanner );

project/gen.cpp

@@ -6,14 +6,14 @@
 #	error Gen.hpp : GEN_TIME not defined
 #endif
 
+#include "gen.hpp"
+
 //! If its desired to roll your own dependencies, define GEN_ROLL_OWN_DEPENDENCIES before including this file.
 //! Dependencies are derived from the c-zpl library: https://github.com/zpl-c/zpl
 #ifndef GEN_ROLL_OWN_DEPENDENCIES
 #	include "gen.dep.cpp"
 #endif
 
-#include "gen.hpp"
-
 GEN_NS_BEGIN
 
 #include "components/static_data.cpp"

project/helpers/helper.hpp

@@ -256,6 +256,8 @@ CodeBody gen_etoktype( char const* etok_path, char const* attr_path )
 
 		if ( idx < attribute_strs.num() - 1 )
 			attribute_define_entries.append( " \\\n");
+		else
+			attribute_define_entries.append( "\n");
 	}
 
 #pragma push_macro( "GEN_DEFINE_ATTRIBUTE_TOKENS" )

scripts/bootstrap.ci.ps1

@@ -62,9 +62,9 @@ Push-location $path_project
 
 	$include = @(
 		'gen.hpp', 'gen.cpp',
-		'gen_dep.hpp', 'gen_dep.cpp',
+		'gen.dep.hpp', 'gen.dep.cpp',
-		'gen_builder.hpp', 'gen_builder.cpp'
+		'gen.builder.hpp', 'gen.builder.cpp'
-		'gen_scanner.hpp', 'gen_scanner.cpp'
+		'gen.scanner.hpp', 'gen.scanner.cpp'
 	)
 	$exclude = $null
 

singleheader/gen.singleheader.cpp

@@ -99,9 +99,9 @@ int gen_main()
 
 			if ( generate_scanner )
 			{
-				header.print_fmt( "pragma region Parsing\n\n" );
+				header.print_fmt( "#pragma region Parsing\n\n" );
 				header.print( scan_file( project_dir "dependencies/parsing.hpp" ) );
-				header.print_fmt( "pragma endregion Parsing\n\n" );
+				header.print_fmt( "#pragma endregion Parsing\n\n" );
 			}
 
 			header.print_fmt( "GEN_NS_END\n" );

test/SOA.cpp

@@ -1,9 +1,9 @@
-#if GEN_TIME
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
 #define GEN_EXPOSE_BACKEND
 #define GEN_BENCHMARK
-#include "gen.hpp"
+#include "gen/gen.hpp"
+#include "gen/gen.builder.hpp"
 using namespace gen;
 
 Code gen_SOA( CodeStruct struct_def, s32 num_entries = 0 )
@@ -118,7 +118,7 @@ void check_SOA()
 {
 	log_fmt("\ncheck_SOA:");
 	gen::init();
-	Builder soa_test; soa_test.open( "SOA.gen.hpp" );
+	Builder soa_test = Builder::open( "SOA.gen.hpp" );
 
 	soa_test.print( parse_using( code(
 		using u16 = unsigned short;
@@ -142,4 +142,3 @@ void check_SOA()
 	gen::deinit();
 	log_fmt(" passed!\n");
 }
-#endif

test/sanity.cpp

@@ -1,11 +1,11 @@
 // Testing to make sure backend of library is operating properly.
 
-#ifdef GEN_TIME
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
 #define GEN_EXPOSE_BACKEND
 #define GEN_BENCHMARK
-#include "gen.hpp"
+#include "gen/gen.hpp"
+#include "gen/gen.builder.hpp"
 
 void check_sanity()
 {
@@ -89,4 +89,3 @@ void check_sanity()
 	gen::deinit();
 	log_fmt("\nSanity passed!\n");
 }
-#endif

test/test.cpp

@@ -2,7 +2,8 @@
 #define GEN_ENFORCE_STRONG_CODE_TYPES
 #define GEN_EXPOSE_BACKEND
 #define GEN_BENCHMARK
-#include "gen.cpp"
+#include "gen/gen.cpp"
+#include "gen/gen.builder.cpp"
 #include "sanity.cpp"
 #include "SOA.cpp"
 #include "test.singleheader_ast.cpp"
@@ -15,7 +16,7 @@ int gen_main()
 
 	// check_sanity();
 
-	check_SOA();
+	// check_SOA();
 
 	check_singleheader_ast();
 

test/test.singleheader_ast.cpp

@@ -1,11 +1,10 @@
-#pragma once
-
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRONG_CODE_TYPES
 #define GEN_EXPOSE_BACKEND
 #define GEN_BENCHMARK
-#include "gen.hpp"
+#include "gen/gen.hpp"
-#include "file_processors/scanner.hpp"
+#include "gen/gen.builder.hpp"
+#include "gen/gen.scanner.hpp"
 using namespace gen;
 
 void check_singleheader_ast()
@@ -20,6 +19,7 @@ void check_singleheader_ast()
 
 	log_fmt("generated AST!!!\n");
 
+#if 0
 	s32 idx = 0;
 	for ( Code entry : ast )
 	{
@@ -30,9 +30,9 @@ void check_singleheader_ast()
 		log_fmt("Entry %d: %s\n", idx, entry.to_string() );
 		idx++;
 	}
+#endif
 
-	Builder builder;
+	Builder builder = Builder::open( "singleheader_copy.gen.hpp" );
-	builder.open( "singleheader_copy.gen.hpp" );
 	log_fmt("\n\nserializng ast\n");
 	builder.print( ast );
 	builder.write();