Single header generates again, some more cleanup.

Looking into properly dealing with empty lines...

I want to preserve the text's empty lines in the AST for serialization purposes (perserve formatting for gapes between definitions).
Don't want to introduce the possibility of it breaking though, so will have to ignore empty_lines in a general way (if they are in a bad spot).
Attempted to cover that by having TokArray::current() auto-skip empty lines and eat as well if the type doesn't match.

.gitignore

@@ -8,6 +8,10 @@ build/*
 **/gen/gen.cpp
 **/gen/gen_dep.hpp
 **/gen/gen_dep.cpp
+**/gen/gen_builder.hpp
+**/gen/gen_builder.cpp
+**/gen/gen_scanner.hpp
+**/gen/gen_scanner.cpp
 
 gencpp.hpp
 gencpp.cpp

project/components/ast.cpp

@@ -1,5 +1,3 @@
-#pragma region AST
-
 Code Code::Global;
 Code Code::Invalid;
 
@@ -916,6 +914,3 @@ bool AST::validate_body()
 
 #undef CheckEntries
 }
-
-#pragma endregion AST
-

project/components/interface.cpp

@@ -383,9 +383,7 @@ StringCached get_cached_string( StrC str )
 	return result;
 }
 
-/*
+// Used internally to retireve a Code object form the CodePool.
-	Used internally to retireve a Code object form the CodePool.
-	*/
 Code make_code()
 {
 	Pool* allocator = & CodePools.back();

project/components/interface.parsing.cpp

@@ -65,17 +65,19 @@ namespace Parser
 
 		Token& current()
 		{
+			while ( Arr[Idx].Type == TokType::Empty_Line )
+				Idx++;
+
 			return Arr[Idx];
 		}
 
 		Token& previous()
 		{
-			return Arr[Idx - 1];
+			s32 idx = this->Idx;
-		}
+			while ( Arr[Idx].Type == TokType::Empty_Line )
+				idx--;
 
-		Token* next()
+			return Arr[idx - 1];
-		{
-			return Idx + 1 < Arr.num() ? &Arr[Idx + 1] : nullptr;
 		}
 
 		Token& operator []( s32 idx )
@@ -168,6 +170,12 @@ namespace Parser
 			return false;
 		}
 
+		if ( Arr[Idx].Type == TokType::Empty_Line && type != TokType::Empty_Line )
+		{
+			Idx++;
+			return log_fmt( "Auto-skipping empty line (%d, %d)\n", current().Line, current().Column );
+		}
+
 		if ( Arr[Idx].Type != type )
 		{
 			String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } );
@@ -203,7 +211,7 @@ namespace Parser
 			if ( current == '\n' )  \
 			{                       \
 				line++;             \
-				column = 0;         \
+				column = 1;         \
 			}                       \
 			else                    \
 			{                       \
@@ -249,13 +257,25 @@ namespace Parser
 		{
 			Token token = { nullptr, 0, TokType::Invalid, line, column, false };
 
-			if ( line == 4921 )
-			{
-				log_fmt("here");
-			}
-
 			bool is_define = false;
 
+			if ( column == 1 )
+			{
+				token.Text = scanner;
+
+				if ( current == '\r')
+					token.Length = 1;
+
+				if ( current == '\n' )
+				{
+					token.Type = TokType::Empty_Line;
+					token.Length ++;
+
+					Tokens.append( token );
+					continue;
+				}
+			}
+
 			SkipWhitespace();
 			if ( left <= 0 )
 				break;
@@ -1022,13 +1042,16 @@ if ( def.Ptr == nullptr )                                                      \
 	return CodeInvalid;                                                        \
 }
 
-#	define nexttok 	    Context.Tokens.next()
 #	define currtok      Context.Tokens.current()
 #	define prevtok      Context.Tokens.previous()
 #	define eat( Type_ ) Context.Tokens.__eat( Type_ )
 #	define left         ( Context.Tokens.Arr.num() - Context.Tokens.Idx )
 
-#	define check( Type_ ) ( left && currtok.Type == Type_ )
+#	define check( Type_ )                     \
+	( left                                    \
+	&& (currtok.Type == TokType::Empty_Line ? \
+		eat( TokType::Empty_Line) : true)     \
+	&& currtok.Type == Type_ )
 
 #	define push_scope()                                                    \
 	StackNode scope { nullptr, currtok, NullToken, txt_StrC( __func__ ) }; \
@@ -2402,6 +2425,12 @@ CodeBody parse_class_struct_body( Parser::TokType which )
 
 		switch ( currtok.Type )
 		{
+			case TokType::Empty_Line:
+				// Empty lines are auto skipped by Tokens.current()
+				member = untyped_str( Context.Tokens.Arr[ Context.Tokens.Idx] );
+				eat( TokType::Empty_Line );
+			break;
+
 			case TokType::Comment:
 				member = def_comment( currtok );
 				eat( TokType::Comment );
@@ -2756,6 +2785,12 @@ CodeBody parse_global_nspace( CodeT which )
 
 		switch ( currtok.Type )
 		{
+			case TokType::Empty_Line:
+				// Empty lines are auto skipped by Tokens.current()
+				member = untyped_str( Context.Tokens.Arr[ Context.Tokens.Idx] );
+				eat( TokType::Empty_Line );
+			break;
+
 			case TokType::Comment:
 				member = def_comment( currtok );
 				eat( TokType::Comment );
@@ -3072,6 +3107,12 @@ CodeEnum parse_enum( bool inplace_def )
 		{
 			switch ( currtok.Type )
 			{
+				case TokType::Empty_Line:
+					// Empty lines are auto skipped by Tokens.current()
+					member = untyped_str( Context.Tokens.Arr[ Context.Tokens.Idx] );
+					eat( TokType::Empty_Line );
+				break;
+
 				case TokType::Comment:
 					member = def_comment( currtok );
 					eat( TokType::Comment );
@@ -4027,11 +4068,6 @@ CodeTypedef parse_typedef()
 
 	eat( TokType::Decl_Typedef );
 
-	if ( currtok.Line == 2196 )
-	{
-		log_fmt("here");
-	}
-
 	constexpr bool from_typedef = true;
 
 	if ( check( TokType::Preprocess_Macro ))
@@ -4144,6 +4180,12 @@ CodeUnion parse_union( bool inplace_def )
 		Code member = { nullptr };
 		switch ( currtok.Type )
 		{
+			case TokType::Empty_Line:
+				// Empty lines are auto skipped by Tokens.current()
+				member = untyped_str( Context.Tokens.Arr[ Context.Tokens.Idx] );
+				eat( TokType::Empty_Line );
+			break;
+
 			case TokType::Comment:
 				member = def_comment( currtok );
 				eat( TokType::Comment );
@@ -4422,7 +4464,6 @@ CodeVar parse_variable( StrC def )
 
 // Undef helper macros
 #	undef check_parse_args
-#	undef nexttok
 #	undef currtok
 #	undef prevtok
 #	undef eat

project/components/interface.upfront.cpp

@@ -1,3 +1,5 @@
+#pragma region Upfront
+
 enum class OpValidateResult : u32
 {
 	Fail,
@@ -375,7 +377,7 @@ OpValidateResult operator__validate( OperatorT op, CodeParam params_code, CodeTy
 
 
 /*
-The implementaiton of the upfront constructors involves bascially doing three things:
+The implementaiton of the upfront constructors involves doing three things:
 * Validate the arguments given to construct the intended type of AST is valid.
 * Construct said AST type.
 * Lock the AST (set to readonly) and return the valid object.
@@ -2130,4 +2132,9 @@ CodeBody def_union_body( s32 num, CodeUnion* codes )
 #	undef name_check
 #	undef null_check
 #	undef null_or_invalid_check
+#	undef def_body_start
+#	undef def_body_code_array_start
+
+
+#pragma endregion Upfront
 

project/components/temp/etoktype.cpp

@@ -50,6 +50,7 @@ namespace Parser
 	Entry( Decl_Typedef,           "typedef" )          \
 	Entry( Decl_Using,             "using" )            \
 	Entry( Decl_Union,             "union" )            \
+	Entry( Empty_Line,             "__empty_line__" )   \
 	Entry( Identifier,             "__identifier__" )   \
 	Entry( Module_Import,          "import" )           \
 	Entry( Module_Export,          "export" )           \

project/dependencies/basic_types.hpp

@@ -117,5 +117,5 @@ typedef s8  b8;
 typedef s16 b16;
 typedef s32 b32;
 
-#pragma region Basic Types
+#pragma endregion Basic Types
 

project/dependencies/debug.cpp

@@ -1,4 +1,4 @@
-#pragma endregion Debug
+#pragma region Debug
 
 void assert_handler( char const* condition, char const* file, s32 line, char const* msg, ... )
 {

project/dependencies/hashing.cpp

@@ -82,5 +82,5 @@ u64 crc64( void const* data, sw len )
 	return result;
 }
 
-#pragma region Hashing
+#pragma endregion Hashing
 

project/dependencies/parsing.cpp

@@ -799,7 +799,6 @@ ADT_Error adt_str_to_number_strict( ADT_Node* node )
 #	define GEN_CSV_ASSERT( msg )
 #endif
 
-
 u8 csv_parse_delimiter( CSV_Object* root, char* text, AllocatorInfo allocator, b32 has_header, char delim )
 {
 	CSV_Error error = ECSV_Error__NONE;

project/dependencies/printing.hpp

@@ -58,3 +58,4 @@ sw fatal(char const* fmt, ...)
 }
 
 #pragma endregion Printing
+

project/enums/ETokType.csv

@@ -32,6 +32,7 @@ Decl_Template,          "template"
 Decl_Typedef,           "typedef"
 Decl_Using,             "using"
 Decl_Union,             "union"
+Empty_Line,             "__empty_line__"
 Identifier,             "__identifier__"
 Module_Import,          "import"
 Module_Export,          "export"

project/file_processors/builder.hpp

@@ -12,3 +12,4 @@ struct Builder
 
 	void write();
 };
+

project/file_processors/scanner.hpp

@@ -1,8 +1,3 @@
-#pragma once
-#include "gen.hpp"
-
-GEN_NS_BEGIN
-
 Code scan_file( char const* path )
 {
 	FileInfo file;
@@ -28,6 +23,7 @@ Code scan_file( char const* path )
 	return untyped_str( str );
 }
 
+#if 0
 struct Policy
 {
 	// Nothing for now.
@@ -69,5 +65,5 @@ struct Scanner
 
 	bool process_requests( Array<Receipt> out_receipts );
 };
+#endif
 
-GEN_NS_END

project/gen.bootstrap.cpp

@@ -44,10 +44,10 @@ int gen_main()
 		Builder
 		deps_header = Builder::open("gen/gen_dep.hpp");
 		deps_header.print_fmt( generation_notice );
-		deps_header.print_fmt("// This file is intended to be included within gen.hpp (There is no pragma diagnostic ignores)\n\n");
+		deps_header.print_fmt( "// This file is intended to be included within gen.hpp (There is no pragma diagnostic ignores)\n\n" );
-		deps_header.print_fmt("#pragma once\n\n");
+		deps_header.print_fmt( "#pragma once\n\n" );
 		deps_header.print( header_start );
-		deps_header.print_fmt( "GEN_NS_BEGIN\n\n");
+		deps_header.print_fmt( "GEN_NS_BEGIN\n\n" );
 
 		deps_header.print( macros );
 		deps_header.print( basic_types );
@@ -61,7 +61,7 @@ int gen_main()
 		deps_header.print( file_handling );
 		deps_header.print( timing );
 
-		deps_header.print_fmt( "GEN_NS_END\n\n");
+		deps_header.print_fmt( "GEN_NS_END\n\n" );
 		deps_header.write();
 	}
 
@@ -78,11 +78,11 @@ int gen_main()
 		Code timing        = scan_file( "dependencies/timing.cpp" );
 
 		Builder
-		deps_impl = Builder::open("gen/gen_dep.cpp");
+		deps_impl = Builder::open( "gen/gen_dep.cpp" );
 		deps_impl.print_fmt( generation_notice );
-		deps_impl.print_fmt("// This file is intended to be included within gen.cpp (There is no pragma diagnostic ignores)\n\n");
+		deps_impl.print_fmt( "// This file is intended to be included within gen.cpp (There is no pragma diagnostic ignores)\n\n" );
 		deps_impl.print( src_start );
-		deps_impl.print_fmt( "GEN_NS_BEGIN\n\n");
+		deps_impl.print_fmt( "GEN_NS_BEGIN\n\n" );
 
 		deps_impl.print( debug );
 		deps_impl.print( string_ops );
@@ -93,7 +93,7 @@ int gen_main()
 		deps_impl.print( file_handling );
 		deps_impl.print( timing );
 
-		deps_impl.print_fmt( "GEN_NS_END\n\n");
+		deps_impl.print_fmt( "GEN_NS_END\n\n" );
 		deps_impl.write();
 	}
 
@@ -115,30 +115,32 @@ int gen_main()
 		Builder
 		header = Builder::open( "gen/gen.hpp" );
 		header.print_fmt( generation_notice );
-		header.print_fmt("#pragma once\n\n");
+		header.print_fmt( "#pragma once\n\n" );
 		header.print( push_ignores );
 		header.print( header_start );
-		header.print_fmt( "GEN_NS_BEGIN\n\n");
+		header.print_fmt( "GEN_NS_BEGIN\n\n" );
 
-		header.print_fmt("#pragma region Types\n\n");
+		header.print_fmt( "#pragma region Types\n\n" );
 		header.print( types );
 		header.print( ecode );
 		header.print( eoperator );
 		header.print( especifier );
-		header.print_fmt("#pragma endregion Types\n\n");
+		header.print_fmt( "#pragma endregion Types\n\n" );
 
-		header.print_fmt("#pragma region AST\n\n");
+		header.print_fmt( "#pragma region AST\n\n" );
 		header.print( ast );
 		header.print( ast_types );
-		header.print_fmt("#pragma endregion AST\n\n");
+		header.print_fmt( "#pragma endregion AST\n\n" );
 
 		header.print( interface );
 
+		header.print_fmt( "#pragma region Inlines\n\n" );
 		header.print( inlines );
 		header.print( ast_inlines );
+		header.print_fmt( "#pragma endregion Inlines\n\n" );
 
 		header.print( header_end );
-		header.print_fmt( "GEN_NS_END\n\n");
+		header.print_fmt( "GEN_NS_END\n\n" );
 		header.print( pop_ignores );
 		header.write();
 	}
@@ -147,7 +149,7 @@ int gen_main()
 	{
 		Code        src_start       = scan_file( "components/src_start.cpp" );
 		CodeInclude header          = def_include( txt_StrC("gen.hpp") );
-		Code        data 	        = scan_file( "components/static_data.cpp" );
+		Code        static_data 	= scan_file( "components/static_data.cpp" );
 		Code        ast_case_macros = scan_file( "components/ast_case_macros.cpp" );
 		Code        ast			    = scan_file( "components/ast.cpp" );
 		Code        interface	    = scan_file( "components/interface.cpp" );
@@ -166,14 +168,22 @@ int gen_main()
 		src.print( header );
 		src.print_fmt( "\nGEN_NS_BEGIN\n\n");
 
-		src.print( data );
+		src.print( static_data );
+
+		src.print_fmt( "#pragma region AST\n\n" );
 		src.print( ast_case_macros );
 		src.print( ast );
+		src.print_fmt( "#pragma endregion AST\n\n" );
+
+		src.print_fmt( "#pragma region Interface\n\n" );
 		src.print( interface );
 		src.print( upfront );
+		src.print_fmt( "#pragma region Parsing\n\n" );
 		src.print( parser_nspace );
 		src.print( parsing );
 		src.print( untyped );
+		src.print_fmt( "#pragma endregion Parsing\n\n" );
+		src.print_fmt( "#pragma endregion Interface\n\n" );
 
 		src.print_fmt( "GEN_NS_END\n\n");
 		src.print( pop_ignores );
@@ -182,7 +192,6 @@ int gen_main()
 
 	// gen_builder.hpp
 	{
-		Code parsing = scan_file( "dependencies/parsing.hpp" );
 		Code builder = scan_file( "file_processors/builder.hpp" );
 
 		Builder
@@ -190,7 +199,6 @@ int gen_main()
 		header.print_fmt( generation_notice );
 		header.print( def_include( txt_StrC("gen.hpp") ));
 		header.print_fmt( "\nGEN_NS_BEGIN\n\n" );
-		header.print( parsing );
 		header.print( builder );
 		header.print_fmt( "\nGEN_NS_END\n\n" );
 		header.write();
@@ -198,19 +206,48 @@ int gen_main()
 
 	// gen_builder.cpp
 	{
-		Code parsing = scan_file( "dependencies/parsing.cpp" );
 		Code builder = scan_file( "file_processors/builder.cpp" );
 
 		Builder
 		src = Builder::open( "gen/gen_builder.cpp" );
+		src.print_fmt( generation_notice );
 		src.print( def_include( txt_StrC("gen_builder.hpp") ) );
 		src.print_fmt( "\nGEN_NS_BEGIN\n\n" );
-		src.print( parsing );
 		src.print( builder );
 		src.print_fmt( "\nGEN_NS_END\n\n" );
 		src.write();
 	}
 
+	// gen_scanner.hpp
+	{
+		Code parsing = scan_file( "dependencies/parsing.hpp" );
+		Code scanner = scan_file( "file_processors/scanner.hpp" );
+
+		Builder
+		header = Builder::open( "gen/gen_scanner.hpp" );
+		header.print( def_include( txt_StrC("gen.hpp") ) );
+		header.print_fmt( "\nGEN_NS_BEGIN\n\n" );
+		header.print( parsing );
+		header.print( scanner );
+		header.print_fmt( "\nGEN_NS_END\n\n" );
+		header.write();
+	}
+
+	// gen_scanner.cpp
+	{
+		Code parsing = scan_file( "dependencies/parsing.cpp" );
+		// Code scanner = scan_file( "file_processors/scanner.cpp" );
+
+		Builder
+		src = Builder::open( "gen/gen_scanner.cpp" );
+		src.print( def_include( txt_StrC("gen_scanner.hpp") ) );
+		src.print_fmt( "\nGEN_NS_BEGIN\n\n" );
+		src.print( parsing );
+		// src.print( scanner );
+		src.print_fmt( "\nGEN_NS_END\n\n" );
+		src.write();
+	}
+
 	gen::deinit();
 	return 0;
 }

project/gen/gen_builder.hpp

@@ -1,448 +0,0 @@
-// This file was generated automatially by gen.bootstrap.cpp (See: https://github.com/Ed94/gencpp)
-
-#include "gen.hpp"
-
-GEN_NS_BEGIN
-
-#pragma region ADT
-
-enum ADT_Type : u32
-{
-	EADT_TYPE_UNINITIALISED, /* node was not initialised, this is a programming error! */
-	EADT_TYPE_ARRAY,
-	EADT_TYPE_OBJECT,
-	EADT_TYPE_STRING,
-	EADT_TYPE_MULTISTRING,
-	EADT_TYPE_INTEGER,
-	EADT_TYPE_REAL,
-};
-
-enum ADT_Props : u32
-{
-	EADT_PROPS_NONE,
-	EADT_PROPS_NAN,
-	EADT_PROPS_NAN_NEG,
-	EADT_PROPS_INFINITY,
-	EADT_PROPS_INFINITY_NEG,
-	EADT_PROPS_FALSE,
-	EADT_PROPS_TRUE,
-	EADT_PROPS_NULL,
-	EADT_PROPS_IS_EXP,
-	EADT_PROPS_IS_HEX,
-
-	// Used internally so that people can fill in real numbers they plan to write.
-	EADT_PROPS_IS_PARSED_REAL,
-};
-
-enum ADT_NamingStyle : u32
-{
-	EADT_NAME_STYLE_DOUBLE_QUOTE,
-	EADT_NAME_STYLE_SINGLE_QUOTE,
-	EADT_NAME_STYLE_NO_QUOTES,
-};
-
-enum ADT_AssignStyle : u32
-{
-	EADT_ASSIGN_STYLE_COLON,
-	EADT_ASSIGN_STYLE_EQUALS,
-	EADT_ASSIGN_STYLE_LINE,
-};
-
-enum ADT_DelimStyle : u32
-{
-	EADT_DELIM_STYLE_COMMA,
-	EADT_DELIM_STYLE_LINE,
-	EADT_DELIM_STYLE_NEWLINE,
-};
-
-enum ADT_Error : u32
-{
-	EADT_ERROR_NONE,
-	EADT_ERROR_INTERNAL,
-	EADT_ERROR_ALREADY_CONVERTED,
-	EADT_ERROR_INVALID_TYPE,
-	EADT_ERROR_OUT_OF_MEMORY,
-};
-
-struct ADT_Node
-{
-	char const*      name;
-	struct ADT_Node* parent;
-
-	/* properties */
-	ADT_Type type  : 4;
-	u8       props : 4;
-#ifndef GEN_PARSER_DISABLE_ANALYSIS
-	u8 cfg_mode          : 1;
-	u8 name_style        : 2;
-	u8 assign_style      : 2;
-	u8 delim_style       : 2;
-	u8 delim_line_width  : 4;
-	u8 assign_line_width : 4;
-#endif
-
-	/* adt data */
-	union
-	{
-		char const*       string;
-		Array< ADT_Node > nodes;    ///< zpl_array
-
-		struct
-		{
-			union
-			{
-				f64 real;
-				s64 integer;
-			};
-
-#ifndef GEN_PARSER_DISABLE_ANALYSIS
-			/* number analysis */
-			s32 base;
-			s32 base2;
-			u8  base2_offset : 4;
-			s8  exp          : 4;
-			u8  neg_zero     : 1;
-			u8  lead_digit   : 1;
-#endif
-		};
-	};
-};
-
-/* ADT NODE LIMITS
- * delimiter and assignment segment width is limited to 128 whitespace symbols each.
- * real number limits decimal position to 128 places.
- * real number exponent is limited to 64 digits.
- */
-
-/**
- * @brief Initialise an ADT object or array
- *
- * @param node
- * @param backing Memory allocator used for descendants
- * @param name Node's name
- * @param is_array
- * @return error code
- */
-u8 adt_make_branch( ADT_Node* node, AllocatorInfo backing, char const* name, b32 is_array );
-
-/**
- * @brief Destroy an ADT branch and its descendants
- *
- * @param node
- * @return error code
- */
-u8 adt_destroy_branch( ADT_Node* node );
-
-/**
- * @brief Initialise an ADT leaf
- *
- * @param node
- * @param name Node's name
- * @param type Node's type (use zpl_adt_make_branch for container nodes)
- * @return error code
- */
-u8 adt_make_leaf( ADT_Node* node, char const* name, ADT_Type type );
-
-
-/**
- * @brief Fetch a node using provided URI string.
- *
- * This method uses a basic syntax to fetch a node from the ADT. The following features are available
- * to retrieve the data:
- *
- * - "a/b/c" navigates through objects "a" and "b" to get to "c"
- * - "arr/[foo=123]/bar" iterates over "arr" to find any object with param "foo" that matches the value "123", then gets its field called "bar"
- * - "arr/3" retrieves the 4th element in "arr"
- * - "arr/[apple]" retrieves the first element of value "apple" in "arr"
- *
- * @param node ADT node
- * @param uri Locator string as described above
- * @return zpl_adt_node*
- *
- * @see code/apps/examples/json_get.c
- */
-ADT_Node* adt_query( ADT_Node* node, char const* uri );
-
-/**
- * @brief Find a field node within an object by the given name.
- *
- * @param node
- * @param name
- * @param deep_search Perform search recursively
- * @return zpl_adt_node * node
- */
-ADT_Node* adt_find( ADT_Node* node, char const* name, b32 deep_search );
-
-/**
- * @brief Allocate an unitialised node within a container at a specified index.
- *
- * @param parent
- * @param index
- * @return zpl_adt_node * node
- */
-ADT_Node* adt_alloc_at( ADT_Node* parent, sw index );
-
-/**
- * @brief Allocate an unitialised node within a container.
- *
- * @param parent
- * @return zpl_adt_node * node
- */
-ADT_Node* adt_alloc( ADT_Node* parent );
-
-/**
- * @brief Move an existing node to a new container at a specified index.
- *
- * @param node
- * @param new_parent
- * @param index
- * @return zpl_adt_node * node
- */
-ADT_Node* adt_move_node_at( ADT_Node* node, ADT_Node* new_parent, sw index );
-
-/**
- * @brief Move an existing node to a new container.
- *
- * @param node
- * @param new_parent
- * @return zpl_adt_node * node
- */
-ADT_Node* adt_move_node( ADT_Node* node, ADT_Node* new_parent );
-
-/**
- * @brief Swap two nodes.
- *
- * @param node
- * @param other_node
- * @return
- */
-void adt_swap_nodes( ADT_Node* node, ADT_Node* other_node );
-
-/**
- * @brief Remove node from container.
- *
- * @param node
- * @return
- */
-void adt_remove_node( ADT_Node* node );
-
-/**
- * @brief Initialise a node as an object
- *
- * @param obj
- * @param name
- * @param backing
- * @return
- */
-b8 adt_set_obj( ADT_Node* obj, char const* name, AllocatorInfo backing );
-
-/**
- * @brief Initialise a node as an array
- *
- * @param obj
- * @param name
- * @param backing
- * @return
- */
-b8 adt_set_arr( ADT_Node* obj, char const* name, AllocatorInfo backing );
-
-/**
- * @brief Initialise a node as a string
- *
- * @param obj
- * @param name
- * @param value
- * @return
- */
-b8 adt_set_str( ADT_Node* obj, char const* name, char const* value );
-
-/**
- * @brief Initialise a node as a float
- *
- * @param obj
- * @param name
- * @param value
- * @return
- */
-b8 adt_set_flt( ADT_Node* obj, char const* name, f64 value );
-
-/**
- * @brief Initialise a node as a signed integer
- *
- * @param obj
- * @param name
- * @param value
- * @return
- */
-b8 adt_set_int( ADT_Node* obj, char const* name, s64 value );
-
-/**
- * @brief Append a new node to a container as an object
- *
- * @param parent
- * @param name
- * @return*
- */
-ADT_Node* adt_append_obj( ADT_Node* parent, char const* name );
-
-/**
- * @brief Append a new node to a container as an array
- *
- * @param parent
- * @param name
- * @return*
- */
-ADT_Node* adt_append_arr( ADT_Node* parent, char const* name );
-
-/**
- * @brief Append a new node to a container as a string
- *
- * @param parent
- * @param name
- * @param value
- * @return*
- */
-ADT_Node* adt_append_str( ADT_Node* parent, char const* name, char const* value );
-
-/**
- * @brief Append a new node to a container as a float
- *
- * @param parent
- * @param name
- * @param value
- * @return*
- */
-ADT_Node* adt_append_flt( ADT_Node* parent, char const* name, f64 value );
-
-/**
- * @brief Append a new node to a container as a signed integer
- *
- * @param parent
- * @param name
- * @param value
- * @return*
- */
-ADT_Node* adt_append_int( ADT_Node* parent, char const* name, s64 value );
-
-/* parser helpers */
-
-/**
- * @brief Parses a text and stores the result into an unitialised node.
- *
- * @param node
- * @param base
- * @return*
- */
-char* adt_parse_number( ADT_Node* node, char* base );
-
-/**
- * @brief Parses a text and stores the result into an unitialised node.
- * This function expects the entire input to be a number.
- *
- * @param node
- * @param base
- * @return*
- */
-char* adt_parse_number_strict( ADT_Node* node, char* base_str );
-
-/**
- * @brief Parses and converts an existing string node into a number.
- *
- * @param node
- * @return
- */
-ADT_Error adt_str_to_number( ADT_Node* node );
-
-/**
- * @brief Parses and converts an existing string node into a number.
- * This function expects the entire input to be a number.
- *
- * @param node
- * @return
- */
-ADT_Error adt_str_to_number_strict( ADT_Node* node );
-
-/**
- * @brief Prints a number into a file stream.
- *
- * The provided file handle can also be a memory mapped stream.
- *
- * @see zpl_file_stream_new
- * @param file
- * @param node
- * @return
- */
-ADT_Error adt_print_number( FileInfo* file, ADT_Node* node );
-
-/**
- * @brief Prints a string into a file stream.
- *
- * The provided file handle can also be a memory mapped stream.
- *
- * @see zpl_file_stream_new
- * @param file
- * @param node
- * @param escaped_chars
- * @param escape_symbol
- * @return
- */
-ADT_Error adt_print_string( FileInfo* file, ADT_Node* node, char const* escaped_chars, char const* escape_symbol );
-
-#pragma endregion ADT
-
-#pragma region CSV
-
-enum CSV_Error : u32
-{
-	ECSV_Error__NONE,
-	ECSV_Error__INTERNAL,
-	ECSV_Error__UNEXPECTED_END_OF_INPUT,
-	ECSV_Error__MISMATCHED_ROWS,
-};
-
-typedef ADT_Node CSV_Object;
-
-GEN_DEF_INLINE u8 csv_parse( CSV_Object* root, char* text, AllocatorInfo allocator, b32 has_header );
-u8                csv_parse_delimiter( CSV_Object* root, char* text, AllocatorInfo allocator, b32 has_header, char delim );
-void              csv_free( CSV_Object* obj );
-
-GEN_DEF_INLINE void   csv_write( FileInfo* file, CSV_Object* obj );
-GEN_DEF_INLINE String csv_write_string( AllocatorInfo a, CSV_Object* obj );
-void                  csv_write_delimiter( FileInfo* file, CSV_Object* obj, char delim );
-String                csv_write_string_delimiter( AllocatorInfo a, CSV_Object* obj, char delim );
-
-/* inline */
-
-GEN_IMPL_INLINE u8 csv_parse( CSV_Object* root, char* text, AllocatorInfo allocator, b32 has_header )
-{
-	return csv_parse_delimiter( root, text, allocator, has_header, ',' );
-}
-
-GEN_IMPL_INLINE void csv_write( FileInfo* file, CSV_Object* obj )
-{
-	csv_write_delimiter( file, obj, ',' );
-}
-
-GEN_IMPL_INLINE String csv_write_string( AllocatorInfo a, CSV_Object* obj )
-{
-	return csv_write_string_delimiter( a, obj, ',' );
-}
-
-#pragma endregion CSV
-
-struct Builder
-{
-	FileInfo File;
-	String   Buffer;
-
-	static Builder open( char const* path );
-
-	void pad_lines( s32 num );
-
-	void print( Code );
-	void print_fmt( char const* fmt, ... );
-
-	void write();
-};
-
-GEN_NS_END

scripts/bootstrap.ci.ps1

@@ -64,6 +64,7 @@ Push-location $path_project
 		'gen.hpp', 'gen.cpp',
 		'gen_dep.hpp', 'gen_dep.cpp',
 		'gen_builder.hpp', 'gen_builder.cpp'
+		'gen_scanner.hpp', 'gen_scanner.cpp'
 	)
 	$exclude = $null
 

singleheader/gen.singleheader.cpp

@@ -2,9 +2,17 @@
 #define GEN_ENFORCE_STRONG_CODE_TYPES
 #define GEN_EXPOSE_BACKEND
 #include "gen.cpp"
-#include "file_processors/scanner.hpp"
+
 #include "helpers/helper.hpp"
 
+GEN_NS_BEGIN
+#include "dependencies/parsing.cpp"
+GEN_NS_END
+
+#include "file_processors/builder.hpp"
+#include "file_processors/builder.cpp"
+#include "file_processors/scanner.hpp"
+
 using namespace gen;
 
 constexpr char const* generation_notice =
@@ -41,30 +49,25 @@ global bool generate_scanner = true;
 
 int gen_main()
 {
-	gen::init();
-
 #define project_dir "../project/"
+	gen::init();
 
 	Code push_ignores        = scan_file( project_dir "helpers/push_ignores.inline.hpp" );
 	Code pop_ignores         = scan_file( project_dir "helpers/pop_ignores.inline.hpp" );
-
+	Code single_header_start = scan_file( "components/header_start.hpp" );
-	Code header_start = scan_file( "components/header_start.hpp" );
 
 	Builder
 	header = Builder::open( "gen/gen.hpp" );
-	header.print( generation_notice );
+	header.print_fmt( generation_notice );
-	header.print( push_ignores );
-
 	header.print_fmt("#pragma once\n\n");
+	header.print( push_ignores );
 
 	// Headers
 	{
-		header.print( header_start );
+		header.print( single_header_start );
 
 		if ( generate_gen_dep )
 		{
-			header.print_fmt( roll_own_dependencies_guard_start );
-
 			Code header_start  = scan_file( project_dir "dependencies/header_start.hpp" );
 			Code macros        = scan_file( project_dir "dependencies/macros.hpp" );
 			Code basic_types   = scan_file( project_dir "dependencies/basic_types.hpp" );
@@ -78,8 +81,10 @@ int gen_main()
 			Code file_handling = scan_file( project_dir "dependencies/file_handling.hpp" );
 			Code timing        = scan_file( project_dir "dependencies/timing.hpp" );
 
+			header.print_fmt( roll_own_dependencies_guard_start );
 			header.print( header_start );
 			header.print_fmt( "GEN_NS_BEGIN\n\n" );
+
 			header.print( macros );
 			header.print( basic_types );
 			header.print( debug );
@@ -91,8 +96,15 @@ int gen_main()
 			header.print( string );
 			header.print( file_handling );
 			header.print( timing );
-			header.print_fmt( "GEN_NS_END\n" );
 
+			if ( generate_scanner )
+			{
+				header.print_fmt( "pragma region Parsing\n\n" );
+				header.print( scan_file( project_dir "dependencies/parsing.hpp" ) );
+				header.print_fmt( "pragma endregion Parsing\n\n" );
+			}
+
+			header.print_fmt( "GEN_NS_END\n" );
 			header.print_fmt( roll_own_dependencies_guard_end );
 		}
 
@@ -124,10 +136,27 @@ int gen_main()
 
 		header.print( interface );
 
-		header.print_fmt( inlines );
+		header.print_fmt( "#pragma region Inlines\n\n" );
-		header.print_fmt( ast_inlines );
+		header.print( inlines );
+		header.print( ast_inlines );
+		header.print_fmt( "#pragma endregion Inlines\n\n" );
 
 		header.print( header_end );
+
+		if ( generate_builder )
+		{
+			header.print_fmt( "#pragma region Builder\n\n" );
+			header.print( scan_file( project_dir "file_processors/builder.hpp" ) );
+			header.print_fmt( "#pragma endregion Builder\n\n" );
+		}
+
+		if ( generate_scanner )
+		{
+			header.print_fmt( "#pragma region Scanner\n\n" );
+			header.print( scan_file( project_dir "file_processors/scanner.hpp" ) );
+			header.print_fmt( "#pragma endregion Scanner\n\n" );
+		}
+
 		header.print_fmt( "GEN_NS_END\n" );
 	}
 
@@ -137,7 +166,7 @@ int gen_main()
 
 		if ( generate_gen_dep )
 		{
-			Code impl_start    = scan_file( project_dir "dependencies/impl_start.cpp" );
+			Code impl_start    = scan_file( project_dir "dependencies/src_start.cpp" );
 			Code debug         = scan_file( project_dir "dependencies/debug.cpp" );
 			Code string_ops    = scan_file( project_dir "dependencies/string_ops.cpp" );
 			Code printing      = scan_file( project_dir "dependencies/printing.cpp" );
@@ -162,11 +191,18 @@ int gen_main()
 			header.print( file_handling );
 			header.print( timing );
 
+			if ( generate_scanner )
+			{
+				header.print_fmt( "#pragma region Parsing\n\n" );
+				header.print( scan_file( project_dir "dependencies/parsing.cpp" ) );
+				header.print_fmt( "#pragma endregion Parsing\n\n" );
+			}
+
 			header.print_fmt( "GEN_NS_END\n");
 			header.print_fmt( roll_own_dependencies_guard_end );
 		}
 
-		Code data 	         = scan_file( project_dir "components/static_data.cpp" );
+		Code static_data 	 = scan_file( project_dir "components/static_data.cpp" );
 		Code ast_case_macros = scan_file( project_dir "components/ast_case_macros.cpp" );
 		Code ast             = scan_file( project_dir "components/ast.cpp" );
 		Code interface       = scan_file( project_dir "components/interface.cpp" );
@@ -178,14 +214,39 @@ int gen_main()
 		CodeNamespace parser_nspace = def_namespace( name(Parser), def_namespace_body( args(etoktype)) );
 
 		header.print_fmt( "GEN_NS_BEGIN\n\n");
-		header.print( data );
+		header.print( static_data );
+
+		header.print_fmt( "#pragma region AST\n\n" );
 		header.print( ast_case_macros );
 		header.print( ast );
+		header.print_fmt( "#pragma endregion AST\n\n" );
+
+		header.print_fmt( "#pragma region Interface\n\n" );
 		header.print( interface );
 		header.print( upfront );
+		header.print_fmt( "#pragma region Parsing\n\n" );
 		header.print( parser_nspace );
 		header.print( parsing );
+		header.print_fmt( "#pragma endregion Parsing\n\n" );
 		header.print( untyped );
+		header.print_fmt( "#pragma endregion Interface\n\n");
+
+		if ( generate_builder )
+		{
+			header.print_fmt( "#pragma region Builder\n\n" );
+			header.print( scan_file( project_dir "file_processors/builder.cpp" ) );
+			header.print_fmt( "#pragma endregion Builder\n\n" );
+		}
+
+#if 0
+		if ( generate_scanner )
+		{
+			header.print_fmt( "#pragma region Scanner\n\n" );
+			header.print( scan_file( project_dir "file_processors/scanner.cpp" ) );
+			header.print_fmt( "#pragma endregion Scanner\n\n" );
+		}
+#endif
+
 		header.print_fmt( "GEN_NS_END\n");
 
 		header.print_fmt( "%s\n", (char const*) implementation_guard_end );
@@ -196,4 +257,5 @@ int gen_main()
 
 	gen::deinit();
 	return 0;
+#undef project_dir
 }