Reorganization of files, refactors, doc updates (WIP)

Removing the gen. namespace from the files for components, dependencies, and file_processors.
They are only necessary if the include directory is transparent, and in my case those are not.

Made a docs directory. I'm offloading information from the main readme to there along with additional informationn I end up elaborating on down the line.
Enum tables were moved to their own directory (project/enums).

Library will not compile for now. Major refactor occuring with parsing related components.

project/components/AttributeTokens.csv

@@ -1,2 +0,0 @@
-API_Export, GEN_API_Export_Code
-API_Import, GEN_API_Import_Code

project/components/ECode.csv

@@ -1,47 +0,0 @@
-Invalid
-Untyped
-Comment
-Access_Private
-Access_Protected
-Access_Public
-PlatformAttributes
-Class
-Class_Fwd
-Class_Body
-Enum
-Enum_Fwd
-Enum_Body
-Enum_Class
-Enum_Class_Fwd
-Execution
-Export_Body
-Extern_Linkage
-Extern_Linkage_Body
-Friend
-Function
-Function_Fwd
-Function_Body
-Global_Body
-Module
-Namespace
-Namespace_Body
-Operator
-Operator_Fwd
-Operator_Member
-Operator_Member_Fwd
-Operator_Cast
-Operator_Cast_Fwd
-Parameters
-Preprocessor_Include
-Specifiers
-Struct
-Struct_Fwd
-Struct_Body
-Template
-Typedef
-Typename
-Union
-Union_Body
-Using
-Using_Namespace
-Variable

project/components/EOperator.csv

@@ -1,43 +0,0 @@
-Invalid,         INVALID
-Assign,          "="
-Assign_Add,      "+="
-Assign_Subtract, "-="
-Assign_Multiply, "*="
-Assign_Divide,   "/="
-Assign_Modulo,   "%="
-Assign_BAnd,     "&="
-Assign_BOr,      "|="
-Assign_BXOr,     "^="
-Assign_LShift,   "<<="
-Assign_RShift,   ">>="
-Increment,       "++"
-Decrement,       "--"
-Unary_Plus,      "+"
-Unary_Minus,     "-"
-UnaryNot,        "!"
-Add,             "+"
-Subtract,        "-"
-Multiply,        "*"
-Divide,          "/"
-Modulo,          "%"
-BNot,            "~"
-BAnd,            "&"
-BOr,             "|"
-BXOr,            "^"
-LShift,          "<<"
-RShift,          ">>"
-LAnd,            "&&"
-LOr,             "||"
-LEqual,          "=="
-LNot,            "!="
-Lesser,          "<"
-Greater,         ">"
-LesserEqual,     "<="
-GreaterEqual,    ">="
-Subscript,       "[]"
-Indirection,     "*"
-AddressOf,       "&"
-MemberOfPointer, "->"
-PtrToMemOfPtr,   "->*"
-FunctionCall,    "()"
-Comma,           ","

project/components/ESpecifier.csv

@@ -1,22 +0,0 @@
-Invalid,          INVALID
-Consteval,        consteval
-Constexpr,        constexpr
-Constinit,        constinit
-Explicit,         explicit
-External_Linkage, extern
-Global,           global
-Inline,           inline
-Internal_Linkage, internal
-Local_Persist,    local_persist
-Mutable,          mutable
-Ptr,              *
-Ref,              &
-Register,         register
-RValue,           &&
-Static,           static
-Thread_Local,     thread_local
-Volatile,         volatile
-Virtual,          virtual
-Const,            const
-Final,            final
-Override,         override

project/components/ETokType.csv

@@ -1,74 +0,0 @@
-Invalid,                "__invalid__"
-Access_Private,         "private"
-Access_Protected,       "protected"
-Access_Public,          "public"
-Access_MemberSymbol,    "."
-Access_StaticSymbol,    "::"
-Ampersand,              "&"
-Ampersand_DBL,          "&&"
-Assign_Classifer,       ":"
-Attribute_Open, 	    "[["
-Attribute_Close, 	    "]]"
-BraceCurly_Open,        "{"
-BraceCurly_Close,       "}"
-BraceSquare_Open,       "["
-BraceSquare_Close,      "]"
-Capture_Start,          "("
-Capture_End,            ")"
-Comment,                "__comment__"
-Char,                   "__char__"
-Comma,                  ","
-Decl_Class,             "class"
-Decl_GNU_Attribute,     "__attribute__"
-Decl_MSVC_Attribute,    "__declspec"
-Decl_Enum,              "enum"
-Decl_Extern_Linkage,    "extern"
-Decl_Friend,            "friend"
-Decl_Module,            "module"
-Decl_Namespace,         "namespace"
-Decl_Operator,          "operator"
-Decl_Struct,            "struct"
-Decl_Template,          "template"
-Decl_Typedef,           "typedef"
-Decl_Using,             "using"
-Decl_Union,             "union"
-Identifier,             "__identifier__"
-Module_Import,          "import"
-Module_Export,          "export"
-Number,                 "number"
-Operator,               "operator"
-Preprocess_Define,      "#define"
-Preprocess_Include,     "#include"
-Preprocess_If,          "#if"
-Preprocess_ElIF,		"#elif"
-Preprocess_Else,        "#else"
-Preprocess_EndIf,       "#endif"
-Spec_Alignas,           "alignas"
-Spec_Const,             "const"
-Spec_Consteval,         "consteval"
-Spec_Constexpr,         "constexpr"
-Spec_Constinit,         "constinit"
-Spec_Explicit, 		    "explicit"
-Spec_Extern,            "extern"
-Spec_Final, 		    "final"
-Spec_Global, 		    "global"
-Spec_Inline,            "inline"
-Spec_Internal_Linkage,  "internal"
-Spec_LocalPersist,      "local_persist"
-Spec_Mutable,           "mutable"
-Spec_Override,          "override"
-Spec_Static,            "static"
-Spec_ThreadLocal,       "thread_local"
-Spec_Volatile,          "volatile"
-Star,                   "*"
-Statement_End,          ";"
-String,                 "__string__"
-Type_Unsigned, 	        "unsigned"
-Type_Signed,            "signed"
-Type_Short,             "short"
-Type_Long,              "long"
-Type_char, 			    "char"
-Type_int, 			    "int"
-Type_double, 		    "double"
-Varadic_Argument,       "..."
-Attributes_Start,       "__attrib_start__"

project/components/ast.cpp

@@ -17,20 +17,9 @@ AST* AST::duplicate()
 
 String AST::to_string()
 {
-#	define ProcessModuleFlags()                                      \
-	if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))  \
-		result.append( "export " );                                  \
-																     \
-	if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Import ))  \
-		result.append( "import " );                                  \
-
 	local_persist thread_local
 	char SerializationLevel = 0;
 
-#if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION)
-	u64 time_start = time_rel_ms();
-#endif
-
 	// TODO : Need to refactor so that intermeidate strings are freed conviently.
 	String result = String::make( GlobalAllocator, "" );
 
@@ -82,7 +71,8 @@ String AST::to_string()
 
 		case Class:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes || ParentType )
 			{
@@ -129,7 +119,8 @@ String AST::to_string()
 
 		case Class_Fwd:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes )
 				result.append_fmt( "class %s %s;", Attributes->to_string(), Name );
@@ -140,7 +131,8 @@ String AST::to_string()
 
 		case Enum:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes || UnderlyingType )
 			{
@@ -170,7 +162,8 @@ String AST::to_string()
 
 		case Enum_Fwd:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes )
 				result.append_fmt( "%s ", Attributes->to_string() );
@@ -181,7 +174,8 @@ String AST::to_string()
 
 		case Enum_Class:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes || UnderlyingType )
 			{
@@ -219,7 +213,8 @@ String AST::to_string()
 
 		case Enum_Class_Fwd:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			result.append( "enum class " );
 
@@ -262,7 +257,8 @@ String AST::to_string()
 
 		case Function:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes )
 				result.append_fmt( "%s ", Attributes->to_string() );
@@ -301,7 +297,8 @@ String AST::to_string()
 
 		case Function_Fwd:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes )
 				result.append_fmt( "%s ", Attributes->to_string() );
@@ -347,7 +344,8 @@ String AST::to_string()
 			break;
 
 		case Namespace:
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			result.append_fmt( "namespace %s\n{\n%s}"
 				, Name
@@ -358,7 +356,8 @@ String AST::to_string()
 		case Operator:
 		case Operator_Member:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes )
 				result.append_fmt( "%s ", Attributes->to_string() );
@@ -395,7 +394,8 @@ String AST::to_string()
 		case Operator_Fwd:
 		case Operator_Member_Fwd:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes )
 				result.append_fmt( "%s ", Attributes->to_string() );
@@ -513,7 +513,8 @@ String AST::to_string()
 
 		case Struct:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Name == nullptr)
 			{
@@ -566,7 +567,8 @@ String AST::to_string()
 
 		case Struct_Fwd:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes )
 				result.append_fmt( "struct %s %s;", Attributes->to_string(), Name );
@@ -577,7 +579,8 @@ String AST::to_string()
 
 		case Template:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			result.append_fmt( "template< %s >\n%s", Params->to_string(), Declaration->to_string() );
 		}
@@ -585,7 +588,8 @@ String AST::to_string()
 
 		case Typedef:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			result.append( "typedef ");
 
@@ -624,7 +628,8 @@ String AST::to_string()
 
 		case Union:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			result.append( "union " );
 
@@ -650,7 +655,8 @@ String AST::to_string()
 
 		case Using:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes )
 				result.append_fmt( "%s ", Attributes->to_string() );
@@ -675,7 +681,8 @@ String AST::to_string()
 
 		case Variable:
 		{
-			ProcessModuleFlags();
+			if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))
+				result.append( "export " );
 
 			if ( Attributes || Specs )
 			{
@@ -726,11 +733,7 @@ String AST::to_string()
 		break;
 	}
 
-#if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION)
-	log_fmt("AST::to_string() time taken: %llu for: %s\n", time_rel_ms() - time_start, result );
-#endif
 	return result;
-#undef ProcessModuleFlags
 }
 
 bool AST::is_equal( AST* other )
@@ -760,19 +763,19 @@ bool AST::validate_body()
 {
 	using namespace ECode;
 
-#define CheckEntries( Unallowed_Types )                                                                \
-	do                                                                                                 \
-	{                                                                                                  \
-		for ( Code entry : cast<CodeBody>() )                                                          \
-		{                                                                                              \
-			switch ( entry->Type )                                                                     \
-			{                                                                                          \
-				Unallowed_Types                                                                        \
-					log_failure( "AST::validate_body: Invalid entry in body %s", entry.debug_str() );  \
-					return false;                                                                      \
-			}                                                                                          \
-		}                                                                                              \
-	}                                                                                                  \
+#define CheckEntries( Unallowed_Types )                                                               \
+	do                                                                                                \
+	{                                                                                                 \
+		for ( Code entry : cast<CodeBody>() )                                                         \
+		{                                                                                             \
+			switch ( entry->Type )                                                                    \
+			{                                                                                         \
+				Unallowed_Types                                                                       \
+					log_failure( "AST::validate_body: Invalid entry in body %s", entry.debug_str() ); \
+					return false;                                                                     \
+			}                                                                                         \
+		}                                                                                             \
+	}                                                                                                 \
 	while (0);
 
 	switch ( Type )
@@ -825,6 +828,8 @@ bool AST::validate_body()
 	}
 
 	return false;
+
+#undef CheckEntries
 }
 
 #pragma endregion AST

project/components/eoperator.hpp

@@ -72,4 +72,4 @@ namespace EOperator
 
 #	undef Define_Operators
 }
-using OperatorT = EOperator::Type;
+using OperatorT = EOperator::Type;

project/components/especifier.hpp

@@ -101,5 +101,4 @@ namespace ESpecifier
 
 #	undef Define_Specifiers
 }
-
 using SpecifierT = ESpecifier::Type;

project/components/etoktype.cpp

@@ -9,8 +9,8 @@ namespace Parser
 	Attributes_Start is only used to indicate the start of the user_defined attribute list.
 */
 
-#ifndef GEN_Define_Attribute_Tokens
-#	define GEN_Define_Attribute_Tokens         \
+#ifndef GEN_DEFINE_ATTRIBUTE_TOKENS
+#	define GEN_DEFINE_ATTRIBUTE_TOKENS         \
 	Entry( API_Export, "GEN_API_Export_Code" ) \
 	Entry( API_Import, "GEN_API_Import_Code" )
 #endif
@@ -97,7 +97,7 @@ namespace Parser
 		{
 		#	define Entry( Name_, Str_ ) Name_,
 			Define_TokType
-			GEN_Define_Attribute_Tokens
+			GEN_DEFINE_ATTRIBUTE_TOKENS
 		#	undef Entry
 			NumTokens,
 		};
@@ -110,7 +110,7 @@ namespace Parser
 			{
 			#	define Entry( Name_, Str_ ) { sizeof(Str_), Str_ },
 				Define_TokType
-				GEN_Define_Attribute_Tokens
+				GEN_DEFINE_ATTRIBUTE_TOKENS
 			#	undef Entry
 			};
 
@@ -137,7 +137,7 @@ namespace Parser
 			{
 			#	define Entry( Name_, Str_ ) Str_,
 				Define_TokType
-				GEN_Define_Attribute_Tokens
+				GEN_DEFINE_ATTRIBUTE_TOKENS
 			#	undef Entry
 			};
 

project/components/interface.cpp

@@ -105,7 +105,7 @@ void define_constants()
 #endif
 #	undef def_constant_code_type
 
-	t_empty = (CodeType) make_code();
+	t_empty          = (CodeType) make_code();
 	t_empty->Type    = ECode::Typename;
 	t_empty->Name    = get_cached_string( txt_StrC("") );
 	t_empty.set_global();
@@ -127,12 +127,12 @@ void define_constants()
 	access_private->Name = get_cached_string( txt_StrC("private:") );
 	access_private.set_global();
 
-	access_protected = make_code();
+	access_protected       = make_code();
 	access_protected->Type = ECode::Access_Protected;
 	access_protected->Name = get_cached_string( txt_StrC("protected:") );
 	access_protected.set_global();
 
-	access_public = make_code();
+	access_public       = make_code();
 	access_public->Type = ECode::Access_Public;
 	access_public->Name = get_cached_string( txt_StrC("public:") );
 	access_public.set_global();

project/components/types.hpp

@@ -68,7 +68,7 @@ ModuleFlag operator|( ModuleFlag A, ModuleFlag B)
 
 	Override these to change the attribute to your own unique identifier convention.
 
-	The tokenizer identifies attribute defines with the GEN_Define_Attribute_Tokens macros.
+	The tokenizer identifies attribute defines with the GEN_DEFINE_ATTRIBUTE_TOKENS macros.
 	See the example below and the Define_TokType macro used in gen.cpp to know the format.
 	While the library can parse raw attributes, most projects use defines to wrap them for compiler
 	platform indendence. The token define allows support for them without having to modify the library.

project/components/untyped.cpp

@@ -12,8 +12,7 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va )
 		char tok_map_mem[ TokenFmt_TokenMap_MemSize ];
 
 		tok_map_arena = Arena::init_from_memory( tok_map_mem, sizeof(tok_map_mem) );
-
-		tok_map = HashTable<StrC>::init( tok_map_arena );
+		tok_map       = HashTable<StrC>::init( tok_map_arena );
 
 		s32 left = num_tokens - 1;