Lexer improvement prep, segmentation of lexer and parser.

project/enums/ETokType_New.csv

@@ -0,0 +1,175 @@
+Invalid,                "__invalid__"
+Access_Private,         "private"
+Access_Protected,       "protected"
+Access_Public,          "public"
+Access_MemberSymbol,    "."
+Access_StaticSymbol,    "::"
+Ampersand,              "&"
+Ampersand_DBL,          "&&"
+Assign_Classifer,       ":"
+Attribute_Open,         "[["
+Attribute_Close,        "]]"
+BraceCurly_Open,        "{"
+BraceCurly_Close,       "}"
+BraceSquare_Open,       "["
+BraceSquare_Close,      "]"
+Capture_Start,          "("
+Capture_End,            ")"
+Comment,                "__comment__"
+Comment_End,            "__comment_end__"
+Comment_Start,          "__comment_start__"
+Char,                   "__character__"
+Comma,                  ","
+
+Decl_Class,             "class"
+Decl_GNU_Attribute,     "__attribute__"
+Decl_MSVC_Attribute,    "__declspec"
+Decl_Enum,              "enum"
+Decl_Extern_Linkage,    "extern"
+Decl_Friend,            "friend"
+Decl_Module,            "module"
+Decl_Namespace,         "namespace"
+Decl_Operator,          "operator"
+Decl_Struct,            "struct"
+Decl_Template,          "template"
+
+Decl_Type,              "decltype"
+
+Decl_Typedef,           "typedef"
+Decl_Using,             "using"
+Decl_Union,             "union"
+
+Identifier,             "__identifier__"
+
+Module_Import,          "import"
+Module_Export,          "export"
+
+NewLine,                "__new_line__"
+
+Number,                 "__number__"
+
+Operator,               "__operator__"
+
+Op_Assign,                    "="
+Op_Assign_Add,                "+="
+Op_Assign_Subtract,           "-="
+Op_Assign_Multiply,           "*="
+Op_Assign_Divide,             "/="
+Op_Assign_Modulo,             "%="
+Op_Assign_Bitwise_And,        "&="
+Op_Assign_Bitwise_Or,         "|="
+Op_Assign_Bitwise_XOr,        "^="
+Op_Assign_Bitwise_LeftShift,  "<<="
+Op_Assign_Bitwise_RightShift, ">>="
+
+Op_Increment, "++"
+Op_Decrement, "--"
+
+Op_Add,                 "+"
+Op_Subtract,            "-"
+Op_Multiply,            "*"
+Op_Divide,              "/"
+Op_Modulo,              "%"
+
+Op_Bitwise_And,         "&"
+Op_Bitwise_Or,          "|"
+Op_Bitwise_XOr,         "^"
+Op_Bitwise_LeftShitf,   "<<"
+Op_Bitwise_RightShift,  ">>"
+
+Op_UnaryAdd,            "+"
+Op_UnaryMinus,          "-"
+Op_UnaryNot,            "~"
+
+Op_Logical_Not,         "!"
+Op_Logical_And,         "&&"
+Op_Logical_Or,          "||"
+
+Op_Equal,               "=="
+Op_NotEqual,            "!="
+Op_Lesser,              "<"
+Op_Greater,             ">"
+Op_LesserEqual,         "<="
+Op_GreaterEqual",       ">=
+
+Op_Subscript,                "[]"
+Op_Indirection,              "*"
+Op_Address_Of,               "&"
+Op_MemberOfObject,           "."
+Op_MemberOfPointer",         "->"
+Op_PointerToMemberOfObject,  ".*"
+Op_PointerToMemberOfPointer, "->*"
+
+Op_Comma,                    ","
+Op_Ternary,                  "?"
+
+Preprocess_Hash,        "#"
+Preprocess_Define,      "define"
+Preprocess_If,          "if"
+Preprocess_IfDef,       "ifdef"
+Preprocess_IfNotDef,    "ifndef"
+Preprocess_ElIf,        "elif"
+Preprocess_Else,        "else"
+Preprocess_EndIf,       "endif"
+Preprocess_Include,     "include"
+Preprocess_Pragma,      "pragma"
+Preprocess_Content,	    "__macro_content__"
+Preprocess_Macro,       "__macro__"
+Preprocess_Unsupported, "__unsupported__"
+
+Spec_Alignof,           "alignof"
+Spec_Const,             "const"
+Spec_Consteval,         "consteval"
+Spec_Constexpr,         "constexpr"
+Spec_Constinit,         "constinit"
+Spec_Explicit,          "explicit"
+Spec_Extern,            "extern"
+Spec_Final,             "final"
+Spec_ForceInline,	    "forceinline"
+Spec_Global,            "global"
+Spec_Inline,            "inline"
+Spec_Internal_Linkage,  "internal"
+Spec_LocalPersist,      "local_persist"
+Spec_Mutable,           "mutable"
+Spec_NeverInline,       "neverinline"
+Spec_Override,          "override"
+Spec_Static,            "static"
+Spec_ThreadLocal,       "thread_local"
+Spec_Volatile,          "volatile"
+Spec_Virtual,           "virtual"
+
+Star,                   "*"
+
+Stmt_Break,             "break"
+Stmt_Case,              "case"
+Stmt_Continue,          "continue"
+Stmt_Do,                "do"
+Stmt_Else,              "else"
+Stmt_End,               ";"
+Stmt_If,                "if"
+Stmt_For,               "for"
+Stmt_Return,            "return"
+Stmt_Switch,            "switch"
+Stmt_While,             "while"
+
+StaticAssert,           "static_assert"
+String,                 "__string__"
+
+Type_Unsigned,          "unsigned"
+Type_Signed,            "signed"
+Type_Short,             "short"
+Type_Long,              "long"
+Type_bool,              "bool"
+Type_char,              "char"
+Type_int,               "int"
+Type_float,             "float"
+Type_double,            "double"
+Type_MS_int8,           "__int8"
+Type_MS_int16,          "__int16"
+Type_MS_int32,          "__int32"
+Type_MS_int64,          "__int64"
+Type_MS_W64,            "_W64"
+
+Varadic_Argument,       "..."
+
+__Attributes_Start,     "__attrib_start__"