TOKEN_KINDs macro swappable with codegen/gen_src.cpp

codegen/gen_src.cpp

@@ -1,10 +1,13 @@
+#if __clang__
 #pragma clang diagnostic ignored "-Wswitch"
+#endif
 
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 #define GEN_ENFORCE_STRING_CODE_TYPES
 #define GEN_EXPOSE_BACKEND
 #include "gencpp/gen.cpp"
 #include "gencpp/gen.builder.cpp"
+#include "gencpp/gen.scanner.cpp"
 using namespace gen;
 
 #ifdef GEN_SYSTEM_WINDOWS
@@ -127,13 +130,87 @@ int gen_main()
 	gen::init();
 	log_fmt("Generating code for Odin's src\n");
 
-	// Remove TOKEN_KINDS usage in tokenizer.cpp
-	if (0)
-	{
+	StrC str_GB_STATIC_ASSERT = txt("GB_STATIC_ASSERT(");
+	PreprocessorDefines.append( get_cached_string(str_GB_STATIC_ASSERT) );
 
+	// Remove TOKEN_KINDS usage in tokenizer.cpp
+	if (1)
+	{
+		CSV_Object csv_nodes;
+		{
+			char  scratch_mem[kilobytes(32)];
+			Arena scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) );
+			file_read_contents( scratch, zero_terminate, path_codegen "token_kinds.csv" );
+
+			csv_parse( &csv_nodes, scratch_mem, GlobalAllocator, false );
+		}
+		Array<ADT_Node> enum_strs = csv_nodes.nodes[0].nodes;
+		Array<ADT_Node> str_strs  = csv_nodes.nodes[1].nodes;
+
+		String enum_entries   = String::make_reserve( GlobalAllocator, kilobytes(32) );
+		String to_str_entries = String::make_reserve( GlobalAllocator, kilobytes(32) );
+
+		to_str_entries.append(txt("{"));
+		for (uw idx = 0; idx < enum_strs.num(); idx++)
+		{
+			char const* enum_str     = enum_strs[idx].string;
+			StrC        entry_to_str = to_str(str_strs [idx].string);
+
+			enum_entries.append_fmt( "Token_%s,\n", enum_str );
+			to_str_entries.append( token_fmt( "str", (StrC)entry_to_str, stringize(
+				{ cast(u8 *) "<str>", gb_size_of("<str>") -1 },\n
+			)));
+		}
+		to_str_entries.append(txt("}"));
+
+		CodeBody src_tokenizer_cpp = parse_file( path_src "tokenizer.cpp" );
+		CodeBody body = def_body( ECode::Global_Body );
+
+		body.append( def_comment(txt("NOTICE(github: Ed94): This is a generated variant of tokenizer.cpp using <repo_root>/codegen/gen_src.cpp")));
+		body.append(fmt_newline);
+
+		for (Code code = src_tokenizer_cpp.begin(); code != src_tokenizer_cpp.end(); ++ code) switch (code->Type)
+		{
+			case ECode::Preprocess_Define:
+				if ( code->Name.starts_with( txt("TOKEN_KINDS"))) {
+					// Skip, we don't want it.
+					continue;
+				}
+			continue;
+
+			case ECode::Enum:
+			{
+				if ( code->Name.starts_with(txt("TokenKind")))
+				{
+					CodeEnum enum_code = code.cast<CodeEnum>();
+					enum_code->Body = untyped_str(enum_entries);
+				}
+				body.append(code);
+			}
+			continue;
+
+			case ECode::Variable:
+				if ( code->Name.starts_with(txt("token_strings")))
+				{
+					CodeVar var = code.cast<CodeVar>();
+					var->Value = untyped_str(to_str_entries);
+				}
+				body.append(code);
+			continue;
+
+			default:
+				body.append(code);
+			continue;
+		}
+
+		Builder header = Builder::open( path_src "tokenizer.cpp" );
+		header.print(body);
+		header.write();
+		format_file( path_src "tokenizer.cpp" );
 	}
 
 	// Remove AST_KINDS macro usage in parser.hpp
+	// Note this doesn't account for an already swapped file. Make sure to discard changes or shut this path off if already generated.
 	if (1)
 	{
 		CodeBody src_parser_header = parse_file( path_src "parser.hpp" );

codegen/gencpp/gen.cpp

@@ -85,7 +85,7 @@ global CodeSpecifiers spec_final;
 global CodeSpecifiers spec_gb_inline;
 global CodeSpecifiers spec_gb_global;
 global CodeSpecifiers spec_inline;
-global CodeSpecifiers spec_internal_linkage;
+global CodeSpecifiers spec_gb_internal;
 global CodeSpecifiers spec_local_persist;
 global CodeSpecifiers spec_mutable;
 global CodeSpecifiers spec_noexcept;
@@ -3016,7 +3016,7 @@ internal void define_constants()
 	def_constant_spec( gb_inline, ESpecifier::gb_inline );
 	def_constant_spec( gb_global, ESpecifier::gb_global );
 	def_constant_spec( inline, ESpecifier::Inline );
-	def_constant_spec( internal_linkage, ESpecifier::Internal_Linkage );
+	def_constant_spec( gb_internal, ESpecifier::gb_internal );
 	def_constant_spec( local_persist, ESpecifier::Local_Persist );
 	def_constant_spec( mutable, ESpecifier::Mutable );
 	def_constant_spec( neverinline, ESpecifier::NeverInline );
@@ -5668,8 +5668,7 @@ namespace parser
 	namespace ETokType
 	{
 #define GEN_DEFINE_ATTRIBUTE_TOKENS \
-	Entry( Attribute_API_Export, "GEN_API_Export_Code" ) Entry( Attribute_API_Import, "GEN_API_Import_Code" ) \
-	Entry( gb_internal, "gb_internal" )
+	Entry( Attribute_API_Export, "GEN_API_Export_Code" ) Entry( Attribute_API_Import, "GEN_API_Import_Code" )
 
 		enum Type : u32
 		{
@@ -5739,7 +5738,7 @@ namespace parser
 			Spec_gb_inline,
 			Spec_gb_global,
 			Spec_Inline,
-			Spec_Internal_Linkage,
+			Spec_gb_internal,
 			Spec_LocalPersist,
 			Spec_Mutable,
 			Spec_NeverInline,
@@ -5770,7 +5769,6 @@ namespace parser
 			__Attributes_Start,
 			Attribute_API_Export,
 			Attribute_API_Import,
-			gb_internal,
 			NumTokens
 		};
 
@@ -5843,7 +5841,7 @@ namespace parser
 				{ sizeof( "gb_inline" ),             "gb_inline"             },
 				{ sizeof( "gb_global" ),             "gb_global"             },
 				{ sizeof( "inline" ),                "inline"                },
-				{ sizeof( "internal" ),              "internal"              },
+				{ sizeof( "gb_internal" ),           "gb_internal"              },
 				{ sizeof( "local_persist" ),         "local_persist"         },
 				{ sizeof( "mutable" ),               "mutable"               },
 				{ sizeof( "neverinline" ),           "neverinline"           },
@@ -5874,7 +5872,6 @@ namespace parser
 				{ sizeof( "__attrib_start__" ),      "__attrib_start__"      },
 				{ sizeof( "GEN_API_Export_Code" ),   "GEN_API_Export_Code"   },
 				{ sizeof( "GEN_API_Import_Code" ),   "GEN_API_Import_Code"   },
-				{ sizeof( "gb_internal" ),           "gb_internal"   },
 			};
 			return lookup[type];
 		}
@@ -8845,7 +8842,7 @@ namespace parser
 				case TokType::Spec_gb_inline :
 				case TokType::Spec_gb_global :
 				case TokType::Spec_Inline :
-				case TokType::Spec_Internal_Linkage :
+				case TokType::Spec_gb_internal :
 				case TokType::Spec_NeverInline :
 				case TokType::Spec_Static :
 				case TokType::Spec_gb_thread_local :
@@ -8866,7 +8863,7 @@ namespace parser
 							case ESpecifier::gb_inline :
 							case ESpecifier::gb_global :
 							case ESpecifier::External_Linkage :
-							case ESpecifier::Internal_Linkage :
+							case ESpecifier::gb_internal :
 							case ESpecifier::Inline :
 							case ESpecifier::Mutable :
 							case ESpecifier::NeverInline :
@@ -9382,7 +9379,7 @@ namespace parser
 			case '<' :
 			{
 				if ( currtok.Text[1] == '=' )
-					op = LEqual;
+					op = LesserEqual;
 
 				else if ( currtok.Text[1] == '<' )
 				{

codegen/gencpp/gen.hpp

@@ -418,7 +418,7 @@ namespace ESpecifier
 		gb_inline,
 		gb_global,
 		Inline,
-		Internal_Linkage,
+		gb_internal,
 		Local_Persist,
 		Mutable,
 		NeverInline,
@@ -455,7 +455,7 @@ namespace ESpecifier
 			{ sizeof( "gb_inline" ),        "gb_inline"     },
 			{ sizeof( "gb_global" ),        "gb_global"     },
 			{ sizeof( "inline" ),           "inline"        },
-			{ sizeof( "internal" ),         "internal"      },
+			{ sizeof( "gb_internal" ),      "gb_internal"      },
 			{ sizeof( "local_persist" ),    "local_persist" },
 			{ sizeof( "mutable" ),          "mutable"       },
 			{ sizeof( "neverinline" ),      "neverinline"   },
@@ -6404,7 +6404,7 @@ extern CodeSpecifiers spec_final;
 extern CodeSpecifiers Spec_gb_inline;
 extern CodeSpecifiers spec_global;
 extern CodeSpecifiers spec_inline;
-extern CodeSpecifiers spec_internal_linkage;
+extern CodeSpecifiers spec_gb_internal;
 extern CodeSpecifiers spec_local_persist;
 extern CodeSpecifiers spec_mutable;
 extern CodeSpecifiers spec_neverinline;

codegen/gencpp/gen.scanner.cpp

@@ -1106,5 +1106,4 @@ String csv_write_string_delimiter( AllocatorInfo a, CSV_Object* obj, char delimi
 
 #pragma endregion CSV
 
-#include "scanner.hpp"
 GEN_NS_END

codegen/token_kinds.csv

@@ -0,0 +1,126 @@
+Invalid, "Invalid"
+EOF,     "EOF"
+Comment, "Comment"
+
+_LiteralBegin, " "
+	Ident,   "identifier"
+	Integer, "integer"
+	Float,   "float"
+	Imag,    "imaginary"
+	Rune,    "rune"
+	String,  "string"
+_LiteralEnd, " "
+
+_OperatorBegin, " "
+	Eq,       "="
+	Not,      "!"
+	Hash,     "#"
+	At,       "@"
+	Dollar,   "$"
+	Pointer,  "^"
+	Question, "?"
+	Add,      "+"
+	Sub,      "-"
+	Mul,      "*"
+	Quo,      "/"
+	Mod,      "%"
+	ModMod,   "%%"
+	And,      "&"
+	Or,       "|"
+	Xor,      "~"
+	AndNot,   "&~"
+	Shl,      "<<"
+	Shr,      ">>"
+	CmpAnd,   "&&"
+	CmpOr,    "||"
+
+_AssignOpBegin, " "
+	AddEq,    "+="
+	SubEq,    "-="
+	MulEq,    "*="
+	QuoEq,    "/="
+	ModEq,    "%="
+	ModModEq, "%%="
+	AndEq,    "&="
+	OrEq,     "|="
+	XorEq,    "~="
+	AndNotEq, "&~="
+	ShlEq,    "<<="
+	ShrEq,    ">>="
+	CmpAndEq, "&&="
+	CmpOrEq,  "||="
+_AssignOpEnd, " "
+
+	Increment,  "++"
+	Decrement,  "--"
+	ArrowRight, "->"
+	Uninit,     "---"
+
+_ComparisonBegin, " "
+	CmpEq, "=="
+	NotEq, "!="
+	Lt,    "<"
+	Gt,    ">"
+	LtEq,  "<="
+	GtEq,  ">="
+_ComparisonEnd, " "
+
+	OpenParen,    "("
+	CloseParen,   ")"
+	OpenBracket,  "["
+	CloseBracket, "]"
+	OpenBrace,    "{"
+	CloseBrace,   "}"
+	Colon,        ":"
+	Semicolon,    ";"
+	Period,       "."
+	Comma,        ","
+	Ellipsis,     ".."
+	RangeFull,    "..="
+	RangeHalf,    "..<"
+	BackSlash,    "\\"
+_OperatorEnd, " "
+
+_KeywordBegin, " "
+	import,      "import"
+	foreign,     "foreign"
+	package,     "package"
+	typeid,      "typeid"
+	when,        "when"
+	where,       "where"
+	if,          "if"
+	else,        "else"
+	for,         "for"
+	switch,      "switch"
+	in,          "in"
+	not_in,      "not_in"
+	do,          "do"
+	case,        "case"
+	break,       "break"
+	continue,    "continue"
+	fallthrough, "fallthrough"
+	defer,       "defer"
+	return,      "return"
+	proc,        "proc"
+	struct,      "struct"
+	union,       "union"
+	enum,        "enum"
+	bit_set,     "bit_set"
+	bit_field,   "bit_field"
+	map,         "map"
+	dynamic,     "dynamic"
+	auto_cast,   "auto_cast"
+	cast,        "cast"
+	transmute,   "transmute"
+	distinct,    "distinct"
+	using,       "using"
+	context,     "context"
+	or_else,     "or_else"
+	or_return,   "or_return"
+	or_break,    "or_break"
+	or_continue, "or_continue"
+	asm,         "asm"
+	matrix,      "matrix"
+_KeywordEnd, " "
+
+Count, " "