From a33492da24595e001ff6ec998c99f44211775960 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 5 May 2024 01:46:32 -0400 Subject: [PATCH] TOKEN_KINDs macro swappable with codegen/gen_src.cpp --- codegen/gen_src.cpp | 83 +++++++++++++++++++++- codegen/gencpp/gen.cpp | 19 +++-- codegen/gencpp/gen.hpp | 6 +- codegen/gencpp/gen.scanner.cpp | 1 - codegen/token_kinds.csv | 126 +++++++++++++++++++++++++++++++++ codegen/token_kinds.hpp | 0 6 files changed, 217 insertions(+), 18 deletions(-) create mode 100644 codegen/token_kinds.csv delete mode 100644 codegen/token_kinds.hpp diff --git a/codegen/gen_src.cpp b/codegen/gen_src.cpp index ce9cd5556..b57b28a44 100644 --- a/codegen/gen_src.cpp +++ b/codegen/gen_src.cpp @@ -1,10 +1,13 @@ +#if __clang__ #pragma clang diagnostic ignored "-Wswitch" +#endif #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS #define GEN_ENFORCE_STRING_CODE_TYPES #define GEN_EXPOSE_BACKEND #include "gencpp/gen.cpp" #include "gencpp/gen.builder.cpp" +#include "gencpp/gen.scanner.cpp" using namespace gen; #ifdef GEN_SYSTEM_WINDOWS @@ -127,13 +130,87 @@ int gen_main() gen::init(); log_fmt("Generating code for Odin's src\n"); - // Remove TOKEN_KINDS usage in tokenizer.cpp - if (0) - { + StrC str_GB_STATIC_ASSERT = txt("GB_STATIC_ASSERT("); + PreprocessorDefines.append( get_cached_string(str_GB_STATIC_ASSERT) ); + // Remove TOKEN_KINDS usage in tokenizer.cpp + if (1) + { + CSV_Object csv_nodes; + { + char scratch_mem[kilobytes(32)]; + Arena scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) ); + file_read_contents( scratch, zero_terminate, path_codegen "token_kinds.csv" ); + + csv_parse( &csv_nodes, scratch_mem, GlobalAllocator, false ); + } + Array enum_strs = csv_nodes.nodes[0].nodes; + Array str_strs = csv_nodes.nodes[1].nodes; + + String enum_entries = String::make_reserve( GlobalAllocator, kilobytes(32) ); + String to_str_entries = String::make_reserve( GlobalAllocator, kilobytes(32) ); + + to_str_entries.append(txt("{")); + for (uw idx = 0; idx < enum_strs.num(); idx++) + { + char const* enum_str = enum_strs[idx].string; + StrC entry_to_str = to_str(str_strs [idx].string); + + enum_entries.append_fmt( "Token_%s,\n", enum_str ); + to_str_entries.append( token_fmt( "str", (StrC)entry_to_str, stringize( + { cast(u8 *) "", gb_size_of("") -1 },\n + ))); + } + to_str_entries.append(txt("}")); + + CodeBody src_tokenizer_cpp = parse_file( path_src "tokenizer.cpp" ); + CodeBody body = def_body( ECode::Global_Body ); + + body.append( def_comment(txt("NOTICE(github: Ed94): This is a generated variant of tokenizer.cpp using /codegen/gen_src.cpp"))); + body.append(fmt_newline); + + for (Code code = src_tokenizer_cpp.begin(); code != src_tokenizer_cpp.end(); ++ code) switch (code->Type) + { + case ECode::Preprocess_Define: + if ( code->Name.starts_with( txt("TOKEN_KINDS"))) { + // Skip, we don't want it. + continue; + } + continue; + + case ECode::Enum: + { + if ( code->Name.starts_with(txt("TokenKind"))) + { + CodeEnum enum_code = code.cast(); + enum_code->Body = untyped_str(enum_entries); + } + body.append(code); + } + continue; + + case ECode::Variable: + if ( code->Name.starts_with(txt("token_strings"))) + { + CodeVar var = code.cast(); + var->Value = untyped_str(to_str_entries); + } + body.append(code); + continue; + + default: + body.append(code); + continue; + } + + Builder header = Builder::open( path_src "tokenizer.cpp" ); + header.print(body); + header.write(); + format_file( path_src "tokenizer.cpp" ); } // Remove AST_KINDS macro usage in parser.hpp + // Note this doesn't account for an already swapped file. Make sure to discard changes or shut this path off if already generated. if (1) { CodeBody src_parser_header = parse_file( path_src "parser.hpp" ); diff --git a/codegen/gencpp/gen.cpp b/codegen/gencpp/gen.cpp index 0994c9502..ed8d554a9 100644 --- a/codegen/gencpp/gen.cpp +++ b/codegen/gencpp/gen.cpp @@ -85,7 +85,7 @@ global CodeSpecifiers spec_final; global CodeSpecifiers spec_gb_inline; global CodeSpecifiers spec_gb_global; global CodeSpecifiers spec_inline; -global CodeSpecifiers spec_internal_linkage; +global CodeSpecifiers spec_gb_internal; global CodeSpecifiers spec_local_persist; global CodeSpecifiers spec_mutable; global CodeSpecifiers spec_noexcept; @@ -3016,7 +3016,7 @@ internal void define_constants() def_constant_spec( gb_inline, ESpecifier::gb_inline ); def_constant_spec( gb_global, ESpecifier::gb_global ); def_constant_spec( inline, ESpecifier::Inline ); - def_constant_spec( internal_linkage, ESpecifier::Internal_Linkage ); + def_constant_spec( gb_internal, ESpecifier::gb_internal ); def_constant_spec( local_persist, ESpecifier::Local_Persist ); def_constant_spec( mutable, ESpecifier::Mutable ); def_constant_spec( neverinline, ESpecifier::NeverInline ); @@ -5668,8 +5668,7 @@ namespace parser namespace ETokType { #define GEN_DEFINE_ATTRIBUTE_TOKENS \ - Entry( Attribute_API_Export, "GEN_API_Export_Code" ) Entry( Attribute_API_Import, "GEN_API_Import_Code" ) \ - Entry( gb_internal, "gb_internal" ) + Entry( Attribute_API_Export, "GEN_API_Export_Code" ) Entry( Attribute_API_Import, "GEN_API_Import_Code" ) enum Type : u32 { @@ -5739,7 +5738,7 @@ namespace parser Spec_gb_inline, Spec_gb_global, Spec_Inline, - Spec_Internal_Linkage, + Spec_gb_internal, Spec_LocalPersist, Spec_Mutable, Spec_NeverInline, @@ -5770,7 +5769,6 @@ namespace parser __Attributes_Start, Attribute_API_Export, Attribute_API_Import, - gb_internal, NumTokens }; @@ -5843,7 +5841,7 @@ namespace parser { sizeof( "gb_inline" ), "gb_inline" }, { sizeof( "gb_global" ), "gb_global" }, { sizeof( "inline" ), "inline" }, - { sizeof( "internal" ), "internal" }, + { sizeof( "gb_internal" ), "gb_internal" }, { sizeof( "local_persist" ), "local_persist" }, { sizeof( "mutable" ), "mutable" }, { sizeof( "neverinline" ), "neverinline" }, @@ -5874,7 +5872,6 @@ namespace parser { sizeof( "__attrib_start__" ), "__attrib_start__" }, { sizeof( "GEN_API_Export_Code" ), "GEN_API_Export_Code" }, { sizeof( "GEN_API_Import_Code" ), "GEN_API_Import_Code" }, - { sizeof( "gb_internal" ), "gb_internal" }, }; return lookup[type]; } @@ -8845,7 +8842,7 @@ namespace parser case TokType::Spec_gb_inline : case TokType::Spec_gb_global : case TokType::Spec_Inline : - case TokType::Spec_Internal_Linkage : + case TokType::Spec_gb_internal : case TokType::Spec_NeverInline : case TokType::Spec_Static : case TokType::Spec_gb_thread_local : @@ -8866,7 +8863,7 @@ namespace parser case ESpecifier::gb_inline : case ESpecifier::gb_global : case ESpecifier::External_Linkage : - case ESpecifier::Internal_Linkage : + case ESpecifier::gb_internal : case ESpecifier::Inline : case ESpecifier::Mutable : case ESpecifier::NeverInline : @@ -9382,7 +9379,7 @@ namespace parser case '<' : { if ( currtok.Text[1] == '=' ) - op = LEqual; + op = LesserEqual; else if ( currtok.Text[1] == '<' ) { diff --git a/codegen/gencpp/gen.hpp b/codegen/gencpp/gen.hpp index e83a6dc1a..915abd700 100644 --- a/codegen/gencpp/gen.hpp +++ b/codegen/gencpp/gen.hpp @@ -418,7 +418,7 @@ namespace ESpecifier gb_inline, gb_global, Inline, - Internal_Linkage, + gb_internal, Local_Persist, Mutable, NeverInline, @@ -455,7 +455,7 @@ namespace ESpecifier { sizeof( "gb_inline" ), "gb_inline" }, { sizeof( "gb_global" ), "gb_global" }, { sizeof( "inline" ), "inline" }, - { sizeof( "internal" ), "internal" }, + { sizeof( "gb_internal" ), "gb_internal" }, { sizeof( "local_persist" ), "local_persist" }, { sizeof( "mutable" ), "mutable" }, { sizeof( "neverinline" ), "neverinline" }, @@ -6404,7 +6404,7 @@ extern CodeSpecifiers spec_final; extern CodeSpecifiers Spec_gb_inline; extern CodeSpecifiers spec_global; extern CodeSpecifiers spec_inline; -extern CodeSpecifiers spec_internal_linkage; +extern CodeSpecifiers spec_gb_internal; extern CodeSpecifiers spec_local_persist; extern CodeSpecifiers spec_mutable; extern CodeSpecifiers spec_neverinline; diff --git a/codegen/gencpp/gen.scanner.cpp b/codegen/gencpp/gen.scanner.cpp index 53df950a8..ba068ecce 100644 --- a/codegen/gencpp/gen.scanner.cpp +++ b/codegen/gencpp/gen.scanner.cpp @@ -1106,5 +1106,4 @@ String csv_write_string_delimiter( AllocatorInfo a, CSV_Object* obj, char delimi #pragma endregion CSV -#include "scanner.hpp" GEN_NS_END diff --git a/codegen/token_kinds.csv b/codegen/token_kinds.csv new file mode 100644 index 000000000..1d5dc1169 --- /dev/null +++ b/codegen/token_kinds.csv @@ -0,0 +1,126 @@ +Invalid, "Invalid" +EOF, "EOF" +Comment, "Comment" + +_LiteralBegin, " " + Ident, "identifier" + Integer, "integer" + Float, "float" + Imag, "imaginary" + Rune, "rune" + String, "string" +_LiteralEnd, " " + +_OperatorBegin, " " + Eq, "=" + Not, "!" + Hash, "#" + At, "@" + Dollar, "$" + Pointer, "^" + Question, "?" + Add, "+" + Sub, "-" + Mul, "*" + Quo, "/" + Mod, "%" + ModMod, "%%" + And, "&" + Or, "|" + Xor, "~" + AndNot, "&~" + Shl, "<<" + Shr, ">>" + CmpAnd, "&&" + CmpOr, "||" + +_AssignOpBegin, " " + AddEq, "+=" + SubEq, "-=" + MulEq, "*=" + QuoEq, "/=" + ModEq, "%=" + ModModEq, "%%=" + AndEq, "&=" + OrEq, "|=" + XorEq, "~=" + AndNotEq, "&~=" + ShlEq, "<<=" + ShrEq, ">>=" + CmpAndEq, "&&=" + CmpOrEq, "||=" +_AssignOpEnd, " " + + Increment, "++" + Decrement, "--" + ArrowRight, "->" + Uninit, "---" + +_ComparisonBegin, " " + CmpEq, "==" + NotEq, "!=" + Lt, "<" + Gt, ">" + LtEq, "<=" + GtEq, ">=" +_ComparisonEnd, " " + + OpenParen, "(" + CloseParen, ")" + OpenBracket, "[" + CloseBracket, "]" + OpenBrace, "{" + CloseBrace, "}" + Colon, ":" + Semicolon, ";" + Period, "." + Comma, "," + Ellipsis, ".." + RangeFull, "..=" + RangeHalf, "..<" + BackSlash, "\\" +_OperatorEnd, " " + +_KeywordBegin, " " + import, "import" + foreign, "foreign" + package, "package" + typeid, "typeid" + when, "when" + where, "where" + if, "if" + else, "else" + for, "for" + switch, "switch" + in, "in" + not_in, "not_in" + do, "do" + case, "case" + break, "break" + continue, "continue" + fallthrough, "fallthrough" + defer, "defer" + return, "return" + proc, "proc" + struct, "struct" + union, "union" + enum, "enum" + bit_set, "bit_set" + bit_field, "bit_field" + map, "map" + dynamic, "dynamic" + auto_cast, "auto_cast" + cast, "cast" + transmute, "transmute" + distinct, "distinct" + using, "using" + context, "context" + or_else, "or_else" + or_return, "or_return" + or_break, "or_break" + or_continue, "or_continue" + asm, "asm" + matrix, "matrix" +_KeywordEnd, " " + +Count, " " diff --git a/codegen/token_kinds.hpp b/codegen/token_kinds.hpp deleted file mode 100644 index e69de29bb..000000000