first pass at unified lexer path, not plugged in yet

src/text/generated/text.meta.c

@@ -0,0 +1,458 @@
+// Copyright (c) Epic Games Tools
+// Licensed under the MIT license (https://opensource.org/license/mit/)
+
+//- GENERATED CODE
+
+C_LINKAGE_BEGIN
+String8 txt_lang_kind_ext_table[8] =
+{
+str8_lit_comp("null"),
+str8_lit_comp("c"),
+str8_lit_comp("cpp"),
+str8_lit_comp("odin"),
+str8_lit_comp("jai"),
+str8_lit_comp("zig"),
+str8_lit_comp("rs"),
+str8_lit_comp("disasm_x64_intel"),
+};
+
+struct {String8 ext; TXT_LangKind kind;} txt_ext_lang_kind_table[24] =
+{
+{str8_lit_comp("null"), TXT_LangKind_Null},
+{str8_lit_comp("c"), TXT_LangKind_C},
+{str8_lit_comp("cpp"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("odin"), TXT_LangKind_Odin},
+{str8_lit_comp("jai"), TXT_LangKind_Jai},
+{str8_lit_comp("zig"), TXT_LangKind_Zig},
+{str8_lit_comp("rs"), TXT_LangKind_Rust},
+{str8_lit_comp("disasm_x64_intel"), TXT_LangKind_DisasmX64Intel},
+{str8_lit_comp("h"), TXT_LangKind_C},
+{str8_lit_comp("cxx"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("cc"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("c++"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("ixx"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("cxxm"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("c++m"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("ccm"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("cppm"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("mpp"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("C"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("hpp"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("hxx"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("hh"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("h++"), TXT_LangKind_CPlusPlus},
+{str8_lit_comp("H"), TXT_LangKind_CPlusPlus},
+};
+
+String8Array txt_keywords_from_lang_kind_table[8] =
+{
+{txt_keywords__null, ArrayCount(txt_keywords__null)},
+{txt_keywords__c, ArrayCount(txt_keywords__c)},
+{txt_keywords__cpp, ArrayCount(txt_keywords__cpp)},
+{txt_keywords__odin, ArrayCount(txt_keywords__odin)},
+{txt_keywords__jai, ArrayCount(txt_keywords__jai)},
+{txt_keywords__zig, ArrayCount(txt_keywords__zig)},
+{txt_keywords__rust, ArrayCount(txt_keywords__rust)},
+{txt_keywords__null, ArrayCount(txt_keywords__null)},
+};
+
+String8Array txt_multichar_symbols_from_lang_kind_table[8] =
+{
+{txt_multichar_symbols__null, ArrayCount(txt_multichar_symbols__null)},
+{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
+{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
+{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
+{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
+{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
+{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
+{txt_multichar_symbols__null, ArrayCount(txt_multichar_symbols__null)},
+};
+
+TXT_TokenizerRuleArray txt_tokenizer_rules_from_lang_kind_table[8] =
+{
+{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
+{txt_tokenizer_rules__c, ArrayCount(txt_tokenizer_rules__c)},
+{txt_tokenizer_rules__c, ArrayCount(txt_tokenizer_rules__c)},
+{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
+{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
+{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
+{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
+{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
+};
+
+String8 txt_keywords__null[1] =
+{
+{0},
+};
+
+String8 txt_multichar_symbols__null[1] =
+{
+{0},
+};
+
+TXT_TokenizerRule txt_tokenizer_rules__null[1] =
+{
+{TXT_TokenKind_Null},
+};
+
+String8 txt_keywords__c[32] =
+{
+str8_lit_comp("auto"),
+str8_lit_comp("break"),
+str8_lit_comp("case"),
+str8_lit_comp("char"),
+str8_lit_comp("const"),
+str8_lit_comp("continue"),
+str8_lit_comp("default"),
+str8_lit_comp("do"),
+str8_lit_comp("double"),
+str8_lit_comp("else"),
+str8_lit_comp("enum"),
+str8_lit_comp("extern"),
+str8_lit_comp("float"),
+str8_lit_comp("for"),
+str8_lit_comp("goto"),
+str8_lit_comp("if"),
+str8_lit_comp("int"),
+str8_lit_comp("long"),
+str8_lit_comp("register"),
+str8_lit_comp("return"),
+str8_lit_comp("short"),
+str8_lit_comp("signed"),
+str8_lit_comp("sizeof"),
+str8_lit_comp("static"),
+str8_lit_comp("struct"),
+str8_lit_comp("switch"),
+str8_lit_comp("typedef"),
+str8_lit_comp("union"),
+str8_lit_comp("unsigned"),
+str8_lit_comp("void"),
+str8_lit_comp("volatile"),
+str8_lit_comp("while"),
+};
+
+String8 txt_multichar_symbols__c[20] =
+{
+str8_lit_comp("<<"),
+str8_lit_comp(">>"),
+str8_lit_comp("<="),
+str8_lit_comp(">="),
+str8_lit_comp("=="),
+str8_lit_comp("!="),
+str8_lit_comp("&&"),
+str8_lit_comp("||"),
+str8_lit_comp("|="),
+str8_lit_comp("&="),
+str8_lit_comp("^="),
+str8_lit_comp("~="),
+str8_lit_comp("+="),
+str8_lit_comp("-="),
+str8_lit_comp("*="),
+str8_lit_comp("/="),
+str8_lit_comp("%="),
+str8_lit_comp("<<="),
+str8_lit_comp(">>="),
+str8_lit_comp("->"),
+};
+
+TXT_TokenizerRule txt_tokenizer_rules__c[7] =
+{
+{TXT_TokenKind_Comment, str8_lit_comp("//"), str8_lit_comp("\n"), 0, 0, 1, 0},
+{TXT_TokenKind_Comment, str8_lit_comp("/*"), str8_lit_comp("*/"), 2, 0, 0, 0},
+{TXT_TokenKind_Meta, str8_lit_comp("#"), str8_lit_comp("\n"), 0, 0, 1, 0},
+{TXT_TokenKind_String, str8_lit_comp("\""), str8_lit_comp("\""), 1, 0, 1, 0},
+{TXT_TokenKind_String, str8_lit_comp("'"), str8_lit_comp("'"), 1, 0, 1, 0},
+{TXT_TokenKind_String, str8_lit_comp("\""), str8_lit_comp("\""), 1, 0, 1, 3},
+{TXT_TokenKind_String, str8_lit_comp("<"), str8_lit_comp(">"), 1, 0, 1, 3},
+};
+
+String8 txt_keywords__cpp[97] =
+{
+str8_lit_comp("alignas"),
+str8_lit_comp("alignof"),
+str8_lit_comp("and"),
+str8_lit_comp("and_eq"),
+str8_lit_comp("asm"),
+str8_lit_comp("atomic_cancel"),
+str8_lit_comp("atomic_commit"),
+str8_lit_comp("atomic_noexcept"),
+str8_lit_comp("auto"),
+str8_lit_comp("bitand"),
+str8_lit_comp("bitor"),
+str8_lit_comp("bool"),
+str8_lit_comp("break"),
+str8_lit_comp("case"),
+str8_lit_comp("catch"),
+str8_lit_comp("char"),
+str8_lit_comp("char8_t"),
+str8_lit_comp("char16_t"),
+str8_lit_comp("char32_t"),
+str8_lit_comp("class"),
+str8_lit_comp("compl"),
+str8_lit_comp("concept"),
+str8_lit_comp("const"),
+str8_lit_comp("consteval"),
+str8_lit_comp("constexpr"),
+str8_lit_comp("constinit"),
+str8_lit_comp("const_cast"),
+str8_lit_comp("continue"),
+str8_lit_comp("co_await"),
+str8_lit_comp("co_return"),
+str8_lit_comp("co_yield"),
+str8_lit_comp("decltype"),
+str8_lit_comp("default"),
+str8_lit_comp("delete"),
+str8_lit_comp("do"),
+str8_lit_comp("double"),
+str8_lit_comp("dynamic_cast"),
+str8_lit_comp("else"),
+str8_lit_comp("enum"),
+str8_lit_comp("explicit"),
+str8_lit_comp("export"),
+str8_lit_comp("extern"),
+str8_lit_comp("false"),
+str8_lit_comp("float"),
+str8_lit_comp("for"),
+str8_lit_comp("friend"),
+str8_lit_comp("goto"),
+str8_lit_comp("if"),
+str8_lit_comp("inline"),
+str8_lit_comp("int"),
+str8_lit_comp("long"),
+str8_lit_comp("mutable"),
+str8_lit_comp("namespace"),
+str8_lit_comp("new"),
+str8_lit_comp("noexcept"),
+str8_lit_comp("not"),
+str8_lit_comp("not_eq"),
+str8_lit_comp("nullptr"),
+str8_lit_comp("operator"),
+str8_lit_comp("or"),
+str8_lit_comp("or_eq"),
+str8_lit_comp("private"),
+str8_lit_comp("protected"),
+str8_lit_comp("public"),
+str8_lit_comp("reflexpr"),
+str8_lit_comp("register"),
+str8_lit_comp("reinterpret_cast"),
+str8_lit_comp("requires"),
+str8_lit_comp("return"),
+str8_lit_comp("short"),
+str8_lit_comp("signed"),
+str8_lit_comp("sizeof"),
+str8_lit_comp("static"),
+str8_lit_comp("static_assert"),
+str8_lit_comp("static_cast"),
+str8_lit_comp("struct"),
+str8_lit_comp("switch"),
+str8_lit_comp("synchronized"),
+str8_lit_comp("template"),
+str8_lit_comp("this"),
+str8_lit_comp("thread_local"),
+str8_lit_comp("throw"),
+str8_lit_comp("true"),
+str8_lit_comp("try"),
+str8_lit_comp("typedef"),
+str8_lit_comp("typeid"),
+str8_lit_comp("typename"),
+str8_lit_comp("union"),
+str8_lit_comp("unsigned"),
+str8_lit_comp("using"),
+str8_lit_comp("virtual"),
+str8_lit_comp("void"),
+str8_lit_comp("volatile"),
+str8_lit_comp("wchar_t"),
+str8_lit_comp("while"),
+str8_lit_comp("xor"),
+str8_lit_comp("xor_eq"),
+};
+
+String8 txt_keywords__odin[40] =
+{
+str8_lit_comp("align_of"),
+str8_lit_comp("asm"),
+str8_lit_comp("auto_cast"),
+str8_lit_comp("bit_set"),
+str8_lit_comp("break"),
+str8_lit_comp("case"),
+str8_lit_comp("cast"),
+str8_lit_comp("context"),
+str8_lit_comp("continue"),
+str8_lit_comp("defer"),
+str8_lit_comp("distinct"),
+str8_lit_comp("do"),
+str8_lit_comp("dynamic"),
+str8_lit_comp("else"),
+str8_lit_comp("enum"),
+str8_lit_comp("fallthrough"),
+str8_lit_comp("for"),
+str8_lit_comp("foreign"),
+str8_lit_comp("if"),
+str8_lit_comp("in"),
+str8_lit_comp("map"),
+str8_lit_comp("matrix"),
+str8_lit_comp("not_in"),
+str8_lit_comp("or_break"),
+str8_lit_comp("or_continue"),
+str8_lit_comp("or_else"),
+str8_lit_comp("or_return"),
+str8_lit_comp("package"),
+str8_lit_comp("proc"),
+str8_lit_comp("return"),
+str8_lit_comp("size_of"),
+str8_lit_comp("struct"),
+str8_lit_comp("switch"),
+str8_lit_comp("transmute"),
+str8_lit_comp("typeid"),
+str8_lit_comp("union"),
+str8_lit_comp("using"),
+str8_lit_comp("when"),
+str8_lit_comp("where"),
+str8_lit_comp("import"),
+};
+
+String8 txt_keywords__jai[39] =
+{
+str8_lit_comp("bool"),
+str8_lit_comp("true"),
+str8_lit_comp("false"),
+str8_lit_comp("int"),
+str8_lit_comp("s8"),
+str8_lit_comp("u8"),
+str8_lit_comp("s16"),
+str8_lit_comp("u16"),
+str8_lit_comp("s32"),
+str8_lit_comp("u32"),
+str8_lit_comp("s64"),
+str8_lit_comp("u64"),
+str8_lit_comp("s128"),
+str8_lit_comp("u128"),
+str8_lit_comp("float"),
+str8_lit_comp("float32"),
+str8_lit_comp("float64"),
+str8_lit_comp("void"),
+str8_lit_comp("enum"),
+str8_lit_comp("enum_flags"),
+str8_lit_comp("size_of"),
+str8_lit_comp("string"),
+str8_lit_comp("type_of"),
+str8_lit_comp("cast"),
+str8_lit_comp("if"),
+str8_lit_comp("ifs"),
+str8_lit_comp("then"),
+str8_lit_comp("else"),
+str8_lit_comp("case"),
+str8_lit_comp("for"),
+str8_lit_comp("while"),
+str8_lit_comp("break"),
+str8_lit_comp("continue"),
+str8_lit_comp("remove"),
+str8_lit_comp("return"),
+str8_lit_comp("inline"),
+str8_lit_comp("null"),
+str8_lit_comp("defer"),
+str8_lit_comp("xx"),
+};
+
+String8 txt_keywords__zig[49] =
+{
+str8_lit_comp("addrspace"),
+str8_lit_comp("align"),
+str8_lit_comp("allowzero"),
+str8_lit_comp("and"),
+str8_lit_comp("anyframe"),
+str8_lit_comp("anytype"),
+str8_lit_comp("asm"),
+str8_lit_comp("async"),
+str8_lit_comp("await"),
+str8_lit_comp("break"),
+str8_lit_comp("callconv"),
+str8_lit_comp("catch"),
+str8_lit_comp("comptime"),
+str8_lit_comp("const"),
+str8_lit_comp("continue"),
+str8_lit_comp("defer"),
+str8_lit_comp("else"),
+str8_lit_comp("enum"),
+str8_lit_comp("errdefer"),
+str8_lit_comp("error"),
+str8_lit_comp("export"),
+str8_lit_comp("extern"),
+str8_lit_comp("fn"),
+str8_lit_comp("for"),
+str8_lit_comp("if"),
+str8_lit_comp("inline"),
+str8_lit_comp("noalias"),
+str8_lit_comp("nosuspend"),
+str8_lit_comp("noinline"),
+str8_lit_comp("opaque"),
+str8_lit_comp("or"),
+str8_lit_comp("orelse"),
+str8_lit_comp("packed"),
+str8_lit_comp("pub"),
+str8_lit_comp("resume"),
+str8_lit_comp("return"),
+str8_lit_comp("linksection"),
+str8_lit_comp("struct"),
+str8_lit_comp("suspend"),
+str8_lit_comp("switch"),
+str8_lit_comp("test"),
+str8_lit_comp("threadlocal"),
+str8_lit_comp("try"),
+str8_lit_comp("union"),
+str8_lit_comp("unreachable"),
+str8_lit_comp("usingnamespace"),
+str8_lit_comp("var"),
+str8_lit_comp("volatile"),
+str8_lit_comp("while"),
+};
+
+String8 txt_keywords__rust[43] =
+{
+str8_lit_comp("as"),
+str8_lit_comp("break"),
+str8_lit_comp("const"),
+str8_lit_comp("continue"),
+str8_lit_comp("crate"),
+str8_lit_comp("else"),
+str8_lit_comp("enum"),
+str8_lit_comp("extern"),
+str8_lit_comp("false"),
+str8_lit_comp("fn"),
+str8_lit_comp("for"),
+str8_lit_comp("if"),
+str8_lit_comp("impl"),
+str8_lit_comp("in"),
+str8_lit_comp("let"),
+str8_lit_comp("loop"),
+str8_lit_comp("match"),
+str8_lit_comp("mod"),
+str8_lit_comp("move"),
+str8_lit_comp("mut"),
+str8_lit_comp("pub"),
+str8_lit_comp("ref"),
+str8_lit_comp("return"),
+str8_lit_comp("self"),
+str8_lit_comp("Self"),
+str8_lit_comp("static"),
+str8_lit_comp("struct"),
+str8_lit_comp("super"),
+str8_lit_comp("trait"),
+str8_lit_comp("true"),
+str8_lit_comp("type"),
+str8_lit_comp("unsafe"),
+str8_lit_comp("use"),
+str8_lit_comp("where"),
+str8_lit_comp("while"),
+str8_lit_comp("yield"),
+str8_lit_comp("async"),
+str8_lit_comp("await"),
+str8_lit_comp("dyn"),
+str8_lit_comp("macro_rules"),
+str8_lit_comp("raw"),
+str8_lit_comp("safe"),
+str8_lit_comp("union"),
+};
+
+C_LINKAGE_END
+

src/text/generated/text.meta.h

@@ -0,0 +1,41 @@
+// Copyright (c) Epic Games Tools
+// Licensed under the MIT license (https://opensource.org/license/mit/)
+
+//- GENERATED CODE
+
+#ifndef TEXT_META_H
+#define TEXT_META_H
+
+typedef enum TXT_LangKind
+{
+TXT_LangKind_Null,
+TXT_LangKind_C,
+TXT_LangKind_CPlusPlus,
+TXT_LangKind_Odin,
+TXT_LangKind_Jai,
+TXT_LangKind_Zig,
+TXT_LangKind_Rust,
+TXT_LangKind_DisasmX64Intel,
+TXT_LangKind_COUNT,
+} TXT_LangKind;
+
+C_LINKAGE_BEGIN
+extern String8 txt_lang_kind_ext_table[8];
+extern String8Array txt_keywords_from_lang_kind_table[8];
+extern String8Array txt_multichar_symbols_from_lang_kind_table[8];
+extern TXT_TokenizerRuleArray txt_tokenizer_rules_from_lang_kind_table[8];
+extern String8 txt_keywords__null[1];
+extern String8 txt_multichar_symbols__null[1];
+extern TXT_TokenizerRule txt_tokenizer_rules__null[1];
+extern String8 txt_keywords__c[32];
+extern String8 txt_multichar_symbols__c[20];
+extern TXT_TokenizerRule txt_tokenizer_rules__c[7];
+extern String8 txt_keywords__cpp[97];
+extern String8 txt_keywords__odin[40];
+extern String8 txt_keywords__jai[39];
+extern String8 txt_keywords__zig[49];
+extern String8 txt_keywords__rust[43];
+
+C_LINKAGE_END
+
+#endif // TEXT_META_H

src/text/text.c

@@ -4,6 +4,11 @@
 #undef LAYER_COLOR
 #define LAYER_COLOR 0xe34cd4ff
 
+////////////////////////////////
+//~ rjf: Generated Code
+
+#include "generated/text.meta.c"
+
 ////////////////////////////////
 //~ rjf: Basic Helpers
 
@@ -11,45 +16,13 @@ internal TXT_LangKind
 txt_lang_kind_from_extension(String8 extension)
 {
   TXT_LangKind kind = TXT_LangKind_Null;
-  if(str8_match(extension, str8_lit("c"), 0) ||
-     str8_match(extension, str8_lit("h"), 0))
+  for EachElement(idx, txt_ext_lang_kind_table)
   {
-    kind = TXT_LangKind_C;
-  }
-  else if(str8_match(extension, str8_lit("cpp"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("cxx"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("cc"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("c++"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("ixx"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("cxxm"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("c++m"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("ccm"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("cppm"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("mpp"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("C"), 0) ||
-          str8_match(extension, str8_lit("hpp"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("hxx"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("hh"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("h++"), StringMatchFlag_CaseInsensitive) ||
-          str8_match(extension, str8_lit("H"), 0))
-  {
-    kind = TXT_LangKind_CPlusPlus;
-  }
-  else if(str8_match(extension, str8_lit("odin"), StringMatchFlag_CaseInsensitive))
-  {
-    kind = TXT_LangKind_Odin;
-  }
-  else if(str8_match(extension, str8_lit("jai"), StringMatchFlag_CaseInsensitive))
-  {
-    kind = TXT_LangKind_Jai;
-  }
-  else if(str8_match(extension, str8_lit("zig"), StringMatchFlag_CaseInsensitive))
-  {
-    kind = TXT_LangKind_Zig;
-  }
-  else if(str8_match(extension, str8_lit("rs"), StringMatchFlag_CaseInsensitive))
-  {
-    kind = TXT_LangKind_Rust;
+    if(str8_match(extension, txt_ext_lang_kind_table[idx].ext, 0))
+    {
+      kind = txt_ext_lang_kind_table[idx].kind;
+      break;
+    }
   }
   return kind;
 }
@@ -57,20 +30,7 @@ txt_lang_kind_from_extension(String8 extension)
 internal String8
 txt_extension_from_lang_kind(TXT_LangKind kind)
 {
-  String8 result = {0};
-  switch(kind)
-  {
-    case TXT_LangKind_Null:
-    case TXT_LangKind_COUNT:
-    case TXT_LangKind_DisasmX64Intel:
-    {}break;
-    case TXT_LangKind_C:               {result = str8_lit("c");}break;
-    case TXT_LangKind_CPlusPlus:       {result = str8_lit("cpp");}break;
-    case TXT_LangKind_Odin:            {result = str8_lit("odin");}break;
-    case TXT_LangKind_Jai:             {result = str8_lit("jai");}break;
-    case TXT_LangKind_Zig:             {result = str8_lit("zig");}break;
-    case TXT_LangKind_Rust:            {result = str8_lit("rs");}break;
-  }
+  String8 result = txt_lang_kind_ext_table[kind];
   return result;
 }
 
@@ -166,6 +126,343 @@ txt_token_array_from_list(Arena *arena, TXT_TokenList *list)
 ////////////////////////////////
 //~ rjf: Lexing Functions
 
+internal TXT_TokenArray
+txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, String8 string)
+{
+  ProfBeginFunction();
+  Temp scratch = scratch_begin(&arena, 1);
+  U64 chunk_size = Clamp(8, string.size/8, 4096);
+  
+  //- rjf: gather keywords
+  U64 keyword_slots_count = txt_keywords_from_lang_kind_table[lang_kind].count;
+  String8Node **keyword_slots = push_array(scratch.arena, String8Node *, keyword_slots_count);
+  for EachIndex(idx, txt_keywords_from_lang_kind_table[lang_kind].count)
+  {
+    String8 keyword = txt_keywords_from_lang_kind_table[lang_kind].v[idx];
+    U64 hash = u64_hash_from_str8(keyword);
+    U64 slot_idx = hash%keyword_slots_count;
+    String8Node *n = push_array(scratch.arena, String8Node, 1);
+    SLLStackPush(keyword_slots[slot_idx], n);
+    n->string = keyword;
+  }
+  
+  //- rjf: gather multichar symbols
+  String8Array multichar_symbols = txt_multichar_symbols_from_lang_kind_table[lang_kind];
+  
+  //- rjf: gather rules
+  U64 tokenizer_rule_slots_count = txt_tokenizer_rules_from_lang_kind_table[lang_kind].count;
+  TXT_TokenizerRulePtrNode **tokenizer_rule_opener_slots = push_array(scratch.arena, TXT_TokenizerRulePtrNode *, tokenizer_rule_slots_count);
+  TXT_TokenizerRulePtrNode **tokenizer_rule_closer_slots = push_array(scratch.arena, TXT_TokenizerRulePtrNode *, tokenizer_rule_slots_count);
+  {
+    TXT_TokenizerRuleArray rules = txt_tokenizer_rules_from_lang_kind_table[lang_kind];
+    for EachIndex(idx, rules.count)
+    {
+      TXT_TokenizerRule *r = &rules.v[idx];
+      U64 open_hash = u64_hash_from_str8(r->open_string);
+      U64 close_hash = u64_hash_from_str8(r->close_string);
+      U64 open_slot_idx = open_hash%tokenizer_rule_slots_count;
+      U64 close_slot_idx = close_hash%tokenizer_rule_slots_count;
+      TXT_TokenizerRulePtrNode *open_n = push_array(scratch.arena, TXT_TokenizerRulePtrNode, 1);
+      TXT_TokenizerRulePtrNode *close_n = push_array(scratch.arena, TXT_TokenizerRulePtrNode, 1);
+      open_n->v = r;
+      close_n->v = r;
+      SLLStackPush(tokenizer_rule_opener_slots[open_slot_idx], open_n);
+      SLLStackPush(tokenizer_rule_closer_slots[close_slot_idx], close_n);
+    }
+  }
+  
+  //- rjf: generate token list
+  TXT_TokenChunkList tokens = {0};
+  {
+    typedef struct TokenTask TokenTask;
+    struct TokenTask
+    {
+      TokenTask *next;
+      TXT_TokenizerRule *rule;
+      TXT_TokenKind kind;
+      U64 start_idx;
+    };
+    TXT_TokenizerRule *nil_rule = &txt_tokenizer_rules__null[0];
+    TokenTask *top_task = 0;
+    TokenTask *free_task = 0;
+    B32 escaped = 0;
+    B32 next_escaped = 0;
+    U64 byte_process_start_idx = 0;
+    for(U64 idx = 0; idx <= string.size;)
+    {
+      U8 byte      = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
+      U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
+      
+      // rjf: unpack next bytes
+      U8 next_bytes[] = {byte, next_byte};
+      String8 string_1byte = str8(next_bytes, 1);
+      String8 string_2byte = str8(next_bytes, 2);
+      
+      // rjf: update counter
+      if((idx-byte_process_start_idx) >= 1000 || idx == string.size)
+      {
+        add_progress(idx - byte_process_start_idx);
+        byte_process_start_idx = idx;
+      }
+      
+      // rjf: escaping
+      if(escaped && (byte != '\r' && byte != '\n'))
+      {
+        next_escaped = 0;
+      }
+      else if(!escaped && byte == '\\')
+      {
+        next_escaped = 1;
+      }
+      
+      // rjf: take starter, push new token tasks
+      {
+        TXT_TokenizerRule *new_rule = nil_rule;
+        TXT_TokenKind new_token_kind = TXT_TokenKind_Null;
+        
+        // rjf: use next bytes to look up a rule from the table
+        if(top_task == 0)
+        {
+          TXT_TokenizerRule *active_rule = top_task ? top_task->rule : nil_rule;
+          U64 hash_1byte = u64_hash_from_str8(string_1byte);
+          U64 hash_2byte = u64_hash_from_str8(string_2byte);
+          U64 slot_1byte = hash_1byte%tokenizer_rule_slots_count;
+          U64 slot_2byte = hash_2byte%tokenizer_rule_slots_count;
+          if(new_rule == nil_rule)
+          {
+            for EachNode(n, TXT_TokenizerRulePtrNode, tokenizer_rule_opener_slots[slot_2byte])
+            {
+              TXT_TokenizerRule *n_parent_rule = nil_rule;
+              if(0 < n->v->parent_num && n->v->parent_num <= txt_tokenizer_rules_from_lang_kind_table[lang_kind].count)
+              {
+                n_parent_rule = &txt_tokenizer_rules_from_lang_kind_table[lang_kind].v[n->v->parent_num-1];
+              }
+              if(str8_match(n->v->open_string, string_2byte, 0) &&
+                 active_rule == n_parent_rule)
+              {
+                new_rule = n->v;
+                break;
+              }
+            }
+          }
+          if(new_rule == nil_rule)
+          {
+            for EachNode(n, TXT_TokenizerRulePtrNode, tokenizer_rule_opener_slots[slot_1byte])
+            {
+              TXT_TokenizerRule *n_parent_rule = nil_rule;
+              if(0 < n->v->parent_num && n->v->parent_num <= txt_tokenizer_rules_from_lang_kind_table[lang_kind].count)
+              {
+                n_parent_rule = &txt_tokenizer_rules_from_lang_kind_table[lang_kind].v[n->v->parent_num-1];
+              }
+              if(str8_match(n->v->open_string, string_1byte, 0) &&
+                 active_rule == n_parent_rule)
+              {
+                new_rule = n->v;
+                break;
+              }
+            }
+          }
+          if(new_rule != nil_rule)
+          {
+            new_token_kind = new_rule->token_kind;
+          }
+        }
+        
+        // rjf: use next bytes to start a new token based on fallback rules
+        if(top_task == 0 && new_rule == nil_rule)
+        {
+          if(0){}
+          else if(char_is_space(byte))             { new_token_kind = TXT_TokenKind_Whitespace; }
+          else if(byte == '_' ||
+                  byte == '$' ||
+                  char_is_alpha(byte))             { new_token_kind = TXT_TokenKind_Identifier; }
+          else if(char_is_digit(byte, 10) ||
+                  (byte == '.' &&
+                   char_is_digit(next_byte, 10)))  { new_token_kind = TXT_TokenKind_Numeric; }
+          else if(byte == '~' || byte == '!' ||
+                  byte == '%' || byte == '^' ||
+                  byte == '&' || byte == '*' ||
+                  byte == '(' || byte == ')' ||
+                  byte == '-' || byte == '=' ||
+                  byte == '+' || byte == '[' ||
+                  byte == ']' || byte == '{' ||
+                  byte == '}' || byte == ':' ||
+                  byte == ';' || byte == ',' ||
+                  byte == '.' || byte == '<' ||
+                  byte == '>' || byte == '/' ||
+                  byte == '?' || byte == '|')      { new_token_kind = TXT_TokenKind_Symbol; }
+        }
+        
+        // rjf: start new token
+        if(new_token_kind != TXT_TokenKind_Null)
+        {
+          TokenTask *task = free_task;
+          if(task != 0)
+          {
+            SLLStackPop(free_task);
+          }
+          else
+          {
+            task = push_array(scratch.arena, TokenTask, 1);
+          }
+          SLLStackPush(top_task, task);
+          top_task->rule = new_rule;
+          top_task->kind = new_token_kind;
+          top_task->start_idx = idx;
+        }
+        
+        // rjf: invalid token kind -> emit error
+        else if(top_task == 0)
+        {
+          TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)};
+          txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
+        }
+      }
+      
+      // rjf: look for ender based on rule's closing symbol
+      U64 ender_pad = 0;
+      B32 ender_found = 0;
+      B32 task_pop = 0;
+      if(top_task != 0 && idx > top_task->start_idx)
+      {
+        TXT_TokenKind active_token_kind = top_task->kind;
+        TXT_TokenizerRule *active_rule = top_task->rule;
+        U64 active_token_start_idx = top_task->start_idx;
+        if(idx == string.size)
+        {
+          ender_pad = 0;
+          ender_found = 1;
+        }
+        else if(active_rule != nil_rule &&
+                str8_match(string_1byte, active_rule->close_string, 0) ||
+                str8_match(string_2byte, active_rule->close_string, 0))
+        {
+          ender_found = 1;
+          ender_pad = active_rule->close_advance;
+        }
+        else switch(active_token_kind)
+        {
+          default:{}break;
+          case TXT_TokenKind_Whitespace:
+          {
+            ender_found = !char_is_space(byte);
+          }break;
+          case TXT_TokenKind_Identifier:
+          {
+            ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128);
+          }break;
+          case TXT_TokenKind_Numeric:
+          {
+            ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '.' && byte != '\'');
+          }break;
+          case TXT_TokenKind_Symbol:
+          {
+            ender_found = (byte != '~' && byte != '!' &&
+                           byte != '%' && byte != '^' &&
+                           byte != '&' && byte != '*' &&
+                           byte != '(' && byte != ')' &&
+                           byte != '-' && byte != '=' &&
+                           byte != '+' && byte != '[' &&
+                           byte != ']' && byte != '{' &&
+                           byte != '}' && byte != ':' &&
+                           byte != ';' && byte != ',' &&
+                           byte != '.' && byte != '<' &&
+                           byte != '>' && byte != '/' &&
+                           byte != '?' && byte != '|');
+          }break;
+        }
+      }
+      
+      // rjf: next byte is ender => emit token
+      if(ender_found)
+      {
+        TXT_Token token = {top_task->kind, r1u64(top_task->start_idx, idx+ender_pad)};
+        TokenTask *popped = top_task;
+        SLLStackPop(top_task);
+        SLLStackPush(free_task, popped);
+        
+        // rjf: trim \r's off of end
+        {
+          String8 token_string = str8_substr(string, token.range);
+          if(token_string.size != 0 && token_string.str[token_string.size-1] == '\r')
+          {
+            token.range.max -= 1;
+            ender_pad += 1;
+          }
+        }
+        
+        // rjf: identifier -> keyword in special cases
+        if(token.kind == TXT_TokenKind_Identifier)
+        {
+          String8 token_string = str8_substr(string, token.range);
+          U64 hash = u64_hash_from_str8(token_string);
+          U64 slot_idx = hash%keyword_slots_count;
+          for EachNode(n, String8Node, keyword_slots[slot_idx])
+          {
+            if(str8_match(token_string, n->string, 0))
+            {
+              token.kind = TXT_TokenKind_Keyword;
+              break;
+            }
+          }
+          txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
+        }
+        
+        // rjf: split symbols by maximum-munch-rule
+        else if(token.kind == TXT_TokenKind_Symbol)
+        {
+          String8 token_string = str8_substr(string, token.range);
+          for(U64 off = 0, next_off = token_string.size; off < token_string.size; off = next_off)
+          {
+            B32 found = 0;
+            for(U64 idx = 0; idx < multichar_symbols.count; idx += 1)
+            {
+              if(str8_match(str8_substr(token_string, r1u64(off, off+multichar_symbols.v[idx].size)),
+                            multichar_symbols.v[idx],
+                            0))
+              {
+                found = 1;
+                next_off = off + Max(1, multichar_symbols.v[idx].size);
+                TXT_Token sub_token = {TXT_TokenKind_Symbol, r1u64(token.range.min+off, token.range.min+next_off)};
+                txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &sub_token);
+                break;
+              }
+            }
+            if(!found)
+            {
+              next_off = off+1;
+              TXT_Token sub_token = {TXT_TokenKind_Symbol, r1u64(token.range.min+off, token.range.min+next_off)};
+              txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &sub_token);
+            }
+          }
+        }
+        
+        // rjf: all other tokens
+        else
+        {
+          txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
+        }
+        
+        // rjf: increment by ender padding
+        idx += ender_pad;
+      }
+      
+      // rjf: advance by 1 byte if we haven't found an ender
+      if(!ender_found)
+      {
+        idx += 1;
+      }
+      escaped = next_escaped;
+    }
+  }
+  
+  //- rjf: token list -> token array
+  TXT_TokenArray result = txt_token_array_from_chunk_list(arena, &tokens);
+  scratch_end(scratch);
+  ProfEnd();
+  return result;
+}
+
 internal TXT_TokenArray
 txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string)
 {
@@ -1451,9 +1748,9 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
                  char_is_digit(next_byte, 10)))  { active_token_kind = TXT_TokenKind_Numeric; }
         else if(byte == '"')                     { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; }
         else if((byte == 'c' || byte == 'b') &&
-                 next_byte == '"')               { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; starter_pad = 1; }
+                next_byte == '"')               { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; starter_pad = 1; }
         else if((byte == 'c' || byte == 'b') &&
-                 next_byte == 'r')               { active_token_kind = TXT_TokenKind_Identifier; token_may_be_string = 1; }
+                next_byte == 'r')               { active_token_kind = TXT_TokenKind_Identifier; token_may_be_string = 1; }
         else if(byte == '_' ||
                 char_is_alpha(byte))             { active_token_kind = TXT_TokenKind_Identifier; }
         else if(byte == '/' && next_byte == '/') { active_token_kind = TXT_TokenKind_Comment; starter_pad = 1; }
@@ -1472,7 +1769,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
                 byte == '?' || byte == '|')      { active_token_kind = TXT_TokenKind_Symbol; }
         else if(byte == '\'')                    { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; token_may_be_lifetime = 1; }
         else if((byte == 'c' || byte == 'b') &&
-                 next_byte == '\'')              { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; starter_pad = 1; }
+                next_byte == '\'')              { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; starter_pad = 1; }
         
         // rjf: start new token
         if(active_token_kind != TXT_TokenKind_Null)
@@ -2459,10 +2756,17 @@ txt_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *gen_ou
     set_progress(Min(data.size, 1024) + data.size + data.size);
     
     //- rjf: lex function * data -> tokens
+#if 1
     if(lane_idx() == 0 && lex_function != 0)
     {
       shared->info.tokens = lex_function(shared->arena, 0, data);
     }
+#else
+    if(lane_idx() == 0)
+    {
+      shared->info.tokens = txt_token_array_from_lang_kind_string(shared->arena, lang, data);
+    }
+#endif
     lane_sync();
     set_progress(Min(data.size, 1024) + data.size + data.size + data.size*(lex_function != 0));
     TXT_TokenArray tokens = shared->info.tokens;

src/text/text.h

@@ -32,6 +32,32 @@ typedef enum TXT_TokenKind
 }
 TXT_TokenKind;
 
+typedef struct TXT_TokenizerRule TXT_TokenizerRule;
+struct TXT_TokenizerRule
+{
+  TXT_TokenKind token_kind;
+  String8 open_string;
+  String8 close_string;
+  U32 close_advance;
+  B32 nesting;
+  B32 escaping;
+  U32 parent_num;
+};
+
+typedef struct TXT_TokenizerRulePtrNode TXT_TokenizerRulePtrNode;
+struct TXT_TokenizerRulePtrNode
+{
+  TXT_TokenizerRulePtrNode *next;
+  TXT_TokenizerRule *v;
+};
+
+typedef struct TXT_TokenizerRuleArray TXT_TokenizerRuleArray;
+struct TXT_TokenizerRuleArray
+{
+  TXT_TokenizerRule *v;
+  U64 count;
+};
+
 typedef struct TXT_Token TXT_Token;
 struct TXT_Token
 {
@@ -138,21 +164,12 @@ struct TXT_LineTokensSlice
 };
 
 ////////////////////////////////
-//~ rjf: Language Kind Types
+//~ rjf: Generated Code
 
-typedef enum TXT_LangKind
-{
-  TXT_LangKind_Null,
-  TXT_LangKind_C,
-  TXT_LangKind_CPlusPlus,
-  TXT_LangKind_Odin,
-  TXT_LangKind_Jai,
-  TXT_LangKind_Zig,
-  TXT_LangKind_Rust,
-  TXT_LangKind_DisasmX64Intel,
-  TXT_LangKind_COUNT
-}
-TXT_LangKind;
+#include "generated/text.meta.h"
+
+////////////////////////////////
+//~ rjf: Language Kind Types
 
 typedef TXT_TokenArray TXT_LangLexFunctionType(Arena *arena, U64 *bytes_processed_counter, String8 string);
 
@@ -180,6 +197,8 @@ internal TXT_TokenArray txt_token_array_from_list(Arena *arena, TXT_TokenList *l
 ////////////////////////////////
 //~ rjf: Lexing Functions
 
+internal TXT_TokenArray txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, String8 string);
+
 internal TXT_TokenArray txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string);
 internal TXT_TokenArray txt_token_array_from_string__odin(Arena *arena, U64 *bytes_processed_counter, String8 string);
 internal TXT_TokenArray txt_token_array_from_string__jai(Arena *arena, U64 *bytes_processed_counter, String8 string);

src/text/text.mdesk

@@ -0,0 +1,246 @@
+// Copyright (c) Epic Games Tools
+// Licensed under the MIT license (https://opensource.org/license/mit/)
+
+@table(name       ext                keywords_table     multichar_symbol_table     tokenizer_rules_table)
+TXT_LangKindTable:
+{
+  { Null           null               null               null                       null}
+  { C              c                  c                  c                          c   }
+  { CPlusPlus      cpp                cpp                c                          c   }
+  { Odin           odin               odin               c                          null}
+  { Jai            jai                jai                c                          null}
+  { Zig            zig                zig                c                          null}
+  { Rust           rs                 rust               c                          null}
+  { DisasmX64Intel disasm_x64_intel   null               null                       null}
+}
+
+@table(string kind)
+TXT_LangKindExtTable:
+{
+  {"h"    C        }
+  {"cxx"  CPlusPlus}
+  {"cc"   CPlusPlus}
+  {"c++"  CPlusPlus}
+  {"ixx"  CPlusPlus}
+  {"cxxm" CPlusPlus}
+  {"c++m" CPlusPlus}
+  {"ccm"  CPlusPlus}
+  {"cppm" CPlusPlus}
+  {"mpp"  CPlusPlus}
+  {"C"    CPlusPlus}
+  {"hpp"  CPlusPlus}
+  {"hxx"  CPlusPlus}
+  {"hh"   CPlusPlus}
+  {"h++"  CPlusPlus}
+  {"H"    CPlusPlus}
+}
+
+@enum TXT_LangKind:
+{
+  @expand(TXT_LangKindTable a) `$(a.name)`,
+  COUNT
+}
+
+@data(String8)
+txt_lang_kind_ext_table:
+{
+  @expand(TXT_LangKindTable a) `str8_lit_comp("$(a.ext)")`
+}
+
+@data(`struct {String8 ext; TXT_LangKind kind;}`) @c_file
+txt_ext_lang_kind_table:
+{
+  @expand(TXT_LangKindTable a) `{str8_lit_comp("$(a.ext)"), TXT_LangKind_$(a.name)}`,
+  @expand(TXT_LangKindExtTable a) `{str8_lit_comp("$(a.string)"), TXT_LangKind_$(a.kind)}`,
+}
+
+@data(String8Array) txt_keywords_from_lang_kind_table:
+{
+  @expand(TXT_LangKindTable a) `{txt_keywords__$(a.keywords_table), ArrayCount(txt_keywords__$(a.keywords_table))}`
+}
+
+@data(String8Array) txt_multichar_symbols_from_lang_kind_table:
+{
+  @expand(TXT_LangKindTable a) `{txt_multichar_symbols__$(a.multichar_symbol_table), ArrayCount(txt_multichar_symbols__$(a.multichar_symbol_table))}`
+}
+
+@data(TXT_TokenizerRuleArray) txt_tokenizer_rules_from_lang_kind_table:
+{
+  @expand(TXT_LangKindTable a) `{txt_tokenizer_rules__$(a.tokenizer_rules_table), ArrayCount(txt_tokenizer_rules__$(a.tokenizer_rules_table))}`
+}
+
+@data(String8) txt_keywords__null:{`{0}`}
+@data(String8) txt_multichar_symbols__null:{`{0}`}
+@data(TXT_TokenizerRule) txt_tokenizer_rules__null:{`{TXT_TokenKind_Null}`}
+
+////////////////////////////////
+//~ rjf: C
+
+@table(string)
+txt_keywords__c:
+{
+  {"auto"} {"break"} {"case"} {"char"}
+  {"const"} {"continue"} {"default"} {"do"}
+  {"double"} {"else"} {"enum"} {"extern"}
+  {"float"} {"for"} {"goto"} {"if"}
+  {"int"} {"long"} {"register"} {"return"}
+  {"short"} {"signed"} {"sizeof"} {"static"}
+  {"struct"} {"switch"} {"typedef"} {"union"}
+  {"unsigned"} {"void"} {"volatile"} {"while"}
+}
+@data(String8) txt_keywords__c: {@expand(txt_keywords__c a) `str8_lit_comp("$(a.string)")`}
+
+@table(string)
+txt_multichar_symbols__c:
+{
+  {"<<"}
+  {">>"}
+  {"<="}
+  {">="}
+  {"=="}
+  {"!="}
+  {"&&"}
+  {"||"}
+  {"|="}
+  {"&="}
+  {"^="}
+  {"~="}
+  {"+="}
+  {"-="}
+  {"*="}
+  {"/="}
+  {"%="}
+  {"<<="}
+  {">>="}
+  {"->"}
+}
+@data(String8) txt_multichar_symbols__c: {@expand(txt_multichar_symbols__c a) `str8_lit_comp("$(a.string)")`}
+
+@table(token_kind open_string close_string close_advance nesting escaping parent_num) txt_tokenizer_rules__c:
+{
+  {Comment "//"  "\\n" 0 0 1 0}
+  {Comment "/*"  "*/"  2 0 0 0}
+  {Meta    "#"   "\\n" 0 0 1 0}
+  {String  '\\"' '\\"' 1 0 1 0}
+  {String  "'"   "'"   1 0 1 0}
+  {String  '\\"' '\\"' 1 0 1 3}
+  {String  "<"   ">"   1 0 1 3}
+}
+@data(TXT_TokenizerRule) txt_tokenizer_rules__c:
+{
+  @expand(txt_tokenizer_rules__c a) `{TXT_TokenKind_$(a.token_kind), str8_lit_comp("$(a.open_string)"), str8_lit_comp("$(a.close_string)"), $(a.close_advance), $(a.nesting), $(a.escaping), $(a.parent_num)}`
+}
+
+////////////////////////////////
+//~ rjf: C++
+
+@table(string)
+txt_keywords__cpp:
+{
+  {"alignas"} {"alignof"} {"and"} {"and_eq"}
+  {"asm"} {"atomic_cancel"} {"atomic_commit"} {"atomic_noexcept"}
+  {"auto"} {"bitand"} {"bitor"} {"bool"}
+  {"break"} {"case"} {"catch"} {"char"}
+  {"char8_t"} {"char16_t"} {"char32_t"} {"class"}
+  {"compl"} {"concept"} {"const"} {"consteval"}
+  {"constexpr"} {"constinit"} {"const_cast"} {"continue"}
+  {"co_await"} {"co_return"} {"co_yield"} {"decltype"}
+  {"default"} {"delete"} {"do"} {"double"}
+  {"dynamic_cast"} {"else"} {"enum"} {"explicit"}
+  {"export"} {"extern"} {"false"} {"float"}
+  {"for"} {"friend"} {"goto"} {"if"}
+  {"inline"} {"int"} {"long"} {"mutable"}
+  {"namespace"} {"new"} {"noexcept"} {"not"}
+  {"not_eq"} {"nullptr"} {"operator"} {"or"}
+  {"or_eq"} {"private"} {"protected"} {"public"}
+  {"reflexpr"} {"register"} {"reinterpret_cast"} {"requires"}
+  {"return"} {"short"} {"signed"} {"sizeof"}
+  {"static"} {"static_assert"} {"static_cast"} {"struct"}
+  {"switch"} {"synchronized"} {"template"} {"this"}
+  {"thread_local"} {"throw"} {"true"} {"try"}
+  {"typedef"} {"typeid"} {"typename"} {"union"}
+  {"unsigned"} {"using"} {"virtual"} {"void"}
+  {"volatile"} {"wchar_t"} {"while"} {"xor"}
+  {"xor_eq"}
+}
+@data(String8) txt_keywords__cpp: {@expand(txt_keywords__cpp a) `str8_lit_comp("$(a.string)")`}
+
+////////////////////////////////
+//~ rjf: Odin
+
+@table(string)
+txt_keywords__odin:
+{
+  {"align_of"} {"asm"} {"auto_cast"} {"bit_set"}
+  {"break"} {"case"} {"cast"} {"context"}
+  {"continue"} {"defer"} {"distinct"} {"do"}
+  {"dynamic"} {"else"} {"enum"} {"fallthrough"}
+  {"for"} {"foreign"} {"if"} {"in"}
+  {"map"} {"matrix"} {"not_in"} {"or_break"}
+  {"or_continue"} {"or_else"} {"or_return"} {"package"}
+  {"proc"} {"return"} {"size_of"} {"struct"}
+  {"switch"} {"transmute"} {"typeid"} {"union"}
+  {"using"} {"when"} {"where"} {"import"}
+}
+@data(String8) txt_keywords__odin: {@expand(txt_keywords__odin a) `str8_lit_comp("$(a.string)")`}
+
+////////////////////////////////
+//~ rjf: Jai
+
+@table(string)
+txt_keywords__jai:
+{
+  {"bool"} {"true"} {"false"} {"int"}
+  {"s8"} {"u8"} {"s16"} {"u16"}
+  {"s32"} {"u32"} {"s64"} {"u64"}
+  {"s128"} {"u128"} {"float"} {"float32"}
+  {"float64"} {"void"} {"enum"} {"enum_flags"}
+  {"size_of"} {"string"} {"type_of"} {"cast"}
+  {"if"} {"ifs"} {"then"} {"else"}
+  {"case"} {"for"} {"while"} {"break"}
+  {"continue"} {"remove"} {"return"} {"inline"}
+  {"null"} {"defer"} {"xx"}
+}
+@data(String8) txt_keywords__jai: {@expand(txt_keywords__jai a) `str8_lit_comp("$(a.string)")`}
+
+////////////////////////////////
+//~ rjf: Zig
+
+@table(string)
+txt_keywords__zig:
+{
+  {"addrspace"} {"align"} {"allowzero"} {"and"}
+  {"anyframe"} {"anytype"} {"asm"} {"async"}
+  {"await"} {"break"} {"callconv"} {"catch"}
+  {"comptime"} {"const"} {"continue"} {"defer"}
+  {"else"} {"enum"} {"errdefer"} {"error"}
+  {"export"} {"extern"} {"fn"} {"for"}
+  {"if"} {"inline"} {"noalias"} {"nosuspend"}
+  {"noinline"} {"opaque"} {"or"} {"orelse"}
+  {"packed"} {"pub"} {"resume"} {"return"}
+  {"linksection"} {"struct"} {"suspend"} {"switch"}
+  {"test"} {"threadlocal"} {"try"} {"union"}
+  {"unreachable"} {"usingnamespace"} {"var"} {"volatile"}
+  {"while"}
+}
+@data(String8) txt_keywords__zig: {@expand(txt_keywords__zig a) `str8_lit_comp("$(a.string)")`}
+
+////////////////////////////////
+//~ rjf: Rust
+
+@table(string)
+txt_keywords__rust:
+{
+  {"as"} {"break"} {"const"} {"continue"}
+  {"crate"} {"else"} {"enum"} {"extern"}
+  {"false"} {"fn"} {"for"} {"if"}
+  {"impl"} {"in"} {"let"} {"loop"}
+  {"match"} {"mod"} {"move"} {"mut"}
+  {"pub"} {"ref"} {"return"} {"self"}
+  {"Self"} {"static"} {"struct"} {"super"}
+  {"trait"} {"true"} {"type"} {"unsafe"}
+  {"use"} {"where"} {"while"} {"yield"}
+  {"async"} {"await"} {"dyn"} {"macro_rules"}
+  {"raw"} {"safe"} {"union"}
+}
+@data(String8) txt_keywords__rust: {@expand(txt_keywords__rust a) `str8_lit_comp("$(a.string)")`}