From 82bc133ba58acead7131daf2829954ae203afebb Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Tue, 4 Nov 2025 15:56:42 -0800 Subject: [PATCH] first pass at unified lexer path, not plugged in yet --- src/text/generated/text.meta.c | 458 +++++++++++++++++++++++++++++++++ src/text/generated/text.meta.h | 41 +++ src/text/text.c | 450 ++++++++++++++++++++++++++------ src/text/text.h | 47 +++- src/text/text.mdesk | 246 ++++++++++++++++++ 5 files changed, 1155 insertions(+), 87 deletions(-) create mode 100644 src/text/generated/text.meta.c create mode 100644 src/text/generated/text.meta.h create mode 100644 src/text/text.mdesk diff --git a/src/text/generated/text.meta.c b/src/text/generated/text.meta.c new file mode 100644 index 00000000..bb00c67a --- /dev/null +++ b/src/text/generated/text.meta.c @@ -0,0 +1,458 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +//- GENERATED CODE + +C_LINKAGE_BEGIN +String8 txt_lang_kind_ext_table[8] = +{ +str8_lit_comp("null"), +str8_lit_comp("c"), +str8_lit_comp("cpp"), +str8_lit_comp("odin"), +str8_lit_comp("jai"), +str8_lit_comp("zig"), +str8_lit_comp("rs"), +str8_lit_comp("disasm_x64_intel"), +}; + +struct {String8 ext; TXT_LangKind kind;} txt_ext_lang_kind_table[24] = +{ +{str8_lit_comp("null"), TXT_LangKind_Null}, +{str8_lit_comp("c"), TXT_LangKind_C}, +{str8_lit_comp("cpp"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("odin"), TXT_LangKind_Odin}, +{str8_lit_comp("jai"), TXT_LangKind_Jai}, +{str8_lit_comp("zig"), TXT_LangKind_Zig}, +{str8_lit_comp("rs"), TXT_LangKind_Rust}, +{str8_lit_comp("disasm_x64_intel"), TXT_LangKind_DisasmX64Intel}, +{str8_lit_comp("h"), TXT_LangKind_C}, +{str8_lit_comp("cxx"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("cc"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("c++"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("ixx"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("cxxm"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("c++m"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("ccm"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("cppm"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("mpp"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("C"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("hpp"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("hxx"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("hh"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("h++"), TXT_LangKind_CPlusPlus}, +{str8_lit_comp("H"), TXT_LangKind_CPlusPlus}, +}; + +String8Array txt_keywords_from_lang_kind_table[8] = +{ +{txt_keywords__null, ArrayCount(txt_keywords__null)}, +{txt_keywords__c, ArrayCount(txt_keywords__c)}, +{txt_keywords__cpp, ArrayCount(txt_keywords__cpp)}, +{txt_keywords__odin, ArrayCount(txt_keywords__odin)}, +{txt_keywords__jai, ArrayCount(txt_keywords__jai)}, +{txt_keywords__zig, ArrayCount(txt_keywords__zig)}, +{txt_keywords__rust, ArrayCount(txt_keywords__rust)}, +{txt_keywords__null, ArrayCount(txt_keywords__null)}, +}; + +String8Array txt_multichar_symbols_from_lang_kind_table[8] = +{ +{txt_multichar_symbols__null, ArrayCount(txt_multichar_symbols__null)}, +{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)}, +{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)}, +{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)}, +{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)}, +{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)}, +{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)}, +{txt_multichar_symbols__null, ArrayCount(txt_multichar_symbols__null)}, +}; + +TXT_TokenizerRuleArray txt_tokenizer_rules_from_lang_kind_table[8] = +{ +{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)}, +{txt_tokenizer_rules__c, ArrayCount(txt_tokenizer_rules__c)}, +{txt_tokenizer_rules__c, ArrayCount(txt_tokenizer_rules__c)}, +{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)}, +{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)}, +{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)}, +{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)}, +{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)}, +}; + +String8 txt_keywords__null[1] = +{ +{0}, +}; + +String8 txt_multichar_symbols__null[1] = +{ +{0}, +}; + +TXT_TokenizerRule txt_tokenizer_rules__null[1] = +{ +{TXT_TokenKind_Null}, +}; + +String8 txt_keywords__c[32] = +{ +str8_lit_comp("auto"), +str8_lit_comp("break"), +str8_lit_comp("case"), +str8_lit_comp("char"), +str8_lit_comp("const"), +str8_lit_comp("continue"), +str8_lit_comp("default"), +str8_lit_comp("do"), +str8_lit_comp("double"), +str8_lit_comp("else"), +str8_lit_comp("enum"), +str8_lit_comp("extern"), +str8_lit_comp("float"), +str8_lit_comp("for"), +str8_lit_comp("goto"), +str8_lit_comp("if"), +str8_lit_comp("int"), +str8_lit_comp("long"), +str8_lit_comp("register"), +str8_lit_comp("return"), +str8_lit_comp("short"), +str8_lit_comp("signed"), +str8_lit_comp("sizeof"), +str8_lit_comp("static"), +str8_lit_comp("struct"), +str8_lit_comp("switch"), +str8_lit_comp("typedef"), +str8_lit_comp("union"), +str8_lit_comp("unsigned"), +str8_lit_comp("void"), +str8_lit_comp("volatile"), +str8_lit_comp("while"), +}; + +String8 txt_multichar_symbols__c[20] = +{ +str8_lit_comp("<<"), +str8_lit_comp(">>"), +str8_lit_comp("<="), +str8_lit_comp(">="), +str8_lit_comp("=="), +str8_lit_comp("!="), +str8_lit_comp("&&"), +str8_lit_comp("||"), +str8_lit_comp("|="), +str8_lit_comp("&="), +str8_lit_comp("^="), +str8_lit_comp("~="), +str8_lit_comp("+="), +str8_lit_comp("-="), +str8_lit_comp("*="), +str8_lit_comp("/="), +str8_lit_comp("%="), +str8_lit_comp("<<="), +str8_lit_comp(">>="), +str8_lit_comp("->"), +}; + +TXT_TokenizerRule txt_tokenizer_rules__c[7] = +{ +{TXT_TokenKind_Comment, str8_lit_comp("//"), str8_lit_comp("\n"), 0, 0, 1, 0}, +{TXT_TokenKind_Comment, str8_lit_comp("/*"), str8_lit_comp("*/"), 2, 0, 0, 0}, +{TXT_TokenKind_Meta, str8_lit_comp("#"), str8_lit_comp("\n"), 0, 0, 1, 0}, +{TXT_TokenKind_String, str8_lit_comp("\""), str8_lit_comp("\""), 1, 0, 1, 0}, +{TXT_TokenKind_String, str8_lit_comp("'"), str8_lit_comp("'"), 1, 0, 1, 0}, +{TXT_TokenKind_String, str8_lit_comp("\""), str8_lit_comp("\""), 1, 0, 1, 3}, +{TXT_TokenKind_String, str8_lit_comp("<"), str8_lit_comp(">"), 1, 0, 1, 3}, +}; + +String8 txt_keywords__cpp[97] = +{ +str8_lit_comp("alignas"), +str8_lit_comp("alignof"), +str8_lit_comp("and"), +str8_lit_comp("and_eq"), +str8_lit_comp("asm"), +str8_lit_comp("atomic_cancel"), +str8_lit_comp("atomic_commit"), +str8_lit_comp("atomic_noexcept"), +str8_lit_comp("auto"), +str8_lit_comp("bitand"), +str8_lit_comp("bitor"), +str8_lit_comp("bool"), +str8_lit_comp("break"), +str8_lit_comp("case"), +str8_lit_comp("catch"), +str8_lit_comp("char"), +str8_lit_comp("char8_t"), +str8_lit_comp("char16_t"), +str8_lit_comp("char32_t"), +str8_lit_comp("class"), +str8_lit_comp("compl"), +str8_lit_comp("concept"), +str8_lit_comp("const"), +str8_lit_comp("consteval"), +str8_lit_comp("constexpr"), +str8_lit_comp("constinit"), +str8_lit_comp("const_cast"), +str8_lit_comp("continue"), +str8_lit_comp("co_await"), +str8_lit_comp("co_return"), +str8_lit_comp("co_yield"), +str8_lit_comp("decltype"), +str8_lit_comp("default"), +str8_lit_comp("delete"), +str8_lit_comp("do"), +str8_lit_comp("double"), +str8_lit_comp("dynamic_cast"), +str8_lit_comp("else"), +str8_lit_comp("enum"), +str8_lit_comp("explicit"), +str8_lit_comp("export"), +str8_lit_comp("extern"), +str8_lit_comp("false"), +str8_lit_comp("float"), +str8_lit_comp("for"), +str8_lit_comp("friend"), +str8_lit_comp("goto"), +str8_lit_comp("if"), +str8_lit_comp("inline"), +str8_lit_comp("int"), +str8_lit_comp("long"), +str8_lit_comp("mutable"), +str8_lit_comp("namespace"), +str8_lit_comp("new"), +str8_lit_comp("noexcept"), +str8_lit_comp("not"), +str8_lit_comp("not_eq"), +str8_lit_comp("nullptr"), +str8_lit_comp("operator"), +str8_lit_comp("or"), +str8_lit_comp("or_eq"), +str8_lit_comp("private"), +str8_lit_comp("protected"), +str8_lit_comp("public"), +str8_lit_comp("reflexpr"), +str8_lit_comp("register"), +str8_lit_comp("reinterpret_cast"), +str8_lit_comp("requires"), +str8_lit_comp("return"), +str8_lit_comp("short"), +str8_lit_comp("signed"), +str8_lit_comp("sizeof"), +str8_lit_comp("static"), +str8_lit_comp("static_assert"), +str8_lit_comp("static_cast"), +str8_lit_comp("struct"), +str8_lit_comp("switch"), +str8_lit_comp("synchronized"), +str8_lit_comp("template"), +str8_lit_comp("this"), +str8_lit_comp("thread_local"), +str8_lit_comp("throw"), +str8_lit_comp("true"), +str8_lit_comp("try"), +str8_lit_comp("typedef"), +str8_lit_comp("typeid"), +str8_lit_comp("typename"), +str8_lit_comp("union"), +str8_lit_comp("unsigned"), +str8_lit_comp("using"), +str8_lit_comp("virtual"), +str8_lit_comp("void"), +str8_lit_comp("volatile"), +str8_lit_comp("wchar_t"), +str8_lit_comp("while"), +str8_lit_comp("xor"), +str8_lit_comp("xor_eq"), +}; + +String8 txt_keywords__odin[40] = +{ +str8_lit_comp("align_of"), +str8_lit_comp("asm"), +str8_lit_comp("auto_cast"), +str8_lit_comp("bit_set"), +str8_lit_comp("break"), +str8_lit_comp("case"), +str8_lit_comp("cast"), +str8_lit_comp("context"), +str8_lit_comp("continue"), +str8_lit_comp("defer"), +str8_lit_comp("distinct"), +str8_lit_comp("do"), +str8_lit_comp("dynamic"), +str8_lit_comp("else"), +str8_lit_comp("enum"), +str8_lit_comp("fallthrough"), +str8_lit_comp("for"), +str8_lit_comp("foreign"), +str8_lit_comp("if"), +str8_lit_comp("in"), +str8_lit_comp("map"), +str8_lit_comp("matrix"), +str8_lit_comp("not_in"), +str8_lit_comp("or_break"), +str8_lit_comp("or_continue"), +str8_lit_comp("or_else"), +str8_lit_comp("or_return"), +str8_lit_comp("package"), +str8_lit_comp("proc"), +str8_lit_comp("return"), +str8_lit_comp("size_of"), +str8_lit_comp("struct"), +str8_lit_comp("switch"), +str8_lit_comp("transmute"), +str8_lit_comp("typeid"), +str8_lit_comp("union"), +str8_lit_comp("using"), +str8_lit_comp("when"), +str8_lit_comp("where"), +str8_lit_comp("import"), +}; + +String8 txt_keywords__jai[39] = +{ +str8_lit_comp("bool"), +str8_lit_comp("true"), +str8_lit_comp("false"), +str8_lit_comp("int"), +str8_lit_comp("s8"), +str8_lit_comp("u8"), +str8_lit_comp("s16"), +str8_lit_comp("u16"), +str8_lit_comp("s32"), +str8_lit_comp("u32"), +str8_lit_comp("s64"), +str8_lit_comp("u64"), +str8_lit_comp("s128"), +str8_lit_comp("u128"), +str8_lit_comp("float"), +str8_lit_comp("float32"), +str8_lit_comp("float64"), +str8_lit_comp("void"), +str8_lit_comp("enum"), +str8_lit_comp("enum_flags"), +str8_lit_comp("size_of"), +str8_lit_comp("string"), +str8_lit_comp("type_of"), +str8_lit_comp("cast"), +str8_lit_comp("if"), +str8_lit_comp("ifs"), +str8_lit_comp("then"), +str8_lit_comp("else"), +str8_lit_comp("case"), +str8_lit_comp("for"), +str8_lit_comp("while"), +str8_lit_comp("break"), +str8_lit_comp("continue"), +str8_lit_comp("remove"), +str8_lit_comp("return"), +str8_lit_comp("inline"), +str8_lit_comp("null"), +str8_lit_comp("defer"), +str8_lit_comp("xx"), +}; + +String8 txt_keywords__zig[49] = +{ +str8_lit_comp("addrspace"), +str8_lit_comp("align"), +str8_lit_comp("allowzero"), +str8_lit_comp("and"), +str8_lit_comp("anyframe"), +str8_lit_comp("anytype"), +str8_lit_comp("asm"), +str8_lit_comp("async"), +str8_lit_comp("await"), +str8_lit_comp("break"), +str8_lit_comp("callconv"), +str8_lit_comp("catch"), +str8_lit_comp("comptime"), +str8_lit_comp("const"), +str8_lit_comp("continue"), +str8_lit_comp("defer"), +str8_lit_comp("else"), +str8_lit_comp("enum"), +str8_lit_comp("errdefer"), +str8_lit_comp("error"), +str8_lit_comp("export"), +str8_lit_comp("extern"), +str8_lit_comp("fn"), +str8_lit_comp("for"), +str8_lit_comp("if"), +str8_lit_comp("inline"), +str8_lit_comp("noalias"), +str8_lit_comp("nosuspend"), +str8_lit_comp("noinline"), +str8_lit_comp("opaque"), +str8_lit_comp("or"), +str8_lit_comp("orelse"), +str8_lit_comp("packed"), +str8_lit_comp("pub"), +str8_lit_comp("resume"), +str8_lit_comp("return"), +str8_lit_comp("linksection"), +str8_lit_comp("struct"), +str8_lit_comp("suspend"), +str8_lit_comp("switch"), +str8_lit_comp("test"), +str8_lit_comp("threadlocal"), +str8_lit_comp("try"), +str8_lit_comp("union"), +str8_lit_comp("unreachable"), +str8_lit_comp("usingnamespace"), +str8_lit_comp("var"), +str8_lit_comp("volatile"), +str8_lit_comp("while"), +}; + +String8 txt_keywords__rust[43] = +{ +str8_lit_comp("as"), +str8_lit_comp("break"), +str8_lit_comp("const"), +str8_lit_comp("continue"), +str8_lit_comp("crate"), +str8_lit_comp("else"), +str8_lit_comp("enum"), +str8_lit_comp("extern"), +str8_lit_comp("false"), +str8_lit_comp("fn"), +str8_lit_comp("for"), +str8_lit_comp("if"), +str8_lit_comp("impl"), +str8_lit_comp("in"), +str8_lit_comp("let"), +str8_lit_comp("loop"), +str8_lit_comp("match"), +str8_lit_comp("mod"), +str8_lit_comp("move"), +str8_lit_comp("mut"), +str8_lit_comp("pub"), +str8_lit_comp("ref"), +str8_lit_comp("return"), +str8_lit_comp("self"), +str8_lit_comp("Self"), +str8_lit_comp("static"), +str8_lit_comp("struct"), +str8_lit_comp("super"), +str8_lit_comp("trait"), +str8_lit_comp("true"), +str8_lit_comp("type"), +str8_lit_comp("unsafe"), +str8_lit_comp("use"), +str8_lit_comp("where"), +str8_lit_comp("while"), +str8_lit_comp("yield"), +str8_lit_comp("async"), +str8_lit_comp("await"), +str8_lit_comp("dyn"), +str8_lit_comp("macro_rules"), +str8_lit_comp("raw"), +str8_lit_comp("safe"), +str8_lit_comp("union"), +}; + +C_LINKAGE_END + diff --git a/src/text/generated/text.meta.h b/src/text/generated/text.meta.h new file mode 100644 index 00000000..f4cc1eab --- /dev/null +++ b/src/text/generated/text.meta.h @@ -0,0 +1,41 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +//- GENERATED CODE + +#ifndef TEXT_META_H +#define TEXT_META_H + +typedef enum TXT_LangKind +{ +TXT_LangKind_Null, +TXT_LangKind_C, +TXT_LangKind_CPlusPlus, +TXT_LangKind_Odin, +TXT_LangKind_Jai, +TXT_LangKind_Zig, +TXT_LangKind_Rust, +TXT_LangKind_DisasmX64Intel, +TXT_LangKind_COUNT, +} TXT_LangKind; + +C_LINKAGE_BEGIN +extern String8 txt_lang_kind_ext_table[8]; +extern String8Array txt_keywords_from_lang_kind_table[8]; +extern String8Array txt_multichar_symbols_from_lang_kind_table[8]; +extern TXT_TokenizerRuleArray txt_tokenizer_rules_from_lang_kind_table[8]; +extern String8 txt_keywords__null[1]; +extern String8 txt_multichar_symbols__null[1]; +extern TXT_TokenizerRule txt_tokenizer_rules__null[1]; +extern String8 txt_keywords__c[32]; +extern String8 txt_multichar_symbols__c[20]; +extern TXT_TokenizerRule txt_tokenizer_rules__c[7]; +extern String8 txt_keywords__cpp[97]; +extern String8 txt_keywords__odin[40]; +extern String8 txt_keywords__jai[39]; +extern String8 txt_keywords__zig[49]; +extern String8 txt_keywords__rust[43]; + +C_LINKAGE_END + +#endif // TEXT_META_H diff --git a/src/text/text.c b/src/text/text.c index 3d39beb9..9ebe6c6b 100644 --- a/src/text/text.c +++ b/src/text/text.c @@ -4,6 +4,11 @@ #undef LAYER_COLOR #define LAYER_COLOR 0xe34cd4ff +//////////////////////////////// +//~ rjf: Generated Code + +#include "generated/text.meta.c" + //////////////////////////////// //~ rjf: Basic Helpers @@ -11,45 +16,13 @@ internal TXT_LangKind txt_lang_kind_from_extension(String8 extension) { TXT_LangKind kind = TXT_LangKind_Null; - if(str8_match(extension, str8_lit("c"), 0) || - str8_match(extension, str8_lit("h"), 0)) + for EachElement(idx, txt_ext_lang_kind_table) { - kind = TXT_LangKind_C; - } - else if(str8_match(extension, str8_lit("cpp"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("cxx"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("cc"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("c++"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("ixx"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("cxxm"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("c++m"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("ccm"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("cppm"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("mpp"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("C"), 0) || - str8_match(extension, str8_lit("hpp"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("hxx"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("hh"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("h++"), StringMatchFlag_CaseInsensitive) || - str8_match(extension, str8_lit("H"), 0)) - { - kind = TXT_LangKind_CPlusPlus; - } - else if(str8_match(extension, str8_lit("odin"), StringMatchFlag_CaseInsensitive)) - { - kind = TXT_LangKind_Odin; - } - else if(str8_match(extension, str8_lit("jai"), StringMatchFlag_CaseInsensitive)) - { - kind = TXT_LangKind_Jai; - } - else if(str8_match(extension, str8_lit("zig"), StringMatchFlag_CaseInsensitive)) - { - kind = TXT_LangKind_Zig; - } - else if(str8_match(extension, str8_lit("rs"), StringMatchFlag_CaseInsensitive)) - { - kind = TXT_LangKind_Rust; + if(str8_match(extension, txt_ext_lang_kind_table[idx].ext, 0)) + { + kind = txt_ext_lang_kind_table[idx].kind; + break; + } } return kind; } @@ -57,20 +30,7 @@ txt_lang_kind_from_extension(String8 extension) internal String8 txt_extension_from_lang_kind(TXT_LangKind kind) { - String8 result = {0}; - switch(kind) - { - case TXT_LangKind_Null: - case TXT_LangKind_COUNT: - case TXT_LangKind_DisasmX64Intel: - {}break; - case TXT_LangKind_C: {result = str8_lit("c");}break; - case TXT_LangKind_CPlusPlus: {result = str8_lit("cpp");}break; - case TXT_LangKind_Odin: {result = str8_lit("odin");}break; - case TXT_LangKind_Jai: {result = str8_lit("jai");}break; - case TXT_LangKind_Zig: {result = str8_lit("zig");}break; - case TXT_LangKind_Rust: {result = str8_lit("rs");}break; - } + String8 result = txt_lang_kind_ext_table[kind]; return result; } @@ -166,6 +126,343 @@ txt_token_array_from_list(Arena *arena, TXT_TokenList *list) //////////////////////////////// //~ rjf: Lexing Functions +internal TXT_TokenArray +txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, String8 string) +{ + ProfBeginFunction(); + Temp scratch = scratch_begin(&arena, 1); + U64 chunk_size = Clamp(8, string.size/8, 4096); + + //- rjf: gather keywords + U64 keyword_slots_count = txt_keywords_from_lang_kind_table[lang_kind].count; + String8Node **keyword_slots = push_array(scratch.arena, String8Node *, keyword_slots_count); + for EachIndex(idx, txt_keywords_from_lang_kind_table[lang_kind].count) + { + String8 keyword = txt_keywords_from_lang_kind_table[lang_kind].v[idx]; + U64 hash = u64_hash_from_str8(keyword); + U64 slot_idx = hash%keyword_slots_count; + String8Node *n = push_array(scratch.arena, String8Node, 1); + SLLStackPush(keyword_slots[slot_idx], n); + n->string = keyword; + } + + //- rjf: gather multichar symbols + String8Array multichar_symbols = txt_multichar_symbols_from_lang_kind_table[lang_kind]; + + //- rjf: gather rules + U64 tokenizer_rule_slots_count = txt_tokenizer_rules_from_lang_kind_table[lang_kind].count; + TXT_TokenizerRulePtrNode **tokenizer_rule_opener_slots = push_array(scratch.arena, TXT_TokenizerRulePtrNode *, tokenizer_rule_slots_count); + TXT_TokenizerRulePtrNode **tokenizer_rule_closer_slots = push_array(scratch.arena, TXT_TokenizerRulePtrNode *, tokenizer_rule_slots_count); + { + TXT_TokenizerRuleArray rules = txt_tokenizer_rules_from_lang_kind_table[lang_kind]; + for EachIndex(idx, rules.count) + { + TXT_TokenizerRule *r = &rules.v[idx]; + U64 open_hash = u64_hash_from_str8(r->open_string); + U64 close_hash = u64_hash_from_str8(r->close_string); + U64 open_slot_idx = open_hash%tokenizer_rule_slots_count; + U64 close_slot_idx = close_hash%tokenizer_rule_slots_count; + TXT_TokenizerRulePtrNode *open_n = push_array(scratch.arena, TXT_TokenizerRulePtrNode, 1); + TXT_TokenizerRulePtrNode *close_n = push_array(scratch.arena, TXT_TokenizerRulePtrNode, 1); + open_n->v = r; + close_n->v = r; + SLLStackPush(tokenizer_rule_opener_slots[open_slot_idx], open_n); + SLLStackPush(tokenizer_rule_closer_slots[close_slot_idx], close_n); + } + } + + //- rjf: generate token list + TXT_TokenChunkList tokens = {0}; + { + typedef struct TokenTask TokenTask; + struct TokenTask + { + TokenTask *next; + TXT_TokenizerRule *rule; + TXT_TokenKind kind; + U64 start_idx; + }; + TXT_TokenizerRule *nil_rule = &txt_tokenizer_rules__null[0]; + TokenTask *top_task = 0; + TokenTask *free_task = 0; + B32 escaped = 0; + B32 next_escaped = 0; + U64 byte_process_start_idx = 0; + for(U64 idx = 0; idx <= string.size;) + { + U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0; + U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0; + + // rjf: unpack next bytes + U8 next_bytes[] = {byte, next_byte}; + String8 string_1byte = str8(next_bytes, 1); + String8 string_2byte = str8(next_bytes, 2); + + // rjf: update counter + if((idx-byte_process_start_idx) >= 1000 || idx == string.size) + { + add_progress(idx - byte_process_start_idx); + byte_process_start_idx = idx; + } + + // rjf: escaping + if(escaped && (byte != '\r' && byte != '\n')) + { + next_escaped = 0; + } + else if(!escaped && byte == '\\') + { + next_escaped = 1; + } + + // rjf: take starter, push new token tasks + { + TXT_TokenizerRule *new_rule = nil_rule; + TXT_TokenKind new_token_kind = TXT_TokenKind_Null; + + // rjf: use next bytes to look up a rule from the table + if(top_task == 0) + { + TXT_TokenizerRule *active_rule = top_task ? top_task->rule : nil_rule; + U64 hash_1byte = u64_hash_from_str8(string_1byte); + U64 hash_2byte = u64_hash_from_str8(string_2byte); + U64 slot_1byte = hash_1byte%tokenizer_rule_slots_count; + U64 slot_2byte = hash_2byte%tokenizer_rule_slots_count; + if(new_rule == nil_rule) + { + for EachNode(n, TXT_TokenizerRulePtrNode, tokenizer_rule_opener_slots[slot_2byte]) + { + TXT_TokenizerRule *n_parent_rule = nil_rule; + if(0 < n->v->parent_num && n->v->parent_num <= txt_tokenizer_rules_from_lang_kind_table[lang_kind].count) + { + n_parent_rule = &txt_tokenizer_rules_from_lang_kind_table[lang_kind].v[n->v->parent_num-1]; + } + if(str8_match(n->v->open_string, string_2byte, 0) && + active_rule == n_parent_rule) + { + new_rule = n->v; + break; + } + } + } + if(new_rule == nil_rule) + { + for EachNode(n, TXT_TokenizerRulePtrNode, tokenizer_rule_opener_slots[slot_1byte]) + { + TXT_TokenizerRule *n_parent_rule = nil_rule; + if(0 < n->v->parent_num && n->v->parent_num <= txt_tokenizer_rules_from_lang_kind_table[lang_kind].count) + { + n_parent_rule = &txt_tokenizer_rules_from_lang_kind_table[lang_kind].v[n->v->parent_num-1]; + } + if(str8_match(n->v->open_string, string_1byte, 0) && + active_rule == n_parent_rule) + { + new_rule = n->v; + break; + } + } + } + if(new_rule != nil_rule) + { + new_token_kind = new_rule->token_kind; + } + } + + // rjf: use next bytes to start a new token based on fallback rules + if(top_task == 0 && new_rule == nil_rule) + { + if(0){} + else if(char_is_space(byte)) { new_token_kind = TXT_TokenKind_Whitespace; } + else if(byte == '_' || + byte == '$' || + char_is_alpha(byte)) { new_token_kind = TXT_TokenKind_Identifier; } + else if(char_is_digit(byte, 10) || + (byte == '.' && + char_is_digit(next_byte, 10))) { new_token_kind = TXT_TokenKind_Numeric; } + else if(byte == '~' || byte == '!' || + byte == '%' || byte == '^' || + byte == '&' || byte == '*' || + byte == '(' || byte == ')' || + byte == '-' || byte == '=' || + byte == '+' || byte == '[' || + byte == ']' || byte == '{' || + byte == '}' || byte == ':' || + byte == ';' || byte == ',' || + byte == '.' || byte == '<' || + byte == '>' || byte == '/' || + byte == '?' || byte == '|') { new_token_kind = TXT_TokenKind_Symbol; } + } + + // rjf: start new token + if(new_token_kind != TXT_TokenKind_Null) + { + TokenTask *task = free_task; + if(task != 0) + { + SLLStackPop(free_task); + } + else + { + task = push_array(scratch.arena, TokenTask, 1); + } + SLLStackPush(top_task, task); + top_task->rule = new_rule; + top_task->kind = new_token_kind; + top_task->start_idx = idx; + } + + // rjf: invalid token kind -> emit error + else if(top_task == 0) + { + TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)}; + txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token); + } + } + + // rjf: look for ender based on rule's closing symbol + U64 ender_pad = 0; + B32 ender_found = 0; + B32 task_pop = 0; + if(top_task != 0 && idx > top_task->start_idx) + { + TXT_TokenKind active_token_kind = top_task->kind; + TXT_TokenizerRule *active_rule = top_task->rule; + U64 active_token_start_idx = top_task->start_idx; + if(idx == string.size) + { + ender_pad = 0; + ender_found = 1; + } + else if(active_rule != nil_rule && + str8_match(string_1byte, active_rule->close_string, 0) || + str8_match(string_2byte, active_rule->close_string, 0)) + { + ender_found = 1; + ender_pad = active_rule->close_advance; + } + else switch(active_token_kind) + { + default:{}break; + case TXT_TokenKind_Whitespace: + { + ender_found = !char_is_space(byte); + }break; + case TXT_TokenKind_Identifier: + { + ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128); + }break; + case TXT_TokenKind_Numeric: + { + ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '.' && byte != '\''); + }break; + case TXT_TokenKind_Symbol: + { + ender_found = (byte != '~' && byte != '!' && + byte != '%' && byte != '^' && + byte != '&' && byte != '*' && + byte != '(' && byte != ')' && + byte != '-' && byte != '=' && + byte != '+' && byte != '[' && + byte != ']' && byte != '{' && + byte != '}' && byte != ':' && + byte != ';' && byte != ',' && + byte != '.' && byte != '<' && + byte != '>' && byte != '/' && + byte != '?' && byte != '|'); + }break; + } + } + + // rjf: next byte is ender => emit token + if(ender_found) + { + TXT_Token token = {top_task->kind, r1u64(top_task->start_idx, idx+ender_pad)}; + TokenTask *popped = top_task; + SLLStackPop(top_task); + SLLStackPush(free_task, popped); + + // rjf: trim \r's off of end + { + String8 token_string = str8_substr(string, token.range); + if(token_string.size != 0 && token_string.str[token_string.size-1] == '\r') + { + token.range.max -= 1; + ender_pad += 1; + } + } + + // rjf: identifier -> keyword in special cases + if(token.kind == TXT_TokenKind_Identifier) + { + String8 token_string = str8_substr(string, token.range); + U64 hash = u64_hash_from_str8(token_string); + U64 slot_idx = hash%keyword_slots_count; + for EachNode(n, String8Node, keyword_slots[slot_idx]) + { + if(str8_match(token_string, n->string, 0)) + { + token.kind = TXT_TokenKind_Keyword; + break; + } + } + txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token); + } + + // rjf: split symbols by maximum-munch-rule + else if(token.kind == TXT_TokenKind_Symbol) + { + String8 token_string = str8_substr(string, token.range); + for(U64 off = 0, next_off = token_string.size; off < token_string.size; off = next_off) + { + B32 found = 0; + for(U64 idx = 0; idx < multichar_symbols.count; idx += 1) + { + if(str8_match(str8_substr(token_string, r1u64(off, off+multichar_symbols.v[idx].size)), + multichar_symbols.v[idx], + 0)) + { + found = 1; + next_off = off + Max(1, multichar_symbols.v[idx].size); + TXT_Token sub_token = {TXT_TokenKind_Symbol, r1u64(token.range.min+off, token.range.min+next_off)}; + txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &sub_token); + break; + } + } + if(!found) + { + next_off = off+1; + TXT_Token sub_token = {TXT_TokenKind_Symbol, r1u64(token.range.min+off, token.range.min+next_off)}; + txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &sub_token); + } + } + } + + // rjf: all other tokens + else + { + txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token); + } + + // rjf: increment by ender padding + idx += ender_pad; + } + + // rjf: advance by 1 byte if we haven't found an ender + if(!ender_found) + { + idx += 1; + } + escaped = next_escaped; + } + } + + //- rjf: token list -> token array + TXT_TokenArray result = txt_token_array_from_chunk_list(arena, &tokens); + scratch_end(scratch); + ProfEnd(); + return result; +} + internal TXT_TokenArray txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string) { @@ -1379,7 +1676,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St { // NOTE(spey): Rust supports unicode identifiers. They are not handled in any way here, // but it might be worth looking into in the future. - + Temp scratch = scratch_begin(&arena, 1); //- rjf: generate token list @@ -1388,7 +1685,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St S32 multiline_comment_nesting_level = 0; S32 raw_string_nesting_level = 0; S32 raw_string_ender_nesting_level = 0; - + // NOTE(spey): Rust's syntax is designed in such a way that we can't be sure what a token // is immediately from the first character, so we have to keep track of some possibilities. B32 token_may_be_char = 0; @@ -1404,14 +1701,14 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St { U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0; U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0; - + // rjf: update counter if(bytes_processed_counter != 0 && ((idx-byte_process_start_idx) >= 1000 || idx == string.size)) { ins_atomic_u64_add_eval(bytes_processed_counter, (idx-byte_process_start_idx)); byte_process_start_idx = idx; } - + // rjf: escaping if(escaped && (byte != '\r' && byte != '\n')) { @@ -1421,10 +1718,10 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St { next_escaped = 1; } - + // rjf: take starter, determine active token kind U64 starter_pad = 0; - + // spey: special case of starter for nested comments if(active_token_kind == TXT_TokenKind_Comment) { @@ -1451,9 +1748,9 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St char_is_digit(next_byte, 10))) { active_token_kind = TXT_TokenKind_Numeric; } else if(byte == '"') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; } else if((byte == 'c' || byte == 'b') && - next_byte == '"') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; starter_pad = 1; } + next_byte == '"') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; starter_pad = 1; } else if((byte == 'c' || byte == 'b') && - next_byte == 'r') { active_token_kind = TXT_TokenKind_Identifier; token_may_be_string = 1; } + next_byte == 'r') { active_token_kind = TXT_TokenKind_Identifier; token_may_be_string = 1; } else if(byte == '_' || char_is_alpha(byte)) { active_token_kind = TXT_TokenKind_Identifier; } else if(byte == '/' && next_byte == '/') { active_token_kind = TXT_TokenKind_Comment; starter_pad = 1; } @@ -1472,8 +1769,8 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St byte == '?' || byte == '|') { active_token_kind = TXT_TokenKind_Symbol; } else if(byte == '\'') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; token_may_be_lifetime = 1; } else if((byte == 'c' || byte == 'b') && - next_byte == '\'') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; starter_pad = 1; } - + next_byte == '\'') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; starter_pad = 1; } + // rjf: start new token if(active_token_kind != TXT_TokenKind_Null) { @@ -1487,14 +1784,14 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token); } } - + B32 is_on_starter = idx <= active_token_start_idx || token_may_be_string; - + // spey: advance by starter padding byte(s) and reset byte/next_byte values idx += starter_pad; byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0; next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0; - + // rjf: look for ender U64 ender_pad = 0; B32 ender_found = 0; @@ -1540,10 +1837,10 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St else { if(0){} - + // spey: regular string else if(raw_string_nesting_level == 0) { ender_found = byte == '"'; } - + // spey: raw string else if(byte == '"' && next_byte == '#' && raw_string_ender_nesting_level == 0) { raw_string_ender_nesting_level++; } @@ -1556,7 +1853,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St raw_string_ender_nesting_level >= 0) { raw_string_ender_nesting_level++; } } } - + ender_pad += 1; }break; case TXT_TokenKind_Symbol: @@ -1584,7 +1881,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St { if (byte == '*' && next_byte == '/') multiline_comment_nesting_level--; - + ender_found = (active_token_start_idx+1 < idx && multiline_comment_nesting_level == 0); ender_pad += 2; } @@ -1642,7 +1939,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St str8_lit_comp("async"), str8_lit_comp("await"), str8_lit_comp("dyn"), - + // weak keywords str8_lit_comp("macro_rules"), str8_lit_comp("raw"), @@ -1712,17 +2009,17 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St } } } - + // rjf: all other tokens else { txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token); } - + // rjf: increment by starter and ender padding idx += ender_pad; } - + // rjf: advance by 1 byte if we haven't found an ender if(!ender_found) { @@ -2459,10 +2756,17 @@ txt_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *gen_ou set_progress(Min(data.size, 1024) + data.size + data.size); //- rjf: lex function * data -> tokens +#if 1 if(lane_idx() == 0 && lex_function != 0) { shared->info.tokens = lex_function(shared->arena, 0, data); } +#else + if(lane_idx() == 0) + { + shared->info.tokens = txt_token_array_from_lang_kind_string(shared->arena, lang, data); + } +#endif lane_sync(); set_progress(Min(data.size, 1024) + data.size + data.size + data.size*(lex_function != 0)); TXT_TokenArray tokens = shared->info.tokens; diff --git a/src/text/text.h b/src/text/text.h index 5978e43d..23293f3c 100644 --- a/src/text/text.h +++ b/src/text/text.h @@ -32,6 +32,32 @@ typedef enum TXT_TokenKind } TXT_TokenKind; +typedef struct TXT_TokenizerRule TXT_TokenizerRule; +struct TXT_TokenizerRule +{ + TXT_TokenKind token_kind; + String8 open_string; + String8 close_string; + U32 close_advance; + B32 nesting; + B32 escaping; + U32 parent_num; +}; + +typedef struct TXT_TokenizerRulePtrNode TXT_TokenizerRulePtrNode; +struct TXT_TokenizerRulePtrNode +{ + TXT_TokenizerRulePtrNode *next; + TXT_TokenizerRule *v; +}; + +typedef struct TXT_TokenizerRuleArray TXT_TokenizerRuleArray; +struct TXT_TokenizerRuleArray +{ + TXT_TokenizerRule *v; + U64 count; +}; + typedef struct TXT_Token TXT_Token; struct TXT_Token { @@ -138,21 +164,12 @@ struct TXT_LineTokensSlice }; //////////////////////////////// -//~ rjf: Language Kind Types +//~ rjf: Generated Code -typedef enum TXT_LangKind -{ - TXT_LangKind_Null, - TXT_LangKind_C, - TXT_LangKind_CPlusPlus, - TXT_LangKind_Odin, - TXT_LangKind_Jai, - TXT_LangKind_Zig, - TXT_LangKind_Rust, - TXT_LangKind_DisasmX64Intel, - TXT_LangKind_COUNT -} -TXT_LangKind; +#include "generated/text.meta.h" + +//////////////////////////////// +//~ rjf: Language Kind Types typedef TXT_TokenArray TXT_LangLexFunctionType(Arena *arena, U64 *bytes_processed_counter, String8 string); @@ -180,6 +197,8 @@ internal TXT_TokenArray txt_token_array_from_list(Arena *arena, TXT_TokenList *l //////////////////////////////// //~ rjf: Lexing Functions +internal TXT_TokenArray txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, String8 string); + internal TXT_TokenArray txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string); internal TXT_TokenArray txt_token_array_from_string__odin(Arena *arena, U64 *bytes_processed_counter, String8 string); internal TXT_TokenArray txt_token_array_from_string__jai(Arena *arena, U64 *bytes_processed_counter, String8 string); diff --git a/src/text/text.mdesk b/src/text/text.mdesk new file mode 100644 index 00000000..6b904df3 --- /dev/null +++ b/src/text/text.mdesk @@ -0,0 +1,246 @@ +// Copyright (c) Epic Games Tools +// Licensed under the MIT license (https://opensource.org/license/mit/) + +@table(name ext keywords_table multichar_symbol_table tokenizer_rules_table) +TXT_LangKindTable: +{ + { Null null null null null} + { C c c c c } + { CPlusPlus cpp cpp c c } + { Odin odin odin c null} + { Jai jai jai c null} + { Zig zig zig c null} + { Rust rs rust c null} + { DisasmX64Intel disasm_x64_intel null null null} +} + +@table(string kind) +TXT_LangKindExtTable: +{ + {"h" C } + {"cxx" CPlusPlus} + {"cc" CPlusPlus} + {"c++" CPlusPlus} + {"ixx" CPlusPlus} + {"cxxm" CPlusPlus} + {"c++m" CPlusPlus} + {"ccm" CPlusPlus} + {"cppm" CPlusPlus} + {"mpp" CPlusPlus} + {"C" CPlusPlus} + {"hpp" CPlusPlus} + {"hxx" CPlusPlus} + {"hh" CPlusPlus} + {"h++" CPlusPlus} + {"H" CPlusPlus} +} + +@enum TXT_LangKind: +{ + @expand(TXT_LangKindTable a) `$(a.name)`, + COUNT +} + +@data(String8) +txt_lang_kind_ext_table: +{ + @expand(TXT_LangKindTable a) `str8_lit_comp("$(a.ext)")` +} + +@data(`struct {String8 ext; TXT_LangKind kind;}`) @c_file +txt_ext_lang_kind_table: +{ + @expand(TXT_LangKindTable a) `{str8_lit_comp("$(a.ext)"), TXT_LangKind_$(a.name)}`, + @expand(TXT_LangKindExtTable a) `{str8_lit_comp("$(a.string)"), TXT_LangKind_$(a.kind)}`, +} + +@data(String8Array) txt_keywords_from_lang_kind_table: +{ + @expand(TXT_LangKindTable a) `{txt_keywords__$(a.keywords_table), ArrayCount(txt_keywords__$(a.keywords_table))}` +} + +@data(String8Array) txt_multichar_symbols_from_lang_kind_table: +{ + @expand(TXT_LangKindTable a) `{txt_multichar_symbols__$(a.multichar_symbol_table), ArrayCount(txt_multichar_symbols__$(a.multichar_symbol_table))}` +} + +@data(TXT_TokenizerRuleArray) txt_tokenizer_rules_from_lang_kind_table: +{ + @expand(TXT_LangKindTable a) `{txt_tokenizer_rules__$(a.tokenizer_rules_table), ArrayCount(txt_tokenizer_rules__$(a.tokenizer_rules_table))}` +} + +@data(String8) txt_keywords__null:{`{0}`} +@data(String8) txt_multichar_symbols__null:{`{0}`} +@data(TXT_TokenizerRule) txt_tokenizer_rules__null:{`{TXT_TokenKind_Null}`} + +//////////////////////////////// +//~ rjf: C + +@table(string) +txt_keywords__c: +{ + {"auto"} {"break"} {"case"} {"char"} + {"const"} {"continue"} {"default"} {"do"} + {"double"} {"else"} {"enum"} {"extern"} + {"float"} {"for"} {"goto"} {"if"} + {"int"} {"long"} {"register"} {"return"} + {"short"} {"signed"} {"sizeof"} {"static"} + {"struct"} {"switch"} {"typedef"} {"union"} + {"unsigned"} {"void"} {"volatile"} {"while"} +} +@data(String8) txt_keywords__c: {@expand(txt_keywords__c a) `str8_lit_comp("$(a.string)")`} + +@table(string) +txt_multichar_symbols__c: +{ + {"<<"} + {">>"} + {"<="} + {">="} + {"=="} + {"!="} + {"&&"} + {"||"} + {"|="} + {"&="} + {"^="} + {"~="} + {"+="} + {"-="} + {"*="} + {"/="} + {"%="} + {"<<="} + {">>="} + {"->"} +} +@data(String8) txt_multichar_symbols__c: {@expand(txt_multichar_symbols__c a) `str8_lit_comp("$(a.string)")`} + +@table(token_kind open_string close_string close_advance nesting escaping parent_num) txt_tokenizer_rules__c: +{ + {Comment "//" "\\n" 0 0 1 0} + {Comment "/*" "*/" 2 0 0 0} + {Meta "#" "\\n" 0 0 1 0} + {String '\\"' '\\"' 1 0 1 0} + {String "'" "'" 1 0 1 0} + {String '\\"' '\\"' 1 0 1 3} + {String "<" ">" 1 0 1 3} +} +@data(TXT_TokenizerRule) txt_tokenizer_rules__c: +{ + @expand(txt_tokenizer_rules__c a) `{TXT_TokenKind_$(a.token_kind), str8_lit_comp("$(a.open_string)"), str8_lit_comp("$(a.close_string)"), $(a.close_advance), $(a.nesting), $(a.escaping), $(a.parent_num)}` +} + +//////////////////////////////// +//~ rjf: C++ + +@table(string) +txt_keywords__cpp: +{ + {"alignas"} {"alignof"} {"and"} {"and_eq"} + {"asm"} {"atomic_cancel"} {"atomic_commit"} {"atomic_noexcept"} + {"auto"} {"bitand"} {"bitor"} {"bool"} + {"break"} {"case"} {"catch"} {"char"} + {"char8_t"} {"char16_t"} {"char32_t"} {"class"} + {"compl"} {"concept"} {"const"} {"consteval"} + {"constexpr"} {"constinit"} {"const_cast"} {"continue"} + {"co_await"} {"co_return"} {"co_yield"} {"decltype"} + {"default"} {"delete"} {"do"} {"double"} + {"dynamic_cast"} {"else"} {"enum"} {"explicit"} + {"export"} {"extern"} {"false"} {"float"} + {"for"} {"friend"} {"goto"} {"if"} + {"inline"} {"int"} {"long"} {"mutable"} + {"namespace"} {"new"} {"noexcept"} {"not"} + {"not_eq"} {"nullptr"} {"operator"} {"or"} + {"or_eq"} {"private"} {"protected"} {"public"} + {"reflexpr"} {"register"} {"reinterpret_cast"} {"requires"} + {"return"} {"short"} {"signed"} {"sizeof"} + {"static"} {"static_assert"} {"static_cast"} {"struct"} + {"switch"} {"synchronized"} {"template"} {"this"} + {"thread_local"} {"throw"} {"true"} {"try"} + {"typedef"} {"typeid"} {"typename"} {"union"} + {"unsigned"} {"using"} {"virtual"} {"void"} + {"volatile"} {"wchar_t"} {"while"} {"xor"} + {"xor_eq"} +} +@data(String8) txt_keywords__cpp: {@expand(txt_keywords__cpp a) `str8_lit_comp("$(a.string)")`} + +//////////////////////////////// +//~ rjf: Odin + +@table(string) +txt_keywords__odin: +{ + {"align_of"} {"asm"} {"auto_cast"} {"bit_set"} + {"break"} {"case"} {"cast"} {"context"} + {"continue"} {"defer"} {"distinct"} {"do"} + {"dynamic"} {"else"} {"enum"} {"fallthrough"} + {"for"} {"foreign"} {"if"} {"in"} + {"map"} {"matrix"} {"not_in"} {"or_break"} + {"or_continue"} {"or_else"} {"or_return"} {"package"} + {"proc"} {"return"} {"size_of"} {"struct"} + {"switch"} {"transmute"} {"typeid"} {"union"} + {"using"} {"when"} {"where"} {"import"} +} +@data(String8) txt_keywords__odin: {@expand(txt_keywords__odin a) `str8_lit_comp("$(a.string)")`} + +//////////////////////////////// +//~ rjf: Jai + +@table(string) +txt_keywords__jai: +{ + {"bool"} {"true"} {"false"} {"int"} + {"s8"} {"u8"} {"s16"} {"u16"} + {"s32"} {"u32"} {"s64"} {"u64"} + {"s128"} {"u128"} {"float"} {"float32"} + {"float64"} {"void"} {"enum"} {"enum_flags"} + {"size_of"} {"string"} {"type_of"} {"cast"} + {"if"} {"ifs"} {"then"} {"else"} + {"case"} {"for"} {"while"} {"break"} + {"continue"} {"remove"} {"return"} {"inline"} + {"null"} {"defer"} {"xx"} +} +@data(String8) txt_keywords__jai: {@expand(txt_keywords__jai a) `str8_lit_comp("$(a.string)")`} + +//////////////////////////////// +//~ rjf: Zig + +@table(string) +txt_keywords__zig: +{ + {"addrspace"} {"align"} {"allowzero"} {"and"} + {"anyframe"} {"anytype"} {"asm"} {"async"} + {"await"} {"break"} {"callconv"} {"catch"} + {"comptime"} {"const"} {"continue"} {"defer"} + {"else"} {"enum"} {"errdefer"} {"error"} + {"export"} {"extern"} {"fn"} {"for"} + {"if"} {"inline"} {"noalias"} {"nosuspend"} + {"noinline"} {"opaque"} {"or"} {"orelse"} + {"packed"} {"pub"} {"resume"} {"return"} + {"linksection"} {"struct"} {"suspend"} {"switch"} + {"test"} {"threadlocal"} {"try"} {"union"} + {"unreachable"} {"usingnamespace"} {"var"} {"volatile"} + {"while"} +} +@data(String8) txt_keywords__zig: {@expand(txt_keywords__zig a) `str8_lit_comp("$(a.string)")`} + +//////////////////////////////// +//~ rjf: Rust + +@table(string) +txt_keywords__rust: +{ + {"as"} {"break"} {"const"} {"continue"} + {"crate"} {"else"} {"enum"} {"extern"} + {"false"} {"fn"} {"for"} {"if"} + {"impl"} {"in"} {"let"} {"loop"} + {"match"} {"mod"} {"move"} {"mut"} + {"pub"} {"ref"} {"return"} {"self"} + {"Self"} {"static"} {"struct"} {"super"} + {"trait"} {"true"} {"type"} {"unsafe"} + {"use"} {"where"} {"while"} {"yield"} + {"async"} {"await"} {"dyn"} {"macro_rules"} + {"raw"} {"safe"} {"union"} +} +@data(String8) txt_keywords__rust: {@expand(txt_keywords__rust a) `str8_lit_comp("$(a.string)")`}