mirror of
https://github.com/Ed94/raddebugger.git
synced 2026-08-14 17:28:07 +00:00
first pass at unified lexer path, not plugged in yet
This commit is contained in:
Ryan Fleury
@@ -0,0 +1,458 @@
|
|||||||
|
// Copyright (c) Epic Games Tools
|
||||||
|
// Licensed under the MIT license (https://opensource.org/license/mit/)
|
||||||
|
|
||||||
|
//- GENERATED CODE
|
||||||
|
|
||||||
|
C_LINKAGE_BEGIN
|
||||||
|
String8 txt_lang_kind_ext_table[8] =
|
||||||
|
{
|
||||||
|
str8_lit_comp("null"),
|
||||||
|
str8_lit_comp("c"),
|
||||||
|
str8_lit_comp("cpp"),
|
||||||
|
str8_lit_comp("odin"),
|
||||||
|
str8_lit_comp("jai"),
|
||||||
|
str8_lit_comp("zig"),
|
||||||
|
str8_lit_comp("rs"),
|
||||||
|
str8_lit_comp("disasm_x64_intel"),
|
||||||
|
};
|
||||||
|
|
||||||
|
struct {String8 ext; TXT_LangKind kind;} txt_ext_lang_kind_table[24] =
|
||||||
|
{
|
||||||
|
{str8_lit_comp("null"), TXT_LangKind_Null},
|
||||||
|
{str8_lit_comp("c"), TXT_LangKind_C},
|
||||||
|
{str8_lit_comp("cpp"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("odin"), TXT_LangKind_Odin},
|
||||||
|
{str8_lit_comp("jai"), TXT_LangKind_Jai},
|
||||||
|
{str8_lit_comp("zig"), TXT_LangKind_Zig},
|
||||||
|
{str8_lit_comp("rs"), TXT_LangKind_Rust},
|
||||||
|
{str8_lit_comp("disasm_x64_intel"), TXT_LangKind_DisasmX64Intel},
|
||||||
|
{str8_lit_comp("h"), TXT_LangKind_C},
|
||||||
|
{str8_lit_comp("cxx"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("cc"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("c++"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("ixx"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("cxxm"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("c++m"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("ccm"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("cppm"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("mpp"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("C"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("hpp"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("hxx"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("hh"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("h++"), TXT_LangKind_CPlusPlus},
|
||||||
|
{str8_lit_comp("H"), TXT_LangKind_CPlusPlus},
|
||||||
|
};
|
||||||
|
|
||||||
|
String8Array txt_keywords_from_lang_kind_table[8] =
|
||||||
|
{
|
||||||
|
{txt_keywords__null, ArrayCount(txt_keywords__null)},
|
||||||
|
{txt_keywords__c, ArrayCount(txt_keywords__c)},
|
||||||
|
{txt_keywords__cpp, ArrayCount(txt_keywords__cpp)},
|
||||||
|
{txt_keywords__odin, ArrayCount(txt_keywords__odin)},
|
||||||
|
{txt_keywords__jai, ArrayCount(txt_keywords__jai)},
|
||||||
|
{txt_keywords__zig, ArrayCount(txt_keywords__zig)},
|
||||||
|
{txt_keywords__rust, ArrayCount(txt_keywords__rust)},
|
||||||
|
{txt_keywords__null, ArrayCount(txt_keywords__null)},
|
||||||
|
};
|
||||||
|
|
||||||
|
String8Array txt_multichar_symbols_from_lang_kind_table[8] =
|
||||||
|
{
|
||||||
|
{txt_multichar_symbols__null, ArrayCount(txt_multichar_symbols__null)},
|
||||||
|
{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
|
||||||
|
{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
|
||||||
|
{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
|
||||||
|
{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
|
||||||
|
{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
|
||||||
|
{txt_multichar_symbols__c, ArrayCount(txt_multichar_symbols__c)},
|
||||||
|
{txt_multichar_symbols__null, ArrayCount(txt_multichar_symbols__null)},
|
||||||
|
};
|
||||||
|
|
||||||
|
TXT_TokenizerRuleArray txt_tokenizer_rules_from_lang_kind_table[8] =
|
||||||
|
{
|
||||||
|
{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
|
||||||
|
{txt_tokenizer_rules__c, ArrayCount(txt_tokenizer_rules__c)},
|
||||||
|
{txt_tokenizer_rules__c, ArrayCount(txt_tokenizer_rules__c)},
|
||||||
|
{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
|
||||||
|
{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
|
||||||
|
{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
|
||||||
|
{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
|
||||||
|
{txt_tokenizer_rules__null, ArrayCount(txt_tokenizer_rules__null)},
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_keywords__null[1] =
|
||||||
|
{
|
||||||
|
{0},
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_multichar_symbols__null[1] =
|
||||||
|
{
|
||||||
|
{0},
|
||||||
|
};
|
||||||
|
|
||||||
|
TXT_TokenizerRule txt_tokenizer_rules__null[1] =
|
||||||
|
{
|
||||||
|
{TXT_TokenKind_Null},
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_keywords__c[32] =
|
||||||
|
{
|
||||||
|
str8_lit_comp("auto"),
|
||||||
|
str8_lit_comp("break"),
|
||||||
|
str8_lit_comp("case"),
|
||||||
|
str8_lit_comp("char"),
|
||||||
|
str8_lit_comp("const"),
|
||||||
|
str8_lit_comp("continue"),
|
||||||
|
str8_lit_comp("default"),
|
||||||
|
str8_lit_comp("do"),
|
||||||
|
str8_lit_comp("double"),
|
||||||
|
str8_lit_comp("else"),
|
||||||
|
str8_lit_comp("enum"),
|
||||||
|
str8_lit_comp("extern"),
|
||||||
|
str8_lit_comp("float"),
|
||||||
|
str8_lit_comp("for"),
|
||||||
|
str8_lit_comp("goto"),
|
||||||
|
str8_lit_comp("if"),
|
||||||
|
str8_lit_comp("int"),
|
||||||
|
str8_lit_comp("long"),
|
||||||
|
str8_lit_comp("register"),
|
||||||
|
str8_lit_comp("return"),
|
||||||
|
str8_lit_comp("short"),
|
||||||
|
str8_lit_comp("signed"),
|
||||||
|
str8_lit_comp("sizeof"),
|
||||||
|
str8_lit_comp("static"),
|
||||||
|
str8_lit_comp("struct"),
|
||||||
|
str8_lit_comp("switch"),
|
||||||
|
str8_lit_comp("typedef"),
|
||||||
|
str8_lit_comp("union"),
|
||||||
|
str8_lit_comp("unsigned"),
|
||||||
|
str8_lit_comp("void"),
|
||||||
|
str8_lit_comp("volatile"),
|
||||||
|
str8_lit_comp("while"),
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_multichar_symbols__c[20] =
|
||||||
|
{
|
||||||
|
str8_lit_comp("<<"),
|
||||||
|
str8_lit_comp(">>"),
|
||||||
|
str8_lit_comp("<="),
|
||||||
|
str8_lit_comp(">="),
|
||||||
|
str8_lit_comp("=="),
|
||||||
|
str8_lit_comp("!="),
|
||||||
|
str8_lit_comp("&&"),
|
||||||
|
str8_lit_comp("||"),
|
||||||
|
str8_lit_comp("|="),
|
||||||
|
str8_lit_comp("&="),
|
||||||
|
str8_lit_comp("^="),
|
||||||
|
str8_lit_comp("~="),
|
||||||
|
str8_lit_comp("+="),
|
||||||
|
str8_lit_comp("-="),
|
||||||
|
str8_lit_comp("*="),
|
||||||
|
str8_lit_comp("/="),
|
||||||
|
str8_lit_comp("%="),
|
||||||
|
str8_lit_comp("<<="),
|
||||||
|
str8_lit_comp(">>="),
|
||||||
|
str8_lit_comp("->"),
|
||||||
|
};
|
||||||
|
|
||||||
|
TXT_TokenizerRule txt_tokenizer_rules__c[7] =
|
||||||
|
{
|
||||||
|
{TXT_TokenKind_Comment, str8_lit_comp("//"), str8_lit_comp("\n"), 0, 0, 1, 0},
|
||||||
|
{TXT_TokenKind_Comment, str8_lit_comp("/*"), str8_lit_comp("*/"), 2, 0, 0, 0},
|
||||||
|
{TXT_TokenKind_Meta, str8_lit_comp("#"), str8_lit_comp("\n"), 0, 0, 1, 0},
|
||||||
|
{TXT_TokenKind_String, str8_lit_comp("\""), str8_lit_comp("\""), 1, 0, 1, 0},
|
||||||
|
{TXT_TokenKind_String, str8_lit_comp("'"), str8_lit_comp("'"), 1, 0, 1, 0},
|
||||||
|
{TXT_TokenKind_String, str8_lit_comp("\""), str8_lit_comp("\""), 1, 0, 1, 3},
|
||||||
|
{TXT_TokenKind_String, str8_lit_comp("<"), str8_lit_comp(">"), 1, 0, 1, 3},
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_keywords__cpp[97] =
|
||||||
|
{
|
||||||
|
str8_lit_comp("alignas"),
|
||||||
|
str8_lit_comp("alignof"),
|
||||||
|
str8_lit_comp("and"),
|
||||||
|
str8_lit_comp("and_eq"),
|
||||||
|
str8_lit_comp("asm"),
|
||||||
|
str8_lit_comp("atomic_cancel"),
|
||||||
|
str8_lit_comp("atomic_commit"),
|
||||||
|
str8_lit_comp("atomic_noexcept"),
|
||||||
|
str8_lit_comp("auto"),
|
||||||
|
str8_lit_comp("bitand"),
|
||||||
|
str8_lit_comp("bitor"),
|
||||||
|
str8_lit_comp("bool"),
|
||||||
|
str8_lit_comp("break"),
|
||||||
|
str8_lit_comp("case"),
|
||||||
|
str8_lit_comp("catch"),
|
||||||
|
str8_lit_comp("char"),
|
||||||
|
str8_lit_comp("char8_t"),
|
||||||
|
str8_lit_comp("char16_t"),
|
||||||
|
str8_lit_comp("char32_t"),
|
||||||
|
str8_lit_comp("class"),
|
||||||
|
str8_lit_comp("compl"),
|
||||||
|
str8_lit_comp("concept"),
|
||||||
|
str8_lit_comp("const"),
|
||||||
|
str8_lit_comp("consteval"),
|
||||||
|
str8_lit_comp("constexpr"),
|
||||||
|
str8_lit_comp("constinit"),
|
||||||
|
str8_lit_comp("const_cast"),
|
||||||
|
str8_lit_comp("continue"),
|
||||||
|
str8_lit_comp("co_await"),
|
||||||
|
str8_lit_comp("co_return"),
|
||||||
|
str8_lit_comp("co_yield"),
|
||||||
|
str8_lit_comp("decltype"),
|
||||||
|
str8_lit_comp("default"),
|
||||||
|
str8_lit_comp("delete"),
|
||||||
|
str8_lit_comp("do"),
|
||||||
|
str8_lit_comp("double"),
|
||||||
|
str8_lit_comp("dynamic_cast"),
|
||||||
|
str8_lit_comp("else"),
|
||||||
|
str8_lit_comp("enum"),
|
||||||
|
str8_lit_comp("explicit"),
|
||||||
|
str8_lit_comp("export"),
|
||||||
|
str8_lit_comp("extern"),
|
||||||
|
str8_lit_comp("false"),
|
||||||
|
str8_lit_comp("float"),
|
||||||
|
str8_lit_comp("for"),
|
||||||
|
str8_lit_comp("friend"),
|
||||||
|
str8_lit_comp("goto"),
|
||||||
|
str8_lit_comp("if"),
|
||||||
|
str8_lit_comp("inline"),
|
||||||
|
str8_lit_comp("int"),
|
||||||
|
str8_lit_comp("long"),
|
||||||
|
str8_lit_comp("mutable"),
|
||||||
|
str8_lit_comp("namespace"),
|
||||||
|
str8_lit_comp("new"),
|
||||||
|
str8_lit_comp("noexcept"),
|
||||||
|
str8_lit_comp("not"),
|
||||||
|
str8_lit_comp("not_eq"),
|
||||||
|
str8_lit_comp("nullptr"),
|
||||||
|
str8_lit_comp("operator"),
|
||||||
|
str8_lit_comp("or"),
|
||||||
|
str8_lit_comp("or_eq"),
|
||||||
|
str8_lit_comp("private"),
|
||||||
|
str8_lit_comp("protected"),
|
||||||
|
str8_lit_comp("public"),
|
||||||
|
str8_lit_comp("reflexpr"),
|
||||||
|
str8_lit_comp("register"),
|
||||||
|
str8_lit_comp("reinterpret_cast"),
|
||||||
|
str8_lit_comp("requires"),
|
||||||
|
str8_lit_comp("return"),
|
||||||
|
str8_lit_comp("short"),
|
||||||
|
str8_lit_comp("signed"),
|
||||||
|
str8_lit_comp("sizeof"),
|
||||||
|
str8_lit_comp("static"),
|
||||||
|
str8_lit_comp("static_assert"),
|
||||||
|
str8_lit_comp("static_cast"),
|
||||||
|
str8_lit_comp("struct"),
|
||||||
|
str8_lit_comp("switch"),
|
||||||
|
str8_lit_comp("synchronized"),
|
||||||
|
str8_lit_comp("template"),
|
||||||
|
str8_lit_comp("this"),
|
||||||
|
str8_lit_comp("thread_local"),
|
||||||
|
str8_lit_comp("throw"),
|
||||||
|
str8_lit_comp("true"),
|
||||||
|
str8_lit_comp("try"),
|
||||||
|
str8_lit_comp("typedef"),
|
||||||
|
str8_lit_comp("typeid"),
|
||||||
|
str8_lit_comp("typename"),
|
||||||
|
str8_lit_comp("union"),
|
||||||
|
str8_lit_comp("unsigned"),
|
||||||
|
str8_lit_comp("using"),
|
||||||
|
str8_lit_comp("virtual"),
|
||||||
|
str8_lit_comp("void"),
|
||||||
|
str8_lit_comp("volatile"),
|
||||||
|
str8_lit_comp("wchar_t"),
|
||||||
|
str8_lit_comp("while"),
|
||||||
|
str8_lit_comp("xor"),
|
||||||
|
str8_lit_comp("xor_eq"),
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_keywords__odin[40] =
|
||||||
|
{
|
||||||
|
str8_lit_comp("align_of"),
|
||||||
|
str8_lit_comp("asm"),
|
||||||
|
str8_lit_comp("auto_cast"),
|
||||||
|
str8_lit_comp("bit_set"),
|
||||||
|
str8_lit_comp("break"),
|
||||||
|
str8_lit_comp("case"),
|
||||||
|
str8_lit_comp("cast"),
|
||||||
|
str8_lit_comp("context"),
|
||||||
|
str8_lit_comp("continue"),
|
||||||
|
str8_lit_comp("defer"),
|
||||||
|
str8_lit_comp("distinct"),
|
||||||
|
str8_lit_comp("do"),
|
||||||
|
str8_lit_comp("dynamic"),
|
||||||
|
str8_lit_comp("else"),
|
||||||
|
str8_lit_comp("enum"),
|
||||||
|
str8_lit_comp("fallthrough"),
|
||||||
|
str8_lit_comp("for"),
|
||||||
|
str8_lit_comp("foreign"),
|
||||||
|
str8_lit_comp("if"),
|
||||||
|
str8_lit_comp("in"),
|
||||||
|
str8_lit_comp("map"),
|
||||||
|
str8_lit_comp("matrix"),
|
||||||
|
str8_lit_comp("not_in"),
|
||||||
|
str8_lit_comp("or_break"),
|
||||||
|
str8_lit_comp("or_continue"),
|
||||||
|
str8_lit_comp("or_else"),
|
||||||
|
str8_lit_comp("or_return"),
|
||||||
|
str8_lit_comp("package"),
|
||||||
|
str8_lit_comp("proc"),
|
||||||
|
str8_lit_comp("return"),
|
||||||
|
str8_lit_comp("size_of"),
|
||||||
|
str8_lit_comp("struct"),
|
||||||
|
str8_lit_comp("switch"),
|
||||||
|
str8_lit_comp("transmute"),
|
||||||
|
str8_lit_comp("typeid"),
|
||||||
|
str8_lit_comp("union"),
|
||||||
|
str8_lit_comp("using"),
|
||||||
|
str8_lit_comp("when"),
|
||||||
|
str8_lit_comp("where"),
|
||||||
|
str8_lit_comp("import"),
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_keywords__jai[39] =
|
||||||
|
{
|
||||||
|
str8_lit_comp("bool"),
|
||||||
|
str8_lit_comp("true"),
|
||||||
|
str8_lit_comp("false"),
|
||||||
|
str8_lit_comp("int"),
|
||||||
|
str8_lit_comp("s8"),
|
||||||
|
str8_lit_comp("u8"),
|
||||||
|
str8_lit_comp("s16"),
|
||||||
|
str8_lit_comp("u16"),
|
||||||
|
str8_lit_comp("s32"),
|
||||||
|
str8_lit_comp("u32"),
|
||||||
|
str8_lit_comp("s64"),
|
||||||
|
str8_lit_comp("u64"),
|
||||||
|
str8_lit_comp("s128"),
|
||||||
|
str8_lit_comp("u128"),
|
||||||
|
str8_lit_comp("float"),
|
||||||
|
str8_lit_comp("float32"),
|
||||||
|
str8_lit_comp("float64"),
|
||||||
|
str8_lit_comp("void"),
|
||||||
|
str8_lit_comp("enum"),
|
||||||
|
str8_lit_comp("enum_flags"),
|
||||||
|
str8_lit_comp("size_of"),
|
||||||
|
str8_lit_comp("string"),
|
||||||
|
str8_lit_comp("type_of"),
|
||||||
|
str8_lit_comp("cast"),
|
||||||
|
str8_lit_comp("if"),
|
||||||
|
str8_lit_comp("ifs"),
|
||||||
|
str8_lit_comp("then"),
|
||||||
|
str8_lit_comp("else"),
|
||||||
|
str8_lit_comp("case"),
|
||||||
|
str8_lit_comp("for"),
|
||||||
|
str8_lit_comp("while"),
|
||||||
|
str8_lit_comp("break"),
|
||||||
|
str8_lit_comp("continue"),
|
||||||
|
str8_lit_comp("remove"),
|
||||||
|
str8_lit_comp("return"),
|
||||||
|
str8_lit_comp("inline"),
|
||||||
|
str8_lit_comp("null"),
|
||||||
|
str8_lit_comp("defer"),
|
||||||
|
str8_lit_comp("xx"),
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_keywords__zig[49] =
|
||||||
|
{
|
||||||
|
str8_lit_comp("addrspace"),
|
||||||
|
str8_lit_comp("align"),
|
||||||
|
str8_lit_comp("allowzero"),
|
||||||
|
str8_lit_comp("and"),
|
||||||
|
str8_lit_comp("anyframe"),
|
||||||
|
str8_lit_comp("anytype"),
|
||||||
|
str8_lit_comp("asm"),
|
||||||
|
str8_lit_comp("async"),
|
||||||
|
str8_lit_comp("await"),
|
||||||
|
str8_lit_comp("break"),
|
||||||
|
str8_lit_comp("callconv"),
|
||||||
|
str8_lit_comp("catch"),
|
||||||
|
str8_lit_comp("comptime"),
|
||||||
|
str8_lit_comp("const"),
|
||||||
|
str8_lit_comp("continue"),
|
||||||
|
str8_lit_comp("defer"),
|
||||||
|
str8_lit_comp("else"),
|
||||||
|
str8_lit_comp("enum"),
|
||||||
|
str8_lit_comp("errdefer"),
|
||||||
|
str8_lit_comp("error"),
|
||||||
|
str8_lit_comp("export"),
|
||||||
|
str8_lit_comp("extern"),
|
||||||
|
str8_lit_comp("fn"),
|
||||||
|
str8_lit_comp("for"),
|
||||||
|
str8_lit_comp("if"),
|
||||||
|
str8_lit_comp("inline"),
|
||||||
|
str8_lit_comp("noalias"),
|
||||||
|
str8_lit_comp("nosuspend"),
|
||||||
|
str8_lit_comp("noinline"),
|
||||||
|
str8_lit_comp("opaque"),
|
||||||
|
str8_lit_comp("or"),
|
||||||
|
str8_lit_comp("orelse"),
|
||||||
|
str8_lit_comp("packed"),
|
||||||
|
str8_lit_comp("pub"),
|
||||||
|
str8_lit_comp("resume"),
|
||||||
|
str8_lit_comp("return"),
|
||||||
|
str8_lit_comp("linksection"),
|
||||||
|
str8_lit_comp("struct"),
|
||||||
|
str8_lit_comp("suspend"),
|
||||||
|
str8_lit_comp("switch"),
|
||||||
|
str8_lit_comp("test"),
|
||||||
|
str8_lit_comp("threadlocal"),
|
||||||
|
str8_lit_comp("try"),
|
||||||
|
str8_lit_comp("union"),
|
||||||
|
str8_lit_comp("unreachable"),
|
||||||
|
str8_lit_comp("usingnamespace"),
|
||||||
|
str8_lit_comp("var"),
|
||||||
|
str8_lit_comp("volatile"),
|
||||||
|
str8_lit_comp("while"),
|
||||||
|
};
|
||||||
|
|
||||||
|
String8 txt_keywords__rust[43] =
|
||||||
|
{
|
||||||
|
str8_lit_comp("as"),
|
||||||
|
str8_lit_comp("break"),
|
||||||
|
str8_lit_comp("const"),
|
||||||
|
str8_lit_comp("continue"),
|
||||||
|
str8_lit_comp("crate"),
|
||||||
|
str8_lit_comp("else"),
|
||||||
|
str8_lit_comp("enum"),
|
||||||
|
str8_lit_comp("extern"),
|
||||||
|
str8_lit_comp("false"),
|
||||||
|
str8_lit_comp("fn"),
|
||||||
|
str8_lit_comp("for"),
|
||||||
|
str8_lit_comp("if"),
|
||||||
|
str8_lit_comp("impl"),
|
||||||
|
str8_lit_comp("in"),
|
||||||
|
str8_lit_comp("let"),
|
||||||
|
str8_lit_comp("loop"),
|
||||||
|
str8_lit_comp("match"),
|
||||||
|
str8_lit_comp("mod"),
|
||||||
|
str8_lit_comp("move"),
|
||||||
|
str8_lit_comp("mut"),
|
||||||
|
str8_lit_comp("pub"),
|
||||||
|
str8_lit_comp("ref"),
|
||||||
|
str8_lit_comp("return"),
|
||||||
|
str8_lit_comp("self"),
|
||||||
|
str8_lit_comp("Self"),
|
||||||
|
str8_lit_comp("static"),
|
||||||
|
str8_lit_comp("struct"),
|
||||||
|
str8_lit_comp("super"),
|
||||||
|
str8_lit_comp("trait"),
|
||||||
|
str8_lit_comp("true"),
|
||||||
|
str8_lit_comp("type"),
|
||||||
|
str8_lit_comp("unsafe"),
|
||||||
|
str8_lit_comp("use"),
|
||||||
|
str8_lit_comp("where"),
|
||||||
|
str8_lit_comp("while"),
|
||||||
|
str8_lit_comp("yield"),
|
||||||
|
str8_lit_comp("async"),
|
||||||
|
str8_lit_comp("await"),
|
||||||
|
str8_lit_comp("dyn"),
|
||||||
|
str8_lit_comp("macro_rules"),
|
||||||
|
str8_lit_comp("raw"),
|
||||||
|
str8_lit_comp("safe"),
|
||||||
|
str8_lit_comp("union"),
|
||||||
|
};
|
||||||
|
|
||||||
|
C_LINKAGE_END
|
||||||
|
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
// Copyright (c) Epic Games Tools
|
||||||
|
// Licensed under the MIT license (https://opensource.org/license/mit/)
|
||||||
|
|
||||||
|
//- GENERATED CODE
|
||||||
|
|
||||||
|
#ifndef TEXT_META_H
|
||||||
|
#define TEXT_META_H
|
||||||
|
|
||||||
|
typedef enum TXT_LangKind
|
||||||
|
{
|
||||||
|
TXT_LangKind_Null,
|
||||||
|
TXT_LangKind_C,
|
||||||
|
TXT_LangKind_CPlusPlus,
|
||||||
|
TXT_LangKind_Odin,
|
||||||
|
TXT_LangKind_Jai,
|
||||||
|
TXT_LangKind_Zig,
|
||||||
|
TXT_LangKind_Rust,
|
||||||
|
TXT_LangKind_DisasmX64Intel,
|
||||||
|
TXT_LangKind_COUNT,
|
||||||
|
} TXT_LangKind;
|
||||||
|
|
||||||
|
C_LINKAGE_BEGIN
|
||||||
|
extern String8 txt_lang_kind_ext_table[8];
|
||||||
|
extern String8Array txt_keywords_from_lang_kind_table[8];
|
||||||
|
extern String8Array txt_multichar_symbols_from_lang_kind_table[8];
|
||||||
|
extern TXT_TokenizerRuleArray txt_tokenizer_rules_from_lang_kind_table[8];
|
||||||
|
extern String8 txt_keywords__null[1];
|
||||||
|
extern String8 txt_multichar_symbols__null[1];
|
||||||
|
extern TXT_TokenizerRule txt_tokenizer_rules__null[1];
|
||||||
|
extern String8 txt_keywords__c[32];
|
||||||
|
extern String8 txt_multichar_symbols__c[20];
|
||||||
|
extern TXT_TokenizerRule txt_tokenizer_rules__c[7];
|
||||||
|
extern String8 txt_keywords__cpp[97];
|
||||||
|
extern String8 txt_keywords__odin[40];
|
||||||
|
extern String8 txt_keywords__jai[39];
|
||||||
|
extern String8 txt_keywords__zig[49];
|
||||||
|
extern String8 txt_keywords__rust[43];
|
||||||
|
|
||||||
|
C_LINKAGE_END
|
||||||
|
|
||||||
|
#endif // TEXT_META_H
|
||||||
+377
-73
@@ -4,6 +4,11 @@
|
|||||||
#undef LAYER_COLOR
|
#undef LAYER_COLOR
|
||||||
#define LAYER_COLOR 0xe34cd4ff
|
#define LAYER_COLOR 0xe34cd4ff
|
||||||
|
|
||||||
|
////////////////////////////////
|
||||||
|
//~ rjf: Generated Code
|
||||||
|
|
||||||
|
#include "generated/text.meta.c"
|
||||||
|
|
||||||
////////////////////////////////
|
////////////////////////////////
|
||||||
//~ rjf: Basic Helpers
|
//~ rjf: Basic Helpers
|
||||||
|
|
||||||
@@ -11,45 +16,13 @@ internal TXT_LangKind
|
|||||||
txt_lang_kind_from_extension(String8 extension)
|
txt_lang_kind_from_extension(String8 extension)
|
||||||
{
|
{
|
||||||
TXT_LangKind kind = TXT_LangKind_Null;
|
TXT_LangKind kind = TXT_LangKind_Null;
|
||||||
if(str8_match(extension, str8_lit("c"), 0) ||
|
for EachElement(idx, txt_ext_lang_kind_table)
|
||||||
str8_match(extension, str8_lit("h"), 0))
|
|
||||||
{
|
{
|
||||||
kind = TXT_LangKind_C;
|
if(str8_match(extension, txt_ext_lang_kind_table[idx].ext, 0))
|
||||||
}
|
{
|
||||||
else if(str8_match(extension, str8_lit("cpp"), StringMatchFlag_CaseInsensitive) ||
|
kind = txt_ext_lang_kind_table[idx].kind;
|
||||||
str8_match(extension, str8_lit("cxx"), StringMatchFlag_CaseInsensitive) ||
|
break;
|
||||||
str8_match(extension, str8_lit("cc"), StringMatchFlag_CaseInsensitive) ||
|
}
|
||||||
str8_match(extension, str8_lit("c++"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("ixx"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("cxxm"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("c++m"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("ccm"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("cppm"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("mpp"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("C"), 0) ||
|
|
||||||
str8_match(extension, str8_lit("hpp"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("hxx"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("hh"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("h++"), StringMatchFlag_CaseInsensitive) ||
|
|
||||||
str8_match(extension, str8_lit("H"), 0))
|
|
||||||
{
|
|
||||||
kind = TXT_LangKind_CPlusPlus;
|
|
||||||
}
|
|
||||||
else if(str8_match(extension, str8_lit("odin"), StringMatchFlag_CaseInsensitive))
|
|
||||||
{
|
|
||||||
kind = TXT_LangKind_Odin;
|
|
||||||
}
|
|
||||||
else if(str8_match(extension, str8_lit("jai"), StringMatchFlag_CaseInsensitive))
|
|
||||||
{
|
|
||||||
kind = TXT_LangKind_Jai;
|
|
||||||
}
|
|
||||||
else if(str8_match(extension, str8_lit("zig"), StringMatchFlag_CaseInsensitive))
|
|
||||||
{
|
|
||||||
kind = TXT_LangKind_Zig;
|
|
||||||
}
|
|
||||||
else if(str8_match(extension, str8_lit("rs"), StringMatchFlag_CaseInsensitive))
|
|
||||||
{
|
|
||||||
kind = TXT_LangKind_Rust;
|
|
||||||
}
|
}
|
||||||
return kind;
|
return kind;
|
||||||
}
|
}
|
||||||
@@ -57,20 +30,7 @@ txt_lang_kind_from_extension(String8 extension)
|
|||||||
internal String8
|
internal String8
|
||||||
txt_extension_from_lang_kind(TXT_LangKind kind)
|
txt_extension_from_lang_kind(TXT_LangKind kind)
|
||||||
{
|
{
|
||||||
String8 result = {0};
|
String8 result = txt_lang_kind_ext_table[kind];
|
||||||
switch(kind)
|
|
||||||
{
|
|
||||||
case TXT_LangKind_Null:
|
|
||||||
case TXT_LangKind_COUNT:
|
|
||||||
case TXT_LangKind_DisasmX64Intel:
|
|
||||||
{}break;
|
|
||||||
case TXT_LangKind_C: {result = str8_lit("c");}break;
|
|
||||||
case TXT_LangKind_CPlusPlus: {result = str8_lit("cpp");}break;
|
|
||||||
case TXT_LangKind_Odin: {result = str8_lit("odin");}break;
|
|
||||||
case TXT_LangKind_Jai: {result = str8_lit("jai");}break;
|
|
||||||
case TXT_LangKind_Zig: {result = str8_lit("zig");}break;
|
|
||||||
case TXT_LangKind_Rust: {result = str8_lit("rs");}break;
|
|
||||||
}
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -166,6 +126,343 @@ txt_token_array_from_list(Arena *arena, TXT_TokenList *list)
|
|||||||
////////////////////////////////
|
////////////////////////////////
|
||||||
//~ rjf: Lexing Functions
|
//~ rjf: Lexing Functions
|
||||||
|
|
||||||
|
internal TXT_TokenArray
|
||||||
|
txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, String8 string)
|
||||||
|
{
|
||||||
|
ProfBeginFunction();
|
||||||
|
Temp scratch = scratch_begin(&arena, 1);
|
||||||
|
U64 chunk_size = Clamp(8, string.size/8, 4096);
|
||||||
|
|
||||||
|
//- rjf: gather keywords
|
||||||
|
U64 keyword_slots_count = txt_keywords_from_lang_kind_table[lang_kind].count;
|
||||||
|
String8Node **keyword_slots = push_array(scratch.arena, String8Node *, keyword_slots_count);
|
||||||
|
for EachIndex(idx, txt_keywords_from_lang_kind_table[lang_kind].count)
|
||||||
|
{
|
||||||
|
String8 keyword = txt_keywords_from_lang_kind_table[lang_kind].v[idx];
|
||||||
|
U64 hash = u64_hash_from_str8(keyword);
|
||||||
|
U64 slot_idx = hash%keyword_slots_count;
|
||||||
|
String8Node *n = push_array(scratch.arena, String8Node, 1);
|
||||||
|
SLLStackPush(keyword_slots[slot_idx], n);
|
||||||
|
n->string = keyword;
|
||||||
|
}
|
||||||
|
|
||||||
|
//- rjf: gather multichar symbols
|
||||||
|
String8Array multichar_symbols = txt_multichar_symbols_from_lang_kind_table[lang_kind];
|
||||||
|
|
||||||
|
//- rjf: gather rules
|
||||||
|
U64 tokenizer_rule_slots_count = txt_tokenizer_rules_from_lang_kind_table[lang_kind].count;
|
||||||
|
TXT_TokenizerRulePtrNode **tokenizer_rule_opener_slots = push_array(scratch.arena, TXT_TokenizerRulePtrNode *, tokenizer_rule_slots_count);
|
||||||
|
TXT_TokenizerRulePtrNode **tokenizer_rule_closer_slots = push_array(scratch.arena, TXT_TokenizerRulePtrNode *, tokenizer_rule_slots_count);
|
||||||
|
{
|
||||||
|
TXT_TokenizerRuleArray rules = txt_tokenizer_rules_from_lang_kind_table[lang_kind];
|
||||||
|
for EachIndex(idx, rules.count)
|
||||||
|
{
|
||||||
|
TXT_TokenizerRule *r = &rules.v[idx];
|
||||||
|
U64 open_hash = u64_hash_from_str8(r->open_string);
|
||||||
|
U64 close_hash = u64_hash_from_str8(r->close_string);
|
||||||
|
U64 open_slot_idx = open_hash%tokenizer_rule_slots_count;
|
||||||
|
U64 close_slot_idx = close_hash%tokenizer_rule_slots_count;
|
||||||
|
TXT_TokenizerRulePtrNode *open_n = push_array(scratch.arena, TXT_TokenizerRulePtrNode, 1);
|
||||||
|
TXT_TokenizerRulePtrNode *close_n = push_array(scratch.arena, TXT_TokenizerRulePtrNode, 1);
|
||||||
|
open_n->v = r;
|
||||||
|
close_n->v = r;
|
||||||
|
SLLStackPush(tokenizer_rule_opener_slots[open_slot_idx], open_n);
|
||||||
|
SLLStackPush(tokenizer_rule_closer_slots[close_slot_idx], close_n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//- rjf: generate token list
|
||||||
|
TXT_TokenChunkList tokens = {0};
|
||||||
|
{
|
||||||
|
typedef struct TokenTask TokenTask;
|
||||||
|
struct TokenTask
|
||||||
|
{
|
||||||
|
TokenTask *next;
|
||||||
|
TXT_TokenizerRule *rule;
|
||||||
|
TXT_TokenKind kind;
|
||||||
|
U64 start_idx;
|
||||||
|
};
|
||||||
|
TXT_TokenizerRule *nil_rule = &txt_tokenizer_rules__null[0];
|
||||||
|
TokenTask *top_task = 0;
|
||||||
|
TokenTask *free_task = 0;
|
||||||
|
B32 escaped = 0;
|
||||||
|
B32 next_escaped = 0;
|
||||||
|
U64 byte_process_start_idx = 0;
|
||||||
|
for(U64 idx = 0; idx <= string.size;)
|
||||||
|
{
|
||||||
|
U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
|
||||||
|
U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
|
||||||
|
|
||||||
|
// rjf: unpack next bytes
|
||||||
|
U8 next_bytes[] = {byte, next_byte};
|
||||||
|
String8 string_1byte = str8(next_bytes, 1);
|
||||||
|
String8 string_2byte = str8(next_bytes, 2);
|
||||||
|
|
||||||
|
// rjf: update counter
|
||||||
|
if((idx-byte_process_start_idx) >= 1000 || idx == string.size)
|
||||||
|
{
|
||||||
|
add_progress(idx - byte_process_start_idx);
|
||||||
|
byte_process_start_idx = idx;
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: escaping
|
||||||
|
if(escaped && (byte != '\r' && byte != '\n'))
|
||||||
|
{
|
||||||
|
next_escaped = 0;
|
||||||
|
}
|
||||||
|
else if(!escaped && byte == '\\')
|
||||||
|
{
|
||||||
|
next_escaped = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: take starter, push new token tasks
|
||||||
|
{
|
||||||
|
TXT_TokenizerRule *new_rule = nil_rule;
|
||||||
|
TXT_TokenKind new_token_kind = TXT_TokenKind_Null;
|
||||||
|
|
||||||
|
// rjf: use next bytes to look up a rule from the table
|
||||||
|
if(top_task == 0)
|
||||||
|
{
|
||||||
|
TXT_TokenizerRule *active_rule = top_task ? top_task->rule : nil_rule;
|
||||||
|
U64 hash_1byte = u64_hash_from_str8(string_1byte);
|
||||||
|
U64 hash_2byte = u64_hash_from_str8(string_2byte);
|
||||||
|
U64 slot_1byte = hash_1byte%tokenizer_rule_slots_count;
|
||||||
|
U64 slot_2byte = hash_2byte%tokenizer_rule_slots_count;
|
||||||
|
if(new_rule == nil_rule)
|
||||||
|
{
|
||||||
|
for EachNode(n, TXT_TokenizerRulePtrNode, tokenizer_rule_opener_slots[slot_2byte])
|
||||||
|
{
|
||||||
|
TXT_TokenizerRule *n_parent_rule = nil_rule;
|
||||||
|
if(0 < n->v->parent_num && n->v->parent_num <= txt_tokenizer_rules_from_lang_kind_table[lang_kind].count)
|
||||||
|
{
|
||||||
|
n_parent_rule = &txt_tokenizer_rules_from_lang_kind_table[lang_kind].v[n->v->parent_num-1];
|
||||||
|
}
|
||||||
|
if(str8_match(n->v->open_string, string_2byte, 0) &&
|
||||||
|
active_rule == n_parent_rule)
|
||||||
|
{
|
||||||
|
new_rule = n->v;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(new_rule == nil_rule)
|
||||||
|
{
|
||||||
|
for EachNode(n, TXT_TokenizerRulePtrNode, tokenizer_rule_opener_slots[slot_1byte])
|
||||||
|
{
|
||||||
|
TXT_TokenizerRule *n_parent_rule = nil_rule;
|
||||||
|
if(0 < n->v->parent_num && n->v->parent_num <= txt_tokenizer_rules_from_lang_kind_table[lang_kind].count)
|
||||||
|
{
|
||||||
|
n_parent_rule = &txt_tokenizer_rules_from_lang_kind_table[lang_kind].v[n->v->parent_num-1];
|
||||||
|
}
|
||||||
|
if(str8_match(n->v->open_string, string_1byte, 0) &&
|
||||||
|
active_rule == n_parent_rule)
|
||||||
|
{
|
||||||
|
new_rule = n->v;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(new_rule != nil_rule)
|
||||||
|
{
|
||||||
|
new_token_kind = new_rule->token_kind;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: use next bytes to start a new token based on fallback rules
|
||||||
|
if(top_task == 0 && new_rule == nil_rule)
|
||||||
|
{
|
||||||
|
if(0){}
|
||||||
|
else if(char_is_space(byte)) { new_token_kind = TXT_TokenKind_Whitespace; }
|
||||||
|
else if(byte == '_' ||
|
||||||
|
byte == '$' ||
|
||||||
|
char_is_alpha(byte)) { new_token_kind = TXT_TokenKind_Identifier; }
|
||||||
|
else if(char_is_digit(byte, 10) ||
|
||||||
|
(byte == '.' &&
|
||||||
|
char_is_digit(next_byte, 10))) { new_token_kind = TXT_TokenKind_Numeric; }
|
||||||
|
else if(byte == '~' || byte == '!' ||
|
||||||
|
byte == '%' || byte == '^' ||
|
||||||
|
byte == '&' || byte == '*' ||
|
||||||
|
byte == '(' || byte == ')' ||
|
||||||
|
byte == '-' || byte == '=' ||
|
||||||
|
byte == '+' || byte == '[' ||
|
||||||
|
byte == ']' || byte == '{' ||
|
||||||
|
byte == '}' || byte == ':' ||
|
||||||
|
byte == ';' || byte == ',' ||
|
||||||
|
byte == '.' || byte == '<' ||
|
||||||
|
byte == '>' || byte == '/' ||
|
||||||
|
byte == '?' || byte == '|') { new_token_kind = TXT_TokenKind_Symbol; }
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: start new token
|
||||||
|
if(new_token_kind != TXT_TokenKind_Null)
|
||||||
|
{
|
||||||
|
TokenTask *task = free_task;
|
||||||
|
if(task != 0)
|
||||||
|
{
|
||||||
|
SLLStackPop(free_task);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
task = push_array(scratch.arena, TokenTask, 1);
|
||||||
|
}
|
||||||
|
SLLStackPush(top_task, task);
|
||||||
|
top_task->rule = new_rule;
|
||||||
|
top_task->kind = new_token_kind;
|
||||||
|
top_task->start_idx = idx;
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: invalid token kind -> emit error
|
||||||
|
else if(top_task == 0)
|
||||||
|
{
|
||||||
|
TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)};
|
||||||
|
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: look for ender based on rule's closing symbol
|
||||||
|
U64 ender_pad = 0;
|
||||||
|
B32 ender_found = 0;
|
||||||
|
B32 task_pop = 0;
|
||||||
|
if(top_task != 0 && idx > top_task->start_idx)
|
||||||
|
{
|
||||||
|
TXT_TokenKind active_token_kind = top_task->kind;
|
||||||
|
TXT_TokenizerRule *active_rule = top_task->rule;
|
||||||
|
U64 active_token_start_idx = top_task->start_idx;
|
||||||
|
if(idx == string.size)
|
||||||
|
{
|
||||||
|
ender_pad = 0;
|
||||||
|
ender_found = 1;
|
||||||
|
}
|
||||||
|
else if(active_rule != nil_rule &&
|
||||||
|
str8_match(string_1byte, active_rule->close_string, 0) ||
|
||||||
|
str8_match(string_2byte, active_rule->close_string, 0))
|
||||||
|
{
|
||||||
|
ender_found = 1;
|
||||||
|
ender_pad = active_rule->close_advance;
|
||||||
|
}
|
||||||
|
else switch(active_token_kind)
|
||||||
|
{
|
||||||
|
default:{}break;
|
||||||
|
case TXT_TokenKind_Whitespace:
|
||||||
|
{
|
||||||
|
ender_found = !char_is_space(byte);
|
||||||
|
}break;
|
||||||
|
case TXT_TokenKind_Identifier:
|
||||||
|
{
|
||||||
|
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128);
|
||||||
|
}break;
|
||||||
|
case TXT_TokenKind_Numeric:
|
||||||
|
{
|
||||||
|
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '.' && byte != '\'');
|
||||||
|
}break;
|
||||||
|
case TXT_TokenKind_Symbol:
|
||||||
|
{
|
||||||
|
ender_found = (byte != '~' && byte != '!' &&
|
||||||
|
byte != '%' && byte != '^' &&
|
||||||
|
byte != '&' && byte != '*' &&
|
||||||
|
byte != '(' && byte != ')' &&
|
||||||
|
byte != '-' && byte != '=' &&
|
||||||
|
byte != '+' && byte != '[' &&
|
||||||
|
byte != ']' && byte != '{' &&
|
||||||
|
byte != '}' && byte != ':' &&
|
||||||
|
byte != ';' && byte != ',' &&
|
||||||
|
byte != '.' && byte != '<' &&
|
||||||
|
byte != '>' && byte != '/' &&
|
||||||
|
byte != '?' && byte != '|');
|
||||||
|
}break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: next byte is ender => emit token
|
||||||
|
if(ender_found)
|
||||||
|
{
|
||||||
|
TXT_Token token = {top_task->kind, r1u64(top_task->start_idx, idx+ender_pad)};
|
||||||
|
TokenTask *popped = top_task;
|
||||||
|
SLLStackPop(top_task);
|
||||||
|
SLLStackPush(free_task, popped);
|
||||||
|
|
||||||
|
// rjf: trim \r's off of end
|
||||||
|
{
|
||||||
|
String8 token_string = str8_substr(string, token.range);
|
||||||
|
if(token_string.size != 0 && token_string.str[token_string.size-1] == '\r')
|
||||||
|
{
|
||||||
|
token.range.max -= 1;
|
||||||
|
ender_pad += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: identifier -> keyword in special cases
|
||||||
|
if(token.kind == TXT_TokenKind_Identifier)
|
||||||
|
{
|
||||||
|
String8 token_string = str8_substr(string, token.range);
|
||||||
|
U64 hash = u64_hash_from_str8(token_string);
|
||||||
|
U64 slot_idx = hash%keyword_slots_count;
|
||||||
|
for EachNode(n, String8Node, keyword_slots[slot_idx])
|
||||||
|
{
|
||||||
|
if(str8_match(token_string, n->string, 0))
|
||||||
|
{
|
||||||
|
token.kind = TXT_TokenKind_Keyword;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: split symbols by maximum-munch-rule
|
||||||
|
else if(token.kind == TXT_TokenKind_Symbol)
|
||||||
|
{
|
||||||
|
String8 token_string = str8_substr(string, token.range);
|
||||||
|
for(U64 off = 0, next_off = token_string.size; off < token_string.size; off = next_off)
|
||||||
|
{
|
||||||
|
B32 found = 0;
|
||||||
|
for(U64 idx = 0; idx < multichar_symbols.count; idx += 1)
|
||||||
|
{
|
||||||
|
if(str8_match(str8_substr(token_string, r1u64(off, off+multichar_symbols.v[idx].size)),
|
||||||
|
multichar_symbols.v[idx],
|
||||||
|
0))
|
||||||
|
{
|
||||||
|
found = 1;
|
||||||
|
next_off = off + Max(1, multichar_symbols.v[idx].size);
|
||||||
|
TXT_Token sub_token = {TXT_TokenKind_Symbol, r1u64(token.range.min+off, token.range.min+next_off)};
|
||||||
|
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &sub_token);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(!found)
|
||||||
|
{
|
||||||
|
next_off = off+1;
|
||||||
|
TXT_Token sub_token = {TXT_TokenKind_Symbol, r1u64(token.range.min+off, token.range.min+next_off)};
|
||||||
|
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &sub_token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: all other tokens
|
||||||
|
else
|
||||||
|
{
|
||||||
|
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: increment by ender padding
|
||||||
|
idx += ender_pad;
|
||||||
|
}
|
||||||
|
|
||||||
|
// rjf: advance by 1 byte if we haven't found an ender
|
||||||
|
if(!ender_found)
|
||||||
|
{
|
||||||
|
idx += 1;
|
||||||
|
}
|
||||||
|
escaped = next_escaped;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//- rjf: token list -> token array
|
||||||
|
TXT_TokenArray result = txt_token_array_from_chunk_list(arena, &tokens);
|
||||||
|
scratch_end(scratch);
|
||||||
|
ProfEnd();
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
internal TXT_TokenArray
|
internal TXT_TokenArray
|
||||||
txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string)
|
txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string)
|
||||||
{
|
{
|
||||||
@@ -1379,7 +1676,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
{
|
{
|
||||||
// NOTE(spey): Rust supports unicode identifiers. They are not handled in any way here,
|
// NOTE(spey): Rust supports unicode identifiers. They are not handled in any way here,
|
||||||
// but it might be worth looking into in the future.
|
// but it might be worth looking into in the future.
|
||||||
|
|
||||||
Temp scratch = scratch_begin(&arena, 1);
|
Temp scratch = scratch_begin(&arena, 1);
|
||||||
|
|
||||||
//- rjf: generate token list
|
//- rjf: generate token list
|
||||||
@@ -1388,7 +1685,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
S32 multiline_comment_nesting_level = 0;
|
S32 multiline_comment_nesting_level = 0;
|
||||||
S32 raw_string_nesting_level = 0;
|
S32 raw_string_nesting_level = 0;
|
||||||
S32 raw_string_ender_nesting_level = 0;
|
S32 raw_string_ender_nesting_level = 0;
|
||||||
|
|
||||||
// NOTE(spey): Rust's syntax is designed in such a way that we can't be sure what a token
|
// NOTE(spey): Rust's syntax is designed in such a way that we can't be sure what a token
|
||||||
// is immediately from the first character, so we have to keep track of some possibilities.
|
// is immediately from the first character, so we have to keep track of some possibilities.
|
||||||
B32 token_may_be_char = 0;
|
B32 token_may_be_char = 0;
|
||||||
@@ -1404,14 +1701,14 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
{
|
{
|
||||||
U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
|
U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
|
||||||
U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
|
U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
|
||||||
|
|
||||||
// rjf: update counter
|
// rjf: update counter
|
||||||
if(bytes_processed_counter != 0 && ((idx-byte_process_start_idx) >= 1000 || idx == string.size))
|
if(bytes_processed_counter != 0 && ((idx-byte_process_start_idx) >= 1000 || idx == string.size))
|
||||||
{
|
{
|
||||||
ins_atomic_u64_add_eval(bytes_processed_counter, (idx-byte_process_start_idx));
|
ins_atomic_u64_add_eval(bytes_processed_counter, (idx-byte_process_start_idx));
|
||||||
byte_process_start_idx = idx;
|
byte_process_start_idx = idx;
|
||||||
}
|
}
|
||||||
|
|
||||||
// rjf: escaping
|
// rjf: escaping
|
||||||
if(escaped && (byte != '\r' && byte != '\n'))
|
if(escaped && (byte != '\r' && byte != '\n'))
|
||||||
{
|
{
|
||||||
@@ -1421,10 +1718,10 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
{
|
{
|
||||||
next_escaped = 1;
|
next_escaped = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// rjf: take starter, determine active token kind
|
// rjf: take starter, determine active token kind
|
||||||
U64 starter_pad = 0;
|
U64 starter_pad = 0;
|
||||||
|
|
||||||
// spey: special case of starter for nested comments
|
// spey: special case of starter for nested comments
|
||||||
if(active_token_kind == TXT_TokenKind_Comment)
|
if(active_token_kind == TXT_TokenKind_Comment)
|
||||||
{
|
{
|
||||||
@@ -1451,9 +1748,9 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
char_is_digit(next_byte, 10))) { active_token_kind = TXT_TokenKind_Numeric; }
|
char_is_digit(next_byte, 10))) { active_token_kind = TXT_TokenKind_Numeric; }
|
||||||
else if(byte == '"') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; }
|
else if(byte == '"') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; }
|
||||||
else if((byte == 'c' || byte == 'b') &&
|
else if((byte == 'c' || byte == 'b') &&
|
||||||
next_byte == '"') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; starter_pad = 1; }
|
next_byte == '"') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 0; starter_pad = 1; }
|
||||||
else if((byte == 'c' || byte == 'b') &&
|
else if((byte == 'c' || byte == 'b') &&
|
||||||
next_byte == 'r') { active_token_kind = TXT_TokenKind_Identifier; token_may_be_string = 1; }
|
next_byte == 'r') { active_token_kind = TXT_TokenKind_Identifier; token_may_be_string = 1; }
|
||||||
else if(byte == '_' ||
|
else if(byte == '_' ||
|
||||||
char_is_alpha(byte)) { active_token_kind = TXT_TokenKind_Identifier; }
|
char_is_alpha(byte)) { active_token_kind = TXT_TokenKind_Identifier; }
|
||||||
else if(byte == '/' && next_byte == '/') { active_token_kind = TXT_TokenKind_Comment; starter_pad = 1; }
|
else if(byte == '/' && next_byte == '/') { active_token_kind = TXT_TokenKind_Comment; starter_pad = 1; }
|
||||||
@@ -1472,8 +1769,8 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
byte == '?' || byte == '|') { active_token_kind = TXT_TokenKind_Symbol; }
|
byte == '?' || byte == '|') { active_token_kind = TXT_TokenKind_Symbol; }
|
||||||
else if(byte == '\'') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; token_may_be_lifetime = 1; }
|
else if(byte == '\'') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; token_may_be_lifetime = 1; }
|
||||||
else if((byte == 'c' || byte == 'b') &&
|
else if((byte == 'c' || byte == 'b') &&
|
||||||
next_byte == '\'') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; starter_pad = 1; }
|
next_byte == '\'') { active_token_kind = TXT_TokenKind_String; token_may_be_char = 1; starter_pad = 1; }
|
||||||
|
|
||||||
// rjf: start new token
|
// rjf: start new token
|
||||||
if(active_token_kind != TXT_TokenKind_Null)
|
if(active_token_kind != TXT_TokenKind_Null)
|
||||||
{
|
{
|
||||||
@@ -1487,14 +1784,14 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
|
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
B32 is_on_starter = idx <= active_token_start_idx || token_may_be_string;
|
B32 is_on_starter = idx <= active_token_start_idx || token_may_be_string;
|
||||||
|
|
||||||
// spey: advance by starter padding byte(s) and reset byte/next_byte values
|
// spey: advance by starter padding byte(s) and reset byte/next_byte values
|
||||||
idx += starter_pad;
|
idx += starter_pad;
|
||||||
byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
|
byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
|
||||||
next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
|
next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
|
||||||
|
|
||||||
// rjf: look for ender
|
// rjf: look for ender
|
||||||
U64 ender_pad = 0;
|
U64 ender_pad = 0;
|
||||||
B32 ender_found = 0;
|
B32 ender_found = 0;
|
||||||
@@ -1540,10 +1837,10 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
else
|
else
|
||||||
{
|
{
|
||||||
if(0){}
|
if(0){}
|
||||||
|
|
||||||
// spey: regular string
|
// spey: regular string
|
||||||
else if(raw_string_nesting_level == 0) { ender_found = byte == '"'; }
|
else if(raw_string_nesting_level == 0) { ender_found = byte == '"'; }
|
||||||
|
|
||||||
// spey: raw string
|
// spey: raw string
|
||||||
else if(byte == '"' && next_byte == '#' &&
|
else if(byte == '"' && next_byte == '#' &&
|
||||||
raw_string_ender_nesting_level == 0) { raw_string_ender_nesting_level++; }
|
raw_string_ender_nesting_level == 0) { raw_string_ender_nesting_level++; }
|
||||||
@@ -1556,7 +1853,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
raw_string_ender_nesting_level >= 0) { raw_string_ender_nesting_level++; }
|
raw_string_ender_nesting_level >= 0) { raw_string_ender_nesting_level++; }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ender_pad += 1;
|
ender_pad += 1;
|
||||||
}break;
|
}break;
|
||||||
case TXT_TokenKind_Symbol:
|
case TXT_TokenKind_Symbol:
|
||||||
@@ -1584,7 +1881,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
{
|
{
|
||||||
if (byte == '*' && next_byte == '/')
|
if (byte == '*' && next_byte == '/')
|
||||||
multiline_comment_nesting_level--;
|
multiline_comment_nesting_level--;
|
||||||
|
|
||||||
ender_found = (active_token_start_idx+1 < idx && multiline_comment_nesting_level == 0);
|
ender_found = (active_token_start_idx+1 < idx && multiline_comment_nesting_level == 0);
|
||||||
ender_pad += 2;
|
ender_pad += 2;
|
||||||
}
|
}
|
||||||
@@ -1642,7 +1939,7 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
str8_lit_comp("async"),
|
str8_lit_comp("async"),
|
||||||
str8_lit_comp("await"),
|
str8_lit_comp("await"),
|
||||||
str8_lit_comp("dyn"),
|
str8_lit_comp("dyn"),
|
||||||
|
|
||||||
// weak keywords
|
// weak keywords
|
||||||
str8_lit_comp("macro_rules"),
|
str8_lit_comp("macro_rules"),
|
||||||
str8_lit_comp("raw"),
|
str8_lit_comp("raw"),
|
||||||
@@ -1712,17 +2009,17 @@ txt_token_array_from_string__rust(Arena *arena, U64 *bytes_processed_counter, St
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// rjf: all other tokens
|
// rjf: all other tokens
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
|
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
|
||||||
}
|
}
|
||||||
|
|
||||||
// rjf: increment by starter and ender padding
|
// rjf: increment by starter and ender padding
|
||||||
idx += ender_pad;
|
idx += ender_pad;
|
||||||
}
|
}
|
||||||
|
|
||||||
// rjf: advance by 1 byte if we haven't found an ender
|
// rjf: advance by 1 byte if we haven't found an ender
|
||||||
if(!ender_found)
|
if(!ender_found)
|
||||||
{
|
{
|
||||||
@@ -2459,10 +2756,17 @@ txt_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *gen_ou
|
|||||||
set_progress(Min(data.size, 1024) + data.size + data.size);
|
set_progress(Min(data.size, 1024) + data.size + data.size);
|
||||||
|
|
||||||
//- rjf: lex function * data -> tokens
|
//- rjf: lex function * data -> tokens
|
||||||
|
#if 1
|
||||||
if(lane_idx() == 0 && lex_function != 0)
|
if(lane_idx() == 0 && lex_function != 0)
|
||||||
{
|
{
|
||||||
shared->info.tokens = lex_function(shared->arena, 0, data);
|
shared->info.tokens = lex_function(shared->arena, 0, data);
|
||||||
}
|
}
|
||||||
|
#else
|
||||||
|
if(lane_idx() == 0)
|
||||||
|
{
|
||||||
|
shared->info.tokens = txt_token_array_from_lang_kind_string(shared->arena, lang, data);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
lane_sync();
|
lane_sync();
|
||||||
set_progress(Min(data.size, 1024) + data.size + data.size + data.size*(lex_function != 0));
|
set_progress(Min(data.size, 1024) + data.size + data.size + data.size*(lex_function != 0));
|
||||||
TXT_TokenArray tokens = shared->info.tokens;
|
TXT_TokenArray tokens = shared->info.tokens;
|
||||||
|
|||||||
+33
-14
@@ -32,6 +32,32 @@ typedef enum TXT_TokenKind
|
|||||||
}
|
}
|
||||||
TXT_TokenKind;
|
TXT_TokenKind;
|
||||||
|
|
||||||
|
typedef struct TXT_TokenizerRule TXT_TokenizerRule;
|
||||||
|
struct TXT_TokenizerRule
|
||||||
|
{
|
||||||
|
TXT_TokenKind token_kind;
|
||||||
|
String8 open_string;
|
||||||
|
String8 close_string;
|
||||||
|
U32 close_advance;
|
||||||
|
B32 nesting;
|
||||||
|
B32 escaping;
|
||||||
|
U32 parent_num;
|
||||||
|
};
|
||||||
|
|
||||||
|
typedef struct TXT_TokenizerRulePtrNode TXT_TokenizerRulePtrNode;
|
||||||
|
struct TXT_TokenizerRulePtrNode
|
||||||
|
{
|
||||||
|
TXT_TokenizerRulePtrNode *next;
|
||||||
|
TXT_TokenizerRule *v;
|
||||||
|
};
|
||||||
|
|
||||||
|
typedef struct TXT_TokenizerRuleArray TXT_TokenizerRuleArray;
|
||||||
|
struct TXT_TokenizerRuleArray
|
||||||
|
{
|
||||||
|
TXT_TokenizerRule *v;
|
||||||
|
U64 count;
|
||||||
|
};
|
||||||
|
|
||||||
typedef struct TXT_Token TXT_Token;
|
typedef struct TXT_Token TXT_Token;
|
||||||
struct TXT_Token
|
struct TXT_Token
|
||||||
{
|
{
|
||||||
@@ -138,21 +164,12 @@ struct TXT_LineTokensSlice
|
|||||||
};
|
};
|
||||||
|
|
||||||
////////////////////////////////
|
////////////////////////////////
|
||||||
//~ rjf: Language Kind Types
|
//~ rjf: Generated Code
|
||||||
|
|
||||||
typedef enum TXT_LangKind
|
#include "generated/text.meta.h"
|
||||||
{
|
|
||||||
TXT_LangKind_Null,
|
////////////////////////////////
|
||||||
TXT_LangKind_C,
|
//~ rjf: Language Kind Types
|
||||||
TXT_LangKind_CPlusPlus,
|
|
||||||
TXT_LangKind_Odin,
|
|
||||||
TXT_LangKind_Jai,
|
|
||||||
TXT_LangKind_Zig,
|
|
||||||
TXT_LangKind_Rust,
|
|
||||||
TXT_LangKind_DisasmX64Intel,
|
|
||||||
TXT_LangKind_COUNT
|
|
||||||
}
|
|
||||||
TXT_LangKind;
|
|
||||||
|
|
||||||
typedef TXT_TokenArray TXT_LangLexFunctionType(Arena *arena, U64 *bytes_processed_counter, String8 string);
|
typedef TXT_TokenArray TXT_LangLexFunctionType(Arena *arena, U64 *bytes_processed_counter, String8 string);
|
||||||
|
|
||||||
@@ -180,6 +197,8 @@ internal TXT_TokenArray txt_token_array_from_list(Arena *arena, TXT_TokenList *l
|
|||||||
////////////////////////////////
|
////////////////////////////////
|
||||||
//~ rjf: Lexing Functions
|
//~ rjf: Lexing Functions
|
||||||
|
|
||||||
|
internal TXT_TokenArray txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, String8 string);
|
||||||
|
|
||||||
internal TXT_TokenArray txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string);
|
internal TXT_TokenArray txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string);
|
||||||
internal TXT_TokenArray txt_token_array_from_string__odin(Arena *arena, U64 *bytes_processed_counter, String8 string);
|
internal TXT_TokenArray txt_token_array_from_string__odin(Arena *arena, U64 *bytes_processed_counter, String8 string);
|
||||||
internal TXT_TokenArray txt_token_array_from_string__jai(Arena *arena, U64 *bytes_processed_counter, String8 string);
|
internal TXT_TokenArray txt_token_array_from_string__jai(Arena *arena, U64 *bytes_processed_counter, String8 string);
|
||||||
|
|||||||
@@ -0,0 +1,246 @@
|
|||||||
|
// Copyright (c) Epic Games Tools
|
||||||
|
// Licensed under the MIT license (https://opensource.org/license/mit/)
|
||||||
|
|
||||||
|
@table(name ext keywords_table multichar_symbol_table tokenizer_rules_table)
|
||||||
|
TXT_LangKindTable:
|
||||||
|
{
|
||||||
|
{ Null null null null null}
|
||||||
|
{ C c c c c }
|
||||||
|
{ CPlusPlus cpp cpp c c }
|
||||||
|
{ Odin odin odin c null}
|
||||||
|
{ Jai jai jai c null}
|
||||||
|
{ Zig zig zig c null}
|
||||||
|
{ Rust rs rust c null}
|
||||||
|
{ DisasmX64Intel disasm_x64_intel null null null}
|
||||||
|
}
|
||||||
|
|
||||||
|
@table(string kind)
|
||||||
|
TXT_LangKindExtTable:
|
||||||
|
{
|
||||||
|
{"h" C }
|
||||||
|
{"cxx" CPlusPlus}
|
||||||
|
{"cc" CPlusPlus}
|
||||||
|
{"c++" CPlusPlus}
|
||||||
|
{"ixx" CPlusPlus}
|
||||||
|
{"cxxm" CPlusPlus}
|
||||||
|
{"c++m" CPlusPlus}
|
||||||
|
{"ccm" CPlusPlus}
|
||||||
|
{"cppm" CPlusPlus}
|
||||||
|
{"mpp" CPlusPlus}
|
||||||
|
{"C" CPlusPlus}
|
||||||
|
{"hpp" CPlusPlus}
|
||||||
|
{"hxx" CPlusPlus}
|
||||||
|
{"hh" CPlusPlus}
|
||||||
|
{"h++" CPlusPlus}
|
||||||
|
{"H" CPlusPlus}
|
||||||
|
}
|
||||||
|
|
||||||
|
@enum TXT_LangKind:
|
||||||
|
{
|
||||||
|
@expand(TXT_LangKindTable a) `$(a.name)`,
|
||||||
|
COUNT
|
||||||
|
}
|
||||||
|
|
||||||
|
@data(String8)
|
||||||
|
txt_lang_kind_ext_table:
|
||||||
|
{
|
||||||
|
@expand(TXT_LangKindTable a) `str8_lit_comp("$(a.ext)")`
|
||||||
|
}
|
||||||
|
|
||||||
|
@data(`struct {String8 ext; TXT_LangKind kind;}`) @c_file
|
||||||
|
txt_ext_lang_kind_table:
|
||||||
|
{
|
||||||
|
@expand(TXT_LangKindTable a) `{str8_lit_comp("$(a.ext)"), TXT_LangKind_$(a.name)}`,
|
||||||
|
@expand(TXT_LangKindExtTable a) `{str8_lit_comp("$(a.string)"), TXT_LangKind_$(a.kind)}`,
|
||||||
|
}
|
||||||
|
|
||||||
|
@data(String8Array) txt_keywords_from_lang_kind_table:
|
||||||
|
{
|
||||||
|
@expand(TXT_LangKindTable a) `{txt_keywords__$(a.keywords_table), ArrayCount(txt_keywords__$(a.keywords_table))}`
|
||||||
|
}
|
||||||
|
|
||||||
|
@data(String8Array) txt_multichar_symbols_from_lang_kind_table:
|
||||||
|
{
|
||||||
|
@expand(TXT_LangKindTable a) `{txt_multichar_symbols__$(a.multichar_symbol_table), ArrayCount(txt_multichar_symbols__$(a.multichar_symbol_table))}`
|
||||||
|
}
|
||||||
|
|
||||||
|
@data(TXT_TokenizerRuleArray) txt_tokenizer_rules_from_lang_kind_table:
|
||||||
|
{
|
||||||
|
@expand(TXT_LangKindTable a) `{txt_tokenizer_rules__$(a.tokenizer_rules_table), ArrayCount(txt_tokenizer_rules__$(a.tokenizer_rules_table))}`
|
||||||
|
}
|
||||||
|
|
||||||
|
@data(String8) txt_keywords__null:{`{0}`}
|
||||||
|
@data(String8) txt_multichar_symbols__null:{`{0}`}
|
||||||
|
@data(TXT_TokenizerRule) txt_tokenizer_rules__null:{`{TXT_TokenKind_Null}`}
|
||||||
|
|
||||||
|
////////////////////////////////
|
||||||
|
//~ rjf: C
|
||||||
|
|
||||||
|
@table(string)
|
||||||
|
txt_keywords__c:
|
||||||
|
{
|
||||||
|
{"auto"} {"break"} {"case"} {"char"}
|
||||||
|
{"const"} {"continue"} {"default"} {"do"}
|
||||||
|
{"double"} {"else"} {"enum"} {"extern"}
|
||||||
|
{"float"} {"for"} {"goto"} {"if"}
|
||||||
|
{"int"} {"long"} {"register"} {"return"}
|
||||||
|
{"short"} {"signed"} {"sizeof"} {"static"}
|
||||||
|
{"struct"} {"switch"} {"typedef"} {"union"}
|
||||||
|
{"unsigned"} {"void"} {"volatile"} {"while"}
|
||||||
|
}
|
||||||
|
@data(String8) txt_keywords__c: {@expand(txt_keywords__c a) `str8_lit_comp("$(a.string)")`}
|
||||||
|
|
||||||
|
@table(string)
|
||||||
|
txt_multichar_symbols__c:
|
||||||
|
{
|
||||||
|
{"<<"}
|
||||||
|
{">>"}
|
||||||
|
{"<="}
|
||||||
|
{">="}
|
||||||
|
{"=="}
|
||||||
|
{"!="}
|
||||||
|
{"&&"}
|
||||||
|
{"||"}
|
||||||
|
{"|="}
|
||||||
|
{"&="}
|
||||||
|
{"^="}
|
||||||
|
{"~="}
|
||||||
|
{"+="}
|
||||||
|
{"-="}
|
||||||
|
{"*="}
|
||||||
|
{"/="}
|
||||||
|
{"%="}
|
||||||
|
{"<<="}
|
||||||
|
{">>="}
|
||||||
|
{"->"}
|
||||||
|
}
|
||||||
|
@data(String8) txt_multichar_symbols__c: {@expand(txt_multichar_symbols__c a) `str8_lit_comp("$(a.string)")`}
|
||||||
|
|
||||||
|
@table(token_kind open_string close_string close_advance nesting escaping parent_num) txt_tokenizer_rules__c:
|
||||||
|
{
|
||||||
|
{Comment "//" "\\n" 0 0 1 0}
|
||||||
|
{Comment "/*" "*/" 2 0 0 0}
|
||||||
|
{Meta "#" "\\n" 0 0 1 0}
|
||||||
|
{String '\\"' '\\"' 1 0 1 0}
|
||||||
|
{String "'" "'" 1 0 1 0}
|
||||||
|
{String '\\"' '\\"' 1 0 1 3}
|
||||||
|
{String "<" ">" 1 0 1 3}
|
||||||
|
}
|
||||||
|
@data(TXT_TokenizerRule) txt_tokenizer_rules__c:
|
||||||
|
{
|
||||||
|
@expand(txt_tokenizer_rules__c a) `{TXT_TokenKind_$(a.token_kind), str8_lit_comp("$(a.open_string)"), str8_lit_comp("$(a.close_string)"), $(a.close_advance), $(a.nesting), $(a.escaping), $(a.parent_num)}`
|
||||||
|
}
|
||||||
|
|
||||||
|
////////////////////////////////
|
||||||
|
//~ rjf: C++
|
||||||
|
|
||||||
|
@table(string)
|
||||||
|
txt_keywords__cpp:
|
||||||
|
{
|
||||||
|
{"alignas"} {"alignof"} {"and"} {"and_eq"}
|
||||||
|
{"asm"} {"atomic_cancel"} {"atomic_commit"} {"atomic_noexcept"}
|
||||||
|
{"auto"} {"bitand"} {"bitor"} {"bool"}
|
||||||
|
{"break"} {"case"} {"catch"} {"char"}
|
||||||
|
{"char8_t"} {"char16_t"} {"char32_t"} {"class"}
|
||||||
|
{"compl"} {"concept"} {"const"} {"consteval"}
|
||||||
|
{"constexpr"} {"constinit"} {"const_cast"} {"continue"}
|
||||||
|
{"co_await"} {"co_return"} {"co_yield"} {"decltype"}
|
||||||
|
{"default"} {"delete"} {"do"} {"double"}
|
||||||
|
{"dynamic_cast"} {"else"} {"enum"} {"explicit"}
|
||||||
|
{"export"} {"extern"} {"false"} {"float"}
|
||||||
|
{"for"} {"friend"} {"goto"} {"if"}
|
||||||
|
{"inline"} {"int"} {"long"} {"mutable"}
|
||||||
|
{"namespace"} {"new"} {"noexcept"} {"not"}
|
||||||
|
{"not_eq"} {"nullptr"} {"operator"} {"or"}
|
||||||
|
{"or_eq"} {"private"} {"protected"} {"public"}
|
||||||
|
{"reflexpr"} {"register"} {"reinterpret_cast"} {"requires"}
|
||||||
|
{"return"} {"short"} {"signed"} {"sizeof"}
|
||||||
|
{"static"} {"static_assert"} {"static_cast"} {"struct"}
|
||||||
|
{"switch"} {"synchronized"} {"template"} {"this"}
|
||||||
|
{"thread_local"} {"throw"} {"true"} {"try"}
|
||||||
|
{"typedef"} {"typeid"} {"typename"} {"union"}
|
||||||
|
{"unsigned"} {"using"} {"virtual"} {"void"}
|
||||||
|
{"volatile"} {"wchar_t"} {"while"} {"xor"}
|
||||||
|
{"xor_eq"}
|
||||||
|
}
|
||||||
|
@data(String8) txt_keywords__cpp: {@expand(txt_keywords__cpp a) `str8_lit_comp("$(a.string)")`}
|
||||||
|
|
||||||
|
////////////////////////////////
|
||||||
|
//~ rjf: Odin
|
||||||
|
|
||||||
|
@table(string)
|
||||||
|
txt_keywords__odin:
|
||||||
|
{
|
||||||
|
{"align_of"} {"asm"} {"auto_cast"} {"bit_set"}
|
||||||
|
{"break"} {"case"} {"cast"} {"context"}
|
||||||
|
{"continue"} {"defer"} {"distinct"} {"do"}
|
||||||
|
{"dynamic"} {"else"} {"enum"} {"fallthrough"}
|
||||||
|
{"for"} {"foreign"} {"if"} {"in"}
|
||||||
|
{"map"} {"matrix"} {"not_in"} {"or_break"}
|
||||||
|
{"or_continue"} {"or_else"} {"or_return"} {"package"}
|
||||||
|
{"proc"} {"return"} {"size_of"} {"struct"}
|
||||||
|
{"switch"} {"transmute"} {"typeid"} {"union"}
|
||||||
|
{"using"} {"when"} {"where"} {"import"}
|
||||||
|
}
|
||||||
|
@data(String8) txt_keywords__odin: {@expand(txt_keywords__odin a) `str8_lit_comp("$(a.string)")`}
|
||||||
|
|
||||||
|
////////////////////////////////
|
||||||
|
//~ rjf: Jai
|
||||||
|
|
||||||
|
@table(string)
|
||||||
|
txt_keywords__jai:
|
||||||
|
{
|
||||||
|
{"bool"} {"true"} {"false"} {"int"}
|
||||||
|
{"s8"} {"u8"} {"s16"} {"u16"}
|
||||||
|
{"s32"} {"u32"} {"s64"} {"u64"}
|
||||||
|
{"s128"} {"u128"} {"float"} {"float32"}
|
||||||
|
{"float64"} {"void"} {"enum"} {"enum_flags"}
|
||||||
|
{"size_of"} {"string"} {"type_of"} {"cast"}
|
||||||
|
{"if"} {"ifs"} {"then"} {"else"}
|
||||||
|
{"case"} {"for"} {"while"} {"break"}
|
||||||
|
{"continue"} {"remove"} {"return"} {"inline"}
|
||||||
|
{"null"} {"defer"} {"xx"}
|
||||||
|
}
|
||||||
|
@data(String8) txt_keywords__jai: {@expand(txt_keywords__jai a) `str8_lit_comp("$(a.string)")`}
|
||||||
|
|
||||||
|
////////////////////////////////
|
||||||
|
//~ rjf: Zig
|
||||||
|
|
||||||
|
@table(string)
|
||||||
|
txt_keywords__zig:
|
||||||
|
{
|
||||||
|
{"addrspace"} {"align"} {"allowzero"} {"and"}
|
||||||
|
{"anyframe"} {"anytype"} {"asm"} {"async"}
|
||||||
|
{"await"} {"break"} {"callconv"} {"catch"}
|
||||||
|
{"comptime"} {"const"} {"continue"} {"defer"}
|
||||||
|
{"else"} {"enum"} {"errdefer"} {"error"}
|
||||||
|
{"export"} {"extern"} {"fn"} {"for"}
|
||||||
|
{"if"} {"inline"} {"noalias"} {"nosuspend"}
|
||||||
|
{"noinline"} {"opaque"} {"or"} {"orelse"}
|
||||||
|
{"packed"} {"pub"} {"resume"} {"return"}
|
||||||
|
{"linksection"} {"struct"} {"suspend"} {"switch"}
|
||||||
|
{"test"} {"threadlocal"} {"try"} {"union"}
|
||||||
|
{"unreachable"} {"usingnamespace"} {"var"} {"volatile"}
|
||||||
|
{"while"}
|
||||||
|
}
|
||||||
|
@data(String8) txt_keywords__zig: {@expand(txt_keywords__zig a) `str8_lit_comp("$(a.string)")`}
|
||||||
|
|
||||||
|
////////////////////////////////
|
||||||
|
//~ rjf: Rust
|
||||||
|
|
||||||
|
@table(string)
|
||||||
|
txt_keywords__rust:
|
||||||
|
{
|
||||||
|
{"as"} {"break"} {"const"} {"continue"}
|
||||||
|
{"crate"} {"else"} {"enum"} {"extern"}
|
||||||
|
{"false"} {"fn"} {"for"} {"if"}
|
||||||
|
{"impl"} {"in"} {"let"} {"loop"}
|
||||||
|
{"match"} {"mod"} {"move"} {"mut"}
|
||||||
|
{"pub"} {"ref"} {"return"} {"self"}
|
||||||
|
{"Self"} {"static"} {"struct"} {"super"}
|
||||||
|
{"trait"} {"true"} {"type"} {"unsafe"}
|
||||||
|
{"use"} {"where"} {"while"} {"yield"}
|
||||||
|
{"async"} {"await"} {"dyn"} {"macro_rules"}
|
||||||
|
{"raw"} {"safe"} {"union"}
|
||||||
|
}
|
||||||
|
@data(String8) txt_keywords__rust: {@expand(txt_keywords__rust a) `str8_lit_comp("$(a.string)")`}
|
||||||
Reference in New Issue
Block a user