ed/raddebugger
1
0
Fork 0
mirror of https://github.com/Ed94/raddebugger.git synced 2026-06-18 18:12:23 -07:00
Code Issues Packages Projects Releases Wiki Activity
Files
f9d1ffa5fc9624ca6d55f3652abf7db3ac1b850b
raddebugger/src/text/text.c
T

2290 lines
80 KiB
C
Raw Blame History

// Copyright (c) Epic Games Tools
// Licensed under the MIT license (https://opensource.org/license/mit/)
#undef LAYER_COLOR
#define LAYER_COLOR 0xe34cd4ff
////////////////////////////////
//~ rjf: Basic Helpers
internal TXT_LangKind
txt_lang_kind_from_extension(String8 extension)
{
TXT_LangKind kind = TXT_LangKind_Null;
if(str8_match(extension, str8_lit("c"), 0) ||
str8_match(extension, str8_lit("h"), 0))
{
kind = TXT_LangKind_C;
}
else if(str8_match(extension, str8_lit("cpp"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("cxx"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("cc"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("c++"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("ixx"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("cxxm"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("c++m"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("ccm"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("cppm"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("mpp"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("C"), 0) ||
str8_match(extension, str8_lit("hpp"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("hxx"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("hh"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("h++"), StringMatchFlag_CaseInsensitive) ||
str8_match(extension, str8_lit("H"), 0))
{
kind = TXT_LangKind_CPlusPlus;
}
else if(str8_match(extension, str8_lit("odin"), StringMatchFlag_CaseInsensitive))
{
kind = TXT_LangKind_Odin;
}
else if(str8_match(extension, str8_lit("jai"), StringMatchFlag_CaseInsensitive))
{
kind = TXT_LangKind_Jai;
}
else if(str8_match(extension, str8_lit("zig"), StringMatchFlag_CaseInsensitive))
{
kind = TXT_LangKind_Zig;
}
return kind;
}
internal String8
txt_extension_from_lang_kind(TXT_LangKind kind)
{
String8 result = {0};
switch(kind)
{
case TXT_LangKind_Null:
case TXT_LangKind_COUNT:
case TXT_LangKind_DisasmX64Intel:
{}break;
case TXT_LangKind_C: {result = str8_lit("c");}break;
case TXT_LangKind_CPlusPlus: {result = str8_lit("cpp");}break;
case TXT_LangKind_Odin: {result = str8_lit("odin");}break;
case TXT_LangKind_Jai: {result = str8_lit("jai");}break;
case TXT_LangKind_Zig: {result = str8_lit("zig");}break;
}
return result;
}
internal TXT_LangKind
txt_lang_kind_from_arch(Arch arch)
{
TXT_LangKind kind = TXT_LangKind_Null;
switch(arch)
{
default:{}break;
case Arch_x64:{kind = TXT_LangKind_DisasmX64Intel;}break;
}
return kind;
}
internal TXT_LangLexFunctionType *
txt_lex_function_from_lang_kind(TXT_LangKind kind)
{
TXT_LangLexFunctionType *fn = 0;
switch(kind)
{
default:{}break;
case TXT_LangKind_C: {fn = txt_token_array_from_string__c_cpp;}break;
case TXT_LangKind_CPlusPlus: {fn = txt_token_array_from_string__c_cpp;}break;
case TXT_LangKind_Odin: {fn = txt_token_array_from_string__odin;}break;
case TXT_LangKind_Jai: {fn = txt_token_array_from_string__jai;}break;
case TXT_LangKind_Zig: {fn = txt_token_array_from_string__zig;}break;
case TXT_LangKind_DisasmX64Intel:{fn = txt_token_array_from_string__disasm_x64_intel;}break;
}
return fn;
}
////////////////////////////////
//~ rjf: Token Type Functions
internal void
txt_token_chunk_list_push(Arena *arena, TXT_TokenChunkList *list, U64 cap, TXT_Token *token)
{
TXT_TokenChunkNode *node = list->last;
if(node == 0 || node->count >= node->cap)
{
node = push_array(arena, TXT_TokenChunkNode, 1);
SLLQueuePush(list->first, list->last, node);
node->cap = cap;
node->v = push_array_no_zero(arena, TXT_Token, node->cap);
list->chunk_count += 1;
}
MemoryCopyStruct(&node->v[node->count], token);
node->count += 1;
list->token_count += 1;
}
internal void
txt_token_list_push(Arena *arena, TXT_TokenList *list, TXT_Token *token)
{
TXT_TokenNode *node = push_array(arena, TXT_TokenNode, 1);
MemoryCopyStruct(&node->v, token);
SLLQueuePush(list->first, list->last, node);
list->count += 1;
}
internal TXT_TokenArray
txt_token_array_from_chunk_list(Arena *arena, TXT_TokenChunkList *list)
{
TXT_TokenArray array = {0};
array.count = list->token_count;
array.v = push_array_no_zero(arena, TXT_Token, array.count);
U64 idx = 0;
for(TXT_TokenChunkNode *n = list->first; n != 0; n = n->next)
{
MemoryCopy(array.v+idx, n->v, n->count*sizeof(TXT_Token));
idx += n->count;
}
return array;
}
internal TXT_TokenArray
txt_token_array_from_list(Arena *arena, TXT_TokenList *list)
{
TXT_TokenArray array = {0};
array.count = list->count;
array.v = push_array_no_zero(arena, TXT_Token, array.count);
U64 idx = 0;
for(TXT_TokenNode *n = list->first; n != 0; n = n->next)
{
MemoryCopyStruct(array.v+idx, &n->v);
idx += 1;
}
return array;
}
////////////////////////////////
//~ rjf: Lexing Functions
internal TXT_TokenArray
txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, String8 string)
{
ProfBeginFunction();
Temp scratch = scratch_begin(&arena, 1);
U64 chunk_size = Clamp(8, string.size/8, 4096);
//- rjf: generate token list
TXT_TokenChunkList tokens = {0};
{
B32 comment_is_single_line = 0;
B32 string_is_char = 0;
TXT_TokenKind active_token_kind = TXT_TokenKind_Null;
U64 active_token_start_idx = 0;
B32 escaped = 0;
B32 next_escaped = 0;
U64 byte_process_start_idx = 0;
for(U64 idx = 0; idx <= string.size;)
{
U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
// rjf: update counter
if(bytes_processed_counter != 0 && ((idx-byte_process_start_idx) >= 1000 || idx == string.size))
{
ins_atomic_u64_add_eval(bytes_processed_counter, (idx-byte_process_start_idx));
byte_process_start_idx = idx;
}
// rjf: escaping
if(escaped && (byte != '\r' && byte != '\n'))
{
next_escaped = 0;
}
else if(!escaped && byte == '\\')
{
next_escaped = 1;
}
// rjf: take starter, determine active token kind
if(active_token_kind == TXT_TokenKind_Null)
{
// rjf: use next bytes to start a new token
if(0){}
else if(char_is_space(byte)) { active_token_kind = TXT_TokenKind_Whitespace; }
else if(byte == '_' ||
byte == '$' ||
char_is_alpha(byte)) { active_token_kind = TXT_TokenKind_Identifier; }
else if(char_is_digit(byte, 10) ||
(byte == '.' &&
char_is_digit(next_byte, 10))) { active_token_kind = TXT_TokenKind_Numeric; }
else if(byte == '"') { active_token_kind = TXT_TokenKind_String; string_is_char = 0; }
else if(byte == '\'') { active_token_kind = TXT_TokenKind_String; string_is_char = 1; }
else if(byte == '/' && next_byte == '/') { active_token_kind = TXT_TokenKind_Comment; comment_is_single_line = 1; }
else if(byte == '/' && next_byte == '*') { active_token_kind = TXT_TokenKind_Comment; comment_is_single_line = 0; }
else if(byte == '~' || byte == '!' ||
byte == '%' || byte == '^' ||
byte == '&' || byte == '*' ||
byte == '(' || byte == ')' ||
byte == '-' || byte == '=' ||
byte == '+' || byte == '[' ||
byte == ']' || byte == '{' ||
byte == '}' || byte == ':' ||
byte == ';' || byte == ',' ||
byte == '.' || byte == '<' ||
byte == '>' || byte == '/' ||
byte == '?' || byte == '|') { active_token_kind = TXT_TokenKind_Symbol; }
else if(byte == '#') { active_token_kind = TXT_TokenKind_Meta; }
// rjf: start new token
if(active_token_kind != TXT_TokenKind_Null)
{
active_token_start_idx = idx;
}
// rjf: invalid token kind -> emit error
else
{
TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)};
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
}
}
// rjf: look for ender
U64 ender_pad = 0;
B32 ender_found = 0;
if(active_token_kind != TXT_TokenKind_Null && idx>active_token_start_idx)
{
if(idx == string.size)
{
ender_pad = 0;
ender_found = 1;
}
else switch(active_token_kind)
{
default:break;
case TXT_TokenKind_Whitespace:
{
ender_found = !char_is_space(byte);
}break;
case TXT_TokenKind_Identifier:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128);
}break;
case TXT_TokenKind_Numeric:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '.' && byte != '\'');
}break;
case TXT_TokenKind_String:
{
ender_found = (!escaped && ((!string_is_char && byte == '"') || (string_is_char && byte == '\'')));
ender_pad += 1;
}break;
case TXT_TokenKind_Symbol:
{
ender_found = (byte != '~' && byte != '!' &&
byte != '%' && byte != '^' &&
byte != '&' && byte != '*' &&
byte != '(' && byte != ')' &&
byte != '-' && byte != '=' &&
byte != '+' && byte != '[' &&
byte != ']' && byte != '{' &&
byte != '}' && byte != ':' &&
byte != ';' && byte != ',' &&
byte != '.' && byte != '<' &&
byte != '>' && byte != '/' &&
byte != '?' && byte != '|');
}break;
case TXT_TokenKind_Comment:
{
if(comment_is_single_line)
{
ender_found = (!escaped && (byte == '\r' || byte == '\n'));
}
else
{
ender_found = (active_token_start_idx+1 < idx && byte == '*' && next_byte == '/');
ender_pad += 2;
}
}break;
case TXT_TokenKind_Meta:
{
ender_found = (!escaped && (byte == '\r' || byte == '\n'));
}break;
}
}
// rjf: next byte is ender => emit token
if(ender_found)
{
TXT_Token token = {active_token_kind, r1u64(active_token_start_idx, idx+ender_pad)};
active_token_kind = TXT_TokenKind_Null;
// rjf: identifier -> keyword in special cases
if(token.kind == TXT_TokenKind_Identifier)
{
read_only local_persist String8 cpp_keywords[] =
{
str8_lit_comp("alignas"),
str8_lit_comp("alignof"),
str8_lit_comp("and"),
str8_lit_comp("and_eq"),
str8_lit_comp("asm"),
str8_lit_comp("atomic_cancel"),
str8_lit_comp("atomic_commit"),
str8_lit_comp("atomic_noexcept"),
str8_lit_comp("auto"),
str8_lit_comp("bitand"),
str8_lit_comp("bitor"),
str8_lit_comp("bool"),
str8_lit_comp("break"),
str8_lit_comp("case"),
str8_lit_comp("catch"),
str8_lit_comp("char"),
str8_lit_comp("char8_t"),
str8_lit_comp("char16_t"),
str8_lit_comp("char32_t"),
str8_lit_comp("class"),
str8_lit_comp("compl"),
str8_lit_comp("concept"),
str8_lit_comp("const"),
str8_lit_comp("consteval"),
str8_lit_comp("constexpr"),
str8_lit_comp("constinit"),
str8_lit_comp("const_cast"),
str8_lit_comp("continue"),
str8_lit_comp("co_await"),
str8_lit_comp("co_return"),
str8_lit_comp("co_yield"),
str8_lit_comp("decltype"),
str8_lit_comp("default"),
str8_lit_comp("delete"),
str8_lit_comp("do"),
str8_lit_comp("double"),
str8_lit_comp("dynamic_cast"),
str8_lit_comp("else"),
str8_lit_comp("enum"),
str8_lit_comp("explicit"),
str8_lit_comp("export"),
str8_lit_comp("extern"),
str8_lit_comp("false"),
str8_lit_comp("float"),
str8_lit_comp("for"),
str8_lit_comp("friend"),
str8_lit_comp("goto"),
str8_lit_comp("if"),
str8_lit_comp("inline"),
str8_lit_comp("int"),
str8_lit_comp("long"),
str8_lit_comp("mutable"),
str8_lit_comp("namespace"),
str8_lit_comp("new"),
str8_lit_comp("noexcept"),
str8_lit_comp("not"),
str8_lit_comp("not_eq"),
str8_lit_comp("nullptr"),
str8_lit_comp("operator"),
str8_lit_comp("or"),
str8_lit_comp("or_eq"),
str8_lit_comp("private"),
str8_lit_comp("protected"),
str8_lit_comp("public"),
str8_lit_comp("reflexpr"),
str8_lit_comp("register"),
str8_lit_comp("reinterpret_cast"),
str8_lit_comp("requires"),
str8_lit_comp("return"),
str8_lit_comp("short"),
str8_lit_comp("signed"),
str8_lit_comp("sizeof"),
str8_lit_comp("static"),
str8_lit_comp("static_assert"),
str8_lit_comp("static_cast"),
str8_lit_comp("struct"),
str8_lit_comp("switch"),
str8_lit_comp("synchronized"),
str8_lit_comp("template"),
str8_lit_comp("this"),
str8_lit_comp("thread_local"),
str8_lit_comp("throw"),
str8_lit_comp("true"),
str8_lit_comp("try"),
str8_lit_comp("typedef"),
str8_lit_comp("typeid"),
str8_lit_comp("typename"),
str8_lit_comp("union"),
str8_lit_comp("unsigned"),
str8_lit_comp("using"),
str8_lit_comp("virtual"),
str8_lit_comp("void"),
str8_lit_comp("volatile"),
str8_lit_comp("wchar_t"),
str8_lit_comp("while"),
str8_lit_comp("xor"),
str8_lit_comp("xor_eq"),
};
String8 token_string = str8_substr(string, r1u64(active_token_start_idx, idx+ender_pad));
for(U64 keyword_idx = 0; keyword_idx < ArrayCount(cpp_keywords); keyword_idx += 1)
{
if(str8_match(cpp_keywords[keyword_idx], token_string, 0))
{
token.kind = TXT_TokenKind_Keyword;
break;
}
}
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
}
// rjf: split symbols by maximum-munch-rule
else if(token.kind == TXT_TokenKind_Symbol)
{
read_only local_persist String8 c_cpp_multichar_symbol_strings[] =
{
str8_lit_comp("<<"),
str8_lit_comp(">>"),
str8_lit_comp("<="),
str8_lit_comp(">="),
str8_lit_comp("=="),
str8_lit_comp("!="),
str8_lit_comp("&&"),
str8_lit_comp("||"),
str8_lit_comp("|="),
str8_lit_comp("&="),
str8_lit_comp("^="),
str8_lit_comp("~="),
str8_lit_comp("+="),
str8_lit_comp("-="),
str8_lit_comp("*="),
str8_lit_comp("/="),
str8_lit_comp("%="),
str8_lit_comp("<<="),
str8_lit_comp(">>="),
str8_lit_comp("->"),
};
String8 token_string = str8_substr(string, r1u64(active_token_start_idx, idx+ender_pad));
for(U64 off = 0, next_off = token_string.size; off < token_string.size; off = next_off)
{
B32 found = 0;
for(U64 idx = 0; idx < ArrayCount(c_cpp_multichar_symbol_strings); idx += 1)
{
if(str8_match(str8_substr(token_string, r1u64(off, off+c_cpp_multichar_symbol_strings[idx].size)),
c_cpp_multichar_symbol_strings[idx],
0))
{
found = 1;
next_off = off + c_cpp_multichar_symbol_strings[idx].size;
TXT_Token token = {TXT_TokenKind_Symbol, r1u64(active_token_start_idx+off, active_token_start_idx+next_off)};
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
break;
}
}
if(!found)
{
next_off = off+1;
TXT_Token token = {TXT_TokenKind_Symbol, r1u64(active_token_start_idx+off, active_token_start_idx+next_off)};
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
}
}
}
// rjf: all other tokens
else
{
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
}
// rjf: increment by ender padding
idx += ender_pad;
}
// rjf: advance by 1 byte if we haven't found an ender
if(!ender_found)
{
idx += 1;
}
escaped = next_escaped;
}
}
//- rjf: token list -> token array
TXT_TokenArray result = txt_token_array_from_chunk_list(arena, &tokens);
scratch_end(scratch);
ProfEnd();
return result;
}
internal TXT_TokenArray
txt_token_array_from_string__odin(Arena *arena, U64 *bytes_processed_counter, String8 string)
{
Temp scratch = scratch_begin(&arena, 1);
//- rjf: generate token list
TXT_TokenChunkList tokens = {0};
{
B32 comment_is_single_line = 0;
B32 string_is_char = 0;
TXT_TokenKind active_token_kind = TXT_TokenKind_Null;
U64 active_token_start_idx = 0;
B32 escaped = 0;
B32 next_escaped = 0;
U64 byte_process_start_idx = 0;
for(U64 idx = 0; idx <= string.size;)
{
U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
// rjf: update counter
if(bytes_processed_counter != 0 && ((idx-byte_process_start_idx) >= 1000 || idx == string.size))
{
ins_atomic_u64_add_eval(bytes_processed_counter, (idx-byte_process_start_idx));
byte_process_start_idx = idx;
}
// rjf: escaping
if(escaped && (byte != '\r' && byte != '\n'))
{
next_escaped = 0;
}
else if(!escaped && byte == '\\')
{
next_escaped = 1;
}
// rjf: take starter, determine active token kind
if(active_token_kind == TXT_TokenKind_Null)
{
// rjf: use next bytes to start a new token
if(0){}
else if(char_is_space(byte)) { active_token_kind = TXT_TokenKind_Whitespace; }
else if(byte == '_' ||
byte == '$' ||
char_is_alpha(byte)) { active_token_kind = TXT_TokenKind_Identifier; }
else if(char_is_digit(byte, 10) ||
(byte == '.' &&
char_is_digit(next_byte, 10))) { active_token_kind = TXT_TokenKind_Numeric; }
else if(byte == '"') { active_token_kind = TXT_TokenKind_String; string_is_char = 0; }
else if(byte == '\'') { active_token_kind = TXT_TokenKind_String; string_is_char = 1; }
else if(byte == '/' && next_byte == '/') { active_token_kind = TXT_TokenKind_Comment; comment_is_single_line = 1; }
else if(byte == '/' && next_byte == '*') { active_token_kind = TXT_TokenKind_Comment; comment_is_single_line = 0; }
else if(byte == '~' || byte == '!' ||
byte == '%' || byte == '^' ||
byte == '&' || byte == '*' ||
byte == '(' || byte == ')' ||
byte == '-' || byte == '=' ||
byte == '+' || byte == '[' ||
byte == ']' || byte == '{' ||
byte == '}' || byte == ':' ||
byte == ';' || byte == ',' ||
byte == '.' || byte == '<' ||
byte == '>' || byte == '/' ||
byte == '?' || byte == '|') { active_token_kind = TXT_TokenKind_Symbol; }
else if(byte == '#') { active_token_kind = TXT_TokenKind_Meta; }
// rjf: start new token
if(active_token_kind != TXT_TokenKind_Null)
{
active_token_start_idx = idx;
}
// rjf: invalid token kind -> emit error
else
{
TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
}
// rjf: look for ender
U64 ender_pad = 0;
B32 ender_found = 0;
if(active_token_kind != TXT_TokenKind_Null && idx>active_token_start_idx)
{
if(idx == string.size)
{
ender_pad = 0;
ender_found = 1;
}
else switch(active_token_kind)
{
default:break;
case TXT_TokenKind_Whitespace:
{
ender_found = !char_is_space(byte);
}break;
case TXT_TokenKind_Identifier:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128);
}break;
case TXT_TokenKind_Numeric:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '.' && byte != '\'');
}break;
case TXT_TokenKind_String:
{
ender_found = (!escaped && ((!string_is_char && byte == '"') || (string_is_char && byte == '\'')));
ender_pad += 1;
}break;
case TXT_TokenKind_Symbol:
{
ender_found = (byte != '~' && byte != '!' &&
byte != '%' && byte != '^' &&
byte != '&' && byte != '*' &&
byte != '(' && byte != ')' &&
byte != '-' && byte != '=' &&
byte != '+' && byte != '[' &&
byte != ']' && byte != '{' &&
byte != '}' && byte != ':' &&
byte != ';' && byte != ',' &&
byte != '.' && byte != '<' &&
byte != '>' && byte != '/' &&
byte != '?' && byte != '|');
}break;
case TXT_TokenKind_Comment:
{
if(comment_is_single_line)
{
ender_found = (!escaped && (byte == '\r' || byte == '\n'));
}
else
{
ender_found = (active_token_start_idx+1 < idx && byte == '*' && next_byte == '/');
ender_pad += 2;
}
}break;
case TXT_TokenKind_Meta:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$');
}break;
}
}
// rjf: next byte is ender => emit token
if(ender_found)
{
TXT_Token token = {active_token_kind, r1u64(active_token_start_idx, idx+ender_pad)};
active_token_kind = TXT_TokenKind_Null;
// rjf: identifier -> keyword in special cases
if(token.kind == TXT_TokenKind_Identifier)
{
read_only local_persist String8 odin_keywords[] =
{
str8_lit_comp("align_of"),
str8_lit_comp("asm"),
str8_lit_comp("auto_cast"),
str8_lit_comp("bit_set"),
str8_lit_comp("break"),
str8_lit_comp("case"),
str8_lit_comp("cast"),
str8_lit_comp("context"),
str8_lit_comp("continue"),
str8_lit_comp("defer"),
str8_lit_comp("distinct"),
str8_lit_comp("do"),
str8_lit_comp("dynamic"),
str8_lit_comp("else"),
str8_lit_comp("enum"),
str8_lit_comp("fallthrough"),
str8_lit_comp("for"),
str8_lit_comp("foreign"),
str8_lit_comp("if"),
str8_lit_comp("in"),
str8_lit_comp("map"),
str8_lit_comp("matrix"),
str8_lit_comp("not_in"),
str8_lit_comp("or_break"),
str8_lit_comp("or_continue"),
str8_lit_comp("or_else"),
str8_lit_comp("or_return"),
str8_lit_comp("package"),
str8_lit_comp("proc"),
str8_lit_comp("return"),
str8_lit_comp("size_of"),
str8_lit_comp("struct"),
str8_lit_comp("switch"),
str8_lit_comp("transmute"),
str8_lit_comp("typeid"),
str8_lit_comp("union"),
str8_lit_comp("using"),
str8_lit_comp("when"),
str8_lit_comp("where"),
str8_lit_comp("import"),
};
String8 token_string = str8_substr(string, r1u64(active_token_start_idx, idx+ender_pad));
for(U64 keyword_idx = 0; keyword_idx < ArrayCount(odin_keywords); keyword_idx += 1)
{
if(str8_match(odin_keywords[keyword_idx], token_string, 0))
{
token.kind = TXT_TokenKind_Keyword;
break;
}
}
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
// rjf: split symbols by maximum-munch-rule
else if(token.kind == TXT_TokenKind_Symbol)
{
read_only local_persist String8 odin_multichar_symbol_strings[] =
{
str8_lit_comp("<<"),
str8_lit_comp(">>"),
str8_lit_comp("<="),
str8_lit_comp(">="),
str8_lit_comp("=="),
str8_lit_comp("!="),
str8_lit_comp("&&"),
str8_lit_comp("||"),
str8_lit_comp("|="),
str8_lit_comp("&="),
str8_lit_comp("^="),
str8_lit_comp("~="),
str8_lit_comp("+="),
str8_lit_comp("-="),
str8_lit_comp("*="),
str8_lit_comp("/="),
str8_lit_comp("%="),
str8_lit_comp("<<="),
str8_lit_comp(">>="),
str8_lit_comp("->"),
};
String8 token_string = str8_substr(string, r1u64(active_token_start_idx, idx+ender_pad));
for(U64 off = 0, next_off = token_string.size; off < token_string.size; off = next_off)
{
B32 found = 0;
for(U64 idx = 0; idx < ArrayCount(odin_multichar_symbol_strings); idx += 1)
{
if(str8_match(str8_substr(token_string, r1u64(off, off+odin_multichar_symbol_strings[idx].size)),
odin_multichar_symbol_strings[idx],
0))
{
found = 1;
next_off = off + odin_multichar_symbol_strings[idx].size;
TXT_Token token = {TXT_TokenKind_Symbol, r1u64(active_token_start_idx+off, active_token_start_idx+next_off)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
break;
}
}
if(!found)
{
next_off = off+1;
TXT_Token token = {TXT_TokenKind_Symbol, r1u64(active_token_start_idx+off, active_token_start_idx+next_off)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
}
}
// rjf: all other tokens
else
{
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
// rjf: increment by ender padding
idx += ender_pad;
}
// rjf: advance by 1 byte if we haven't found an ender
if(!ender_found)
{
idx += 1;
}
escaped = next_escaped;
}
}
//- rjf: token list -> token array
TXT_TokenArray result = txt_token_array_from_chunk_list(arena, &tokens);
scratch_end(scratch);
return result;
}
internal TXT_TokenArray
txt_token_array_from_string__jai(Arena *arena, U64 *bytes_processed_counter, String8 string)
{
Temp scratch = scratch_begin(&arena, 1);
//- rjf: generate token list
TXT_TokenChunkList tokens = {0};
{
B32 comment_is_single_line = 0;
B32 string_is_char = 0;
TXT_TokenKind active_token_kind = TXT_TokenKind_Null;
U64 active_token_start_idx = 0;
B32 escaped = 0;
B32 next_escaped = 0;
U64 byte_process_start_idx = 0;
for(U64 idx = 0; idx <= string.size;)
{
U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
// rjf: update counter
if(bytes_processed_counter != 0 && ((idx-byte_process_start_idx) >= 1000 || idx == string.size))
{
ins_atomic_u64_add_eval(bytes_processed_counter, (idx-byte_process_start_idx));
byte_process_start_idx = idx;
}
// rjf: escaping
if(escaped && (byte != '\r' && byte != '\n'))
{
next_escaped = 0;
}
else if(!escaped && byte == '\\')
{
next_escaped = 1;
}
// rjf: take starter, determine active token kind
if(active_token_kind == TXT_TokenKind_Null)
{
// rjf: use next bytes to start a new token
if(0){}
else if(char_is_space(byte)) { active_token_kind = TXT_TokenKind_Whitespace; }
else if(byte == '_' ||
byte == '$' ||
char_is_alpha(byte)) { active_token_kind = TXT_TokenKind_Identifier; }
else if(char_is_digit(byte, 10) ||
(byte == '.' &&
char_is_digit(next_byte, 10))) { active_token_kind = TXT_TokenKind_Numeric; }
else if(byte == '"') { active_token_kind = TXT_TokenKind_String; string_is_char = 0; }
else if(byte == '\'') { active_token_kind = TXT_TokenKind_String; string_is_char = 1; }
else if(byte == '/' && next_byte == '/') { active_token_kind = TXT_TokenKind_Comment; comment_is_single_line = 1; }
else if(byte == '/' && next_byte == '*') { active_token_kind = TXT_TokenKind_Comment; comment_is_single_line = 0; }
else if(byte == '~' || byte == '!' ||
byte == '%' || byte == '^' ||
byte == '&' || byte == '*' ||
byte == '(' || byte == ')' ||
byte == '-' || byte == '=' ||
byte == '+' || byte == '[' ||
byte == ']' || byte == '{' ||
byte == '}' || byte == ':' ||
byte == ';' || byte == ',' ||
byte == '.' || byte == '<' ||
byte == '>' || byte == '/' ||
byte == '?' || byte == '|') { active_token_kind = TXT_TokenKind_Symbol; }
else if(byte == '#') { active_token_kind = TXT_TokenKind_Meta; }
// rjf: start new token
if(active_token_kind != TXT_TokenKind_Null)
{
active_token_start_idx = idx;
}
// rjf: invalid token kind -> emit error
else
{
TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
}
// rjf: look for ender
U64 ender_pad = 0;
B32 ender_found = 0;
if(active_token_kind != TXT_TokenKind_Null && idx>active_token_start_idx)
{
if(idx == string.size)
{
ender_pad = 0;
ender_found = 1;
}
else switch(active_token_kind)
{
default:break;
case TXT_TokenKind_Whitespace:
{
ender_found = !char_is_space(byte);
}break;
case TXT_TokenKind_Identifier:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128);
}break;
case TXT_TokenKind_Numeric:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '.' && byte != '\'');
}break;
case TXT_TokenKind_String:
{
ender_found = (!escaped && ((!string_is_char && byte == '"') || (string_is_char && byte == '\'')));
ender_pad += 1;
}break;
case TXT_TokenKind_Symbol:
{
ender_found = (byte != '~' && byte != '!' &&
byte != '%' && byte != '^' &&
byte != '&' && byte != '*' &&
byte != '(' && byte != ')' &&
byte != '-' && byte != '=' &&
byte != '+' && byte != '[' &&
byte != ']' && byte != '{' &&
byte != '}' && byte != ':' &&
byte != ';' && byte != ',' &&
byte != '.' && byte != '<' &&
byte != '>' && byte != '/' &&
byte != '?' && byte != '|');
}break;
case TXT_TokenKind_Comment:
{
if(comment_is_single_line)
{
ender_found = (!escaped && (byte == '\r' || byte == '\n'));
}
else
{
ender_found = (active_token_start_idx+1 < idx && byte == '*' && next_byte == '/');
ender_pad += 2;
}
}break;
case TXT_TokenKind_Meta:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$');
}break;
}
}
// rjf: next byte is ender => emit token
if(ender_found)
{
TXT_Token token = {active_token_kind, r1u64(active_token_start_idx, idx+ender_pad)};
active_token_kind = TXT_TokenKind_Null;
// rjf: identifier -> keyword in special cases
if(token.kind == TXT_TokenKind_Identifier)
{
read_only local_persist String8 jai_keywords[] =
{
str8_lit_comp("bool"),
str8_lit_comp("true"),
str8_lit_comp("false"),
str8_lit_comp("int"),
str8_lit_comp("s8"),
str8_lit_comp("u8"),
str8_lit_comp("s16"),
str8_lit_comp("u16"),
str8_lit_comp("s32"),
str8_lit_comp("u32"),
str8_lit_comp("s64"),
str8_lit_comp("u64"),
str8_lit_comp("s128"),
str8_lit_comp("u128"),
str8_lit_comp("float"),
str8_lit_comp("float32"),
str8_lit_comp("float64"),
str8_lit_comp("void"),
str8_lit_comp("enum"),
str8_lit_comp("enum_flags"),
str8_lit_comp("size_of"),
str8_lit_comp("string"),
str8_lit_comp("type_of"),
str8_lit_comp("cast"),
str8_lit_comp("if"),
str8_lit_comp("ifs"),
str8_lit_comp("then"),
str8_lit_comp("else"),
str8_lit_comp("case"),
str8_lit_comp("for"),
str8_lit_comp("while"),
str8_lit_comp("break"),
str8_lit_comp("continue"),
str8_lit_comp("remove"),
str8_lit_comp("return"),
str8_lit_comp("inline"),
str8_lit_comp("null"),
str8_lit_comp("defer"),
str8_lit_comp("xx"),
};
String8 token_string = str8_substr(string, r1u64(active_token_start_idx, idx+ender_pad));
for(U64 keyword_idx = 0; keyword_idx < ArrayCount(jai_keywords); keyword_idx += 1)
{
if(str8_match(jai_keywords[keyword_idx], token_string, 0))
{
token.kind = TXT_TokenKind_Keyword;
break;
}
}
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
// rjf: split symbols by maximum-munch-rule
else if(token.kind == TXT_TokenKind_Symbol)
{
read_only local_persist String8 jai_multichar_symbol_strings[] =
{
str8_lit_comp("<<"),
str8_lit_comp(">>"),
str8_lit_comp("<="),
str8_lit_comp(">="),
str8_lit_comp("=="),
str8_lit_comp("!="),
str8_lit_comp("&&"),
str8_lit_comp("||"),
str8_lit_comp("|="),
str8_lit_comp("&="),
str8_lit_comp("^="),
str8_lit_comp("~="),
str8_lit_comp("+="),
str8_lit_comp("-="),
str8_lit_comp("*="),
str8_lit_comp("/="),
str8_lit_comp("%="),
str8_lit_comp("<<="),
str8_lit_comp(">>="),
str8_lit_comp("->"),
};
String8 token_string = str8_substr(string, r1u64(active_token_start_idx, idx+ender_pad));
for(U64 off = 0, next_off = token_string.size; off < token_string.size; off = next_off)
{
B32 found = 0;
for(U64 idx = 0; idx < ArrayCount(jai_multichar_symbol_strings); idx += 1)
{
if(str8_match(str8_substr(token_string, r1u64(off, off+jai_multichar_symbol_strings[idx].size)),
jai_multichar_symbol_strings[idx],
0))
{
found = 1;
next_off = off + jai_multichar_symbol_strings[idx].size;
TXT_Token token = {TXT_TokenKind_Symbol, r1u64(active_token_start_idx+off, active_token_start_idx+next_off)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
break;
}
}
if(!found)
{
next_off = off+1;
TXT_Token token = {TXT_TokenKind_Symbol, r1u64(active_token_start_idx+off, active_token_start_idx+next_off)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
}
}
// rjf: all other tokens
else
{
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
// rjf: increment by ender padding
idx += ender_pad;
}
// rjf: advance by 1 byte if we haven't found an ender
if(!ender_found)
{
idx += 1;
}
escaped = next_escaped;
}
}
//- rjf: token list -> token array
TXT_TokenArray result = txt_token_array_from_chunk_list(arena, &tokens);
scratch_end(scratch);
return result;
}
internal TXT_TokenArray
txt_token_array_from_string__zig(Arena *arena, U64 *bytes_processed_counter, String8 string)
{
Temp scratch = scratch_begin(&arena, 1);
//- rjf: generate token list
TXT_TokenChunkList tokens = {0};
{
B32 string_is_char = 0;
B32 string_is_line = 0;
TXT_TokenKind active_token_kind = TXT_TokenKind_Null;
U64 active_token_start_idx = 0;
B32 escaped = 0;
B32 next_escaped = 0;
U64 byte_process_start_idx = 0;
for(U64 idx = 0; idx <= string.size;)
{
U8 byte = (idx+0 < string.size) ? (string.str[idx+0]) : 0;
U8 next_byte = (idx+1 < string.size) ? (string.str[idx+1]) : 0;
// rjf: update counter
if(bytes_processed_counter != 0 && ((idx-byte_process_start_idx) >= 1000 || idx == string.size))
{
ins_atomic_u64_add_eval(bytes_processed_counter, (idx-byte_process_start_idx));
byte_process_start_idx = idx;
}
// rjf: escaping
if(escaped && (byte != '\r' && byte != '\n'))
{
next_escaped = 0;
}
else if(!escaped && byte == '\\')
{
next_escaped = 1;
}
// rjf: take starter, determine active token kind
if(active_token_kind == TXT_TokenKind_Null)
{
// rjf: use next bytes to start a new token
if(0){}
else if(char_is_space(byte)) { active_token_kind = TXT_TokenKind_Whitespace; }
else if(byte == '_' ||
char_is_alpha(byte)) { active_token_kind = TXT_TokenKind_Identifier; }
else if(char_is_digit(byte, 10) ||
(byte == '.' &&
char_is_digit(next_byte, 10))) { active_token_kind = TXT_TokenKind_Numeric; }
else if(byte == '"') { active_token_kind = TXT_TokenKind_String; string_is_char = 0; }
else if(byte == '\'') { active_token_kind = TXT_TokenKind_String; string_is_char = 1; }
else if(byte == '\\' &&
next_byte == '\\') { active_token_kind = TXT_TokenKind_String; string_is_line = 1; }
else if(byte == '/' && next_byte == '/') { active_token_kind = TXT_TokenKind_Comment; }
else if(byte == '~' || byte == '!' ||
byte == '%' || byte == '^' ||
byte == '&' || byte == '*' ||
byte == '(' || byte == ')' ||
byte == '-' || byte == '=' ||
byte == '+' || byte == '[' ||
byte == ']' || byte == '{' ||
byte == '}' || byte == ':' ||
byte == ';' || byte == ',' ||
byte == '.' || byte == '<' ||
byte == '>' || byte == '/' ||
byte == '?' || byte == '|' ||
byte == 'c') { active_token_kind = TXT_TokenKind_Symbol; }
// rjf: start new token
if(active_token_kind != TXT_TokenKind_Null)
{
active_token_start_idx = idx;
}
// rjf: invalid token kind -> emit error
else
{
TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
}
// rjf: look for ender
U64 ender_pad = 0;
B32 ender_found = 0;
if(active_token_kind != TXT_TokenKind_Null && idx>active_token_start_idx)
{
if(idx == string.size)
{
ender_pad = 0;
ender_found = 1;
}
else switch(active_token_kind)
{
default:break;
case TXT_TokenKind_Whitespace:
{
ender_found = !char_is_space(byte);
}break;
case TXT_TokenKind_Identifier:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128);
}break;
case TXT_TokenKind_Numeric:
{
ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '.' && byte != '\'');
}break;
case TXT_TokenKind_String:
{
if (string_is_line)
{
ender_found = (!escaped && (byte == '\r' || byte == '\n'));
}
else
{
ender_found = (!escaped && ((!string_is_char && byte == '"') || (string_is_char && byte == '\'')));
ender_pad += 1;
}
}break;
case TXT_TokenKind_Symbol:
{
ender_found = (byte != '~' && byte != '!' &&
byte != '%' && byte != '^' &&
byte != '&' && byte != '*' &&
byte != '(' && byte != ')' &&
byte != '-' && byte != '=' &&
byte != '+' && byte != '[' &&
byte != ']' && byte != '{' &&
byte != '}' && byte != ':' &&
byte != ';' && byte != ',' &&
byte != '.' && byte != '<' &&
byte != '>' && byte != '/' &&
byte != '?' && byte != '|' &&
byte != 'c');
}break;
case TXT_TokenKind_Comment:
{
ender_found = (!escaped && (byte == '\r' || byte == '\n'));
}break;
}
}
// rjf: next byte is ender => emit token
if(ender_found)
{
TXT_Token token = {active_token_kind, r1u64(active_token_start_idx, idx+ender_pad)};
active_token_kind = TXT_TokenKind_Null;
// rjf: identifier -> keyword in special cases
if(token.kind == TXT_TokenKind_Identifier)
{
read_only local_persist String8 zig_keywords[] =
{
str8_lit_comp("addrspace"),
str8_lit_comp("align"),
str8_lit_comp("allowzero"),
str8_lit_comp("and"),
str8_lit_comp("anyframe"),
str8_lit_comp("anytype"),
str8_lit_comp("asm"),
str8_lit_comp("async"),
str8_lit_comp("await"),
str8_lit_comp("break"),
str8_lit_comp("callconv"),
str8_lit_comp("catch"),
str8_lit_comp("comptime"),
str8_lit_comp("const"),
str8_lit_comp("continue"),
str8_lit_comp("defer"),
str8_lit_comp("else"),
str8_lit_comp("enum"),
str8_lit_comp("errdefer"),
str8_lit_comp("error"),
str8_lit_comp("export"),
str8_lit_comp("extern"),
str8_lit_comp("fn"),
str8_lit_comp("for"),
str8_lit_comp("if"),
str8_lit_comp("inline"),
str8_lit_comp("noalias"),
str8_lit_comp("nosuspend"),
str8_lit_comp("noinline"),
str8_lit_comp("opaque"),
str8_lit_comp("or"),
str8_lit_comp("orelse"),
str8_lit_comp("packed"),
str8_lit_comp("pub"),
str8_lit_comp("resume"),
str8_lit_comp("return"),
str8_lit_comp("linksection"),
str8_lit_comp("struct"),
str8_lit_comp("suspend"),
str8_lit_comp("switch"),
str8_lit_comp("test"),
str8_lit_comp("threadlocal"),
str8_lit_comp("try"),
str8_lit_comp("union"),
str8_lit_comp("unreachable"),
str8_lit_comp("usingnamespace"),
str8_lit_comp("var"),
str8_lit_comp("volatile"),
str8_lit_comp("while"),
};
String8 token_string = str8_substr(string, r1u64(active_token_start_idx, idx+ender_pad));
for(U64 keyword_idx = 0; keyword_idx < ArrayCount(zig_keywords); keyword_idx += 1)
{
if(str8_match(zig_keywords[keyword_idx], token_string, 0))
{
token.kind = TXT_TokenKind_Keyword;
break;
}
}
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
// rjf: split symbols by maximum-munch-rule
else if(token.kind == TXT_TokenKind_Symbol)
{
read_only local_persist String8 zig_multichar_symbol_strings[] =
{
str8_lit_comp("<<"),
str8_lit_comp(">>"),
str8_lit_comp("<="),
str8_lit_comp(">="),
str8_lit_comp("=="),
str8_lit_comp("!="),
str8_lit_comp("&&"),
str8_lit_comp("||"),
str8_lit_comp("|="),
str8_lit_comp("&="),
str8_lit_comp("^="),
str8_lit_comp("~="),
str8_lit_comp("+="),
str8_lit_comp("-="),
str8_lit_comp("*="),
str8_lit_comp("/="),
str8_lit_comp("%="),
str8_lit_comp("<<="),
str8_lit_comp(">>="),
str8_lit_comp("->"),
};
String8 token_string = str8_substr(string, r1u64(active_token_start_idx, idx+ender_pad));
for(U64 off = 0, next_off = token_string.size; off < token_string.size; off = next_off)
{
B32 found = 0;
for(U64 idx = 0; idx < ArrayCount(zig_multichar_symbol_strings); idx += 1)
{
if(str8_match(str8_substr(token_string, r1u64(off, off+zig_multichar_symbol_strings[idx].size)),
zig_multichar_symbol_strings[idx],
0))
{
found = 1;
next_off = off + zig_multichar_symbol_strings[idx].size;
TXT_Token token = {TXT_TokenKind_Symbol, r1u64(active_token_start_idx+off, active_token_start_idx+next_off)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
break;
}
}
if(!found)
{
next_off = off+1;
TXT_Token token = {TXT_TokenKind_Symbol, r1u64(active_token_start_idx+off, active_token_start_idx+next_off)};
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
}
}
// rjf: all other tokens
else
{
txt_token_chunk_list_push(scratch.arena, &tokens, 4096, &token);
}
// rjf: increment by ender padding
idx += ender_pad;
}
// rjf: advance by 1 byte if we haven't found an ender
if(!ender_found)
{
idx += 1;
}
escaped = next_escaped;
}
}
//- rjf: token list -> token array
TXT_TokenArray result = txt_token_array_from_chunk_list(arena, &tokens);
scratch_end(scratch);
return result;
}
internal TXT_TokenArray
txt_token_array_from_string__disasm_x64_intel(Arena *arena, U64 *bytes_processed_counter, String8 string)
{
Temp scratch = scratch_begin(&arena, 1);
//- rjf: parse tokens
TXT_TokenChunkList tokens = {0};
{
TXT_TokenKind active_token_kind = TXT_TokenKind_Null;
U64 active_token_start_off = 0;
U64 off = 0;
B32 escaped = 0;
B32 string_is_char = 0;
S32 brace_nest = 0;
S32 paren_nest = 0;
S32 string_tick_nest = 0;
for(U64 advance = 0; off <= string.size; off += advance)
{
U8 byte = (off+0 < string.size) ? string.str[off+0] : 0;
U8 next_byte = (off+1 < string.size) ? string.str[off+1] : 0;
B32 ender_found = 0;
advance = (active_token_kind != TXT_TokenKind_Null ? 1 : 0);
if(off == string.size && active_token_kind != TXT_TokenKind_Null)
{
ender_found = 1;
advance = 1;
}
switch(active_token_kind)
{
default:
case TXT_TokenKind_Null:
{
if(byte == ' ' || byte == '\t' || byte == '\v' || byte == '\f' || byte == '\r' || byte == '\n')
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_Whitespace;
advance = 1;
}
else if(byte == '>' && brace_nest == 0 && paren_nest == 0)
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_Comment;
advance = 1;
}
else if(('a' <= byte && byte <= 'z') || ('A' <= byte && byte <= 'Z') || byte == '_')
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_Keyword;
advance = 1;
}
else if(byte == '\'')
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_String;
advance = 1;
string_is_char = 1;
}
else if(byte == '"')
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_String;
advance = 1;
string_is_char = 0;
}
else if(byte == '`')
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_String;
advance = 1;
string_tick_nest += 1;
}
else if(('0' <= byte && byte <= '9') || (byte == '.' && '0' <= next_byte && next_byte <= '9'))
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_Numeric;
advance = 1;
}
else if(byte == '~' || byte == '!' || byte == '%' || byte == '^' ||
byte == '&' || byte == '*' || byte == '(' || byte == ')' ||
byte == '-' || byte == '=' || byte == '+' || byte == '[' ||
byte == ']' || byte == '{' || byte == '}' || byte == ';' ||
byte == ':' || byte == '?' || byte == '/' || byte == '<' ||
byte == '>' || byte == ',' || byte == '.')
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_Symbol;
advance = 1;
if(byte == '{')
{
brace_nest += 1;
}
else if(byte == '}')
{
brace_nest -= 1;
}
if(byte == '(')
{
paren_nest += 1;
}
else if(byte == ')')
{
paren_nest -= 1;
}
}
else
{
active_token_start_off = off;
active_token_kind = TXT_TokenKind_Error;
advance = 1;
}
}break;
case TXT_TokenKind_Whitespace:
if(byte != ' ' && byte != '\t' && byte != '\v' && byte != '\f')
{
ender_found = 1;
advance = 0;
}break;
case TXT_TokenKind_Keyword:
if((byte < 'a' || 'z' < byte) && (byte < 'A' || 'Z' < byte) && (byte < '0' || '9' < byte) && byte != '_')
{
ender_found = 1;
advance = 0;
}break;
case TXT_TokenKind_String:
{
U8 ender_byte = (string_tick_nest > 0 ? '\'' :
string_is_char ? '\''
: '"');
if(!escaped && byte == ender_byte)
{
if(string_tick_nest > 0)
{
string_tick_nest -= 1;
}
if(string_tick_nest == 0)
{
ender_found = 1;
advance = 1;
}
}
else if(escaped)
{
escaped = 0;
advance = 1;
}
else if(byte == '\\')
{
escaped = 1;
advance = 1;
}
else if(string_tick_nest > 0 && byte == '`')
{
string_tick_nest += 1;
}
else
{
U8 byte_class = utf8_class[byte>>3];
if(byte_class > 1)
{
advance = (U64)byte_class;
}
}
}break;
case TXT_TokenKind_Numeric:
if((byte < 'a' || 'z' < byte) && (byte < 'A' || 'Z' < byte) && (byte < '0' || '9' < byte) && byte != '.')
{
ender_found = 1;
advance = 0;
}break;
case TXT_TokenKind_Symbol:
if(1)
{
// NOTE(rjf): avoiding maximum munch rule for now
ender_found = 1;
advance = 0;
}
else if(byte != '~' && byte != '!' && byte != '#' && byte != '%' &&
byte != '^' && byte != '&' && byte != '*' && byte != '(' &&
byte != ')' && byte != '-' && byte != '=' && byte != '+' &&
byte != '[' && byte != ']' && byte != '{' && byte != '}' &&
byte != ';' && byte != ':' && byte != '?' && byte != '/' &&
byte != '<' && byte != '>' && byte != ',' && byte != '.')
{
ender_found = 1;
advance = 0;
}break;
case TXT_TokenKind_Error:
{
ender_found = 1;
advance = 0;
}break;
case TXT_TokenKind_Comment:
if(byte == '\n')
{
ender_found = 1;
advance = 1;
}break;
}
if(ender_found != 0)
{
if(brace_nest != 0 && active_token_kind == TXT_TokenKind_Keyword)
{
active_token_kind = TXT_TokenKind_Numeric;
}
if(paren_nest != 0 && active_token_kind == TXT_TokenKind_Keyword)
{
active_token_kind = TXT_TokenKind_Identifier;
}
TXT_Token token = {active_token_kind, r1u64(active_token_start_off, off+advance)};
txt_token_chunk_list_push(arena, &tokens, 1024, &token);
active_token_kind = TXT_TokenKind_Null;
active_token_start_off = token.range.max;
}
}
}
//- rjf: token list -> token array
TXT_TokenArray result = txt_token_array_from_chunk_list(arena, &tokens);
scratch_end(scratch);
return result;
}
////////////////////////////////
//~ rjf: Text Info Extractor Helpers
internal U64
txt_off_from_info_pt(TXT_TextInfo *info, TxtPt pt)
{
U64 off = 0;
if(1 <= pt.line && pt.line <= info->lines_count)
{
Rng1U64 line_range = info->lines_ranges[pt.line-1];
off = line_range.min + (pt.column-1);
}
return off;
}
internal TxtPt
txt_pt_from_info_off__linear_scan(TXT_TextInfo *info, U64 off)
{
TxtPt pt = {0};
{
for(U64 line_idx = 0; line_idx < info->lines_count; line_idx += 1)
{
if(contains_1u64(info->lines_ranges[line_idx], off))
{
pt.line = (S64)line_idx + 1;
pt.column = (S64)(off - info->lines_ranges[line_idx].min) + 1;
}
}
}
return pt;
}
internal TXT_TokenArray
txt_token_array_from_info_line_num__linear_scan(TXT_TextInfo *info, S64 line_num)
{
TXT_TokenArray line_tokens = {0};
if(1 <= line_num && line_num <= info->lines_count)
{
Rng1U64 line_range = info->lines_ranges[line_num-1];
for(U64 token_idx = 0; token_idx < info->tokens.count; token_idx += 1)
{
Rng1U64 token_range = info->tokens.v[token_idx].range;
Rng1U64 token_x_line = intersect_1u64(token_range, line_range);
if(token_x_line.max > token_x_line.min)
{
if(line_tokens.v == 0)
{
line_tokens.v = info->tokens.v+token_idx;
}
line_tokens.count += 1;
}
else if(line_tokens.v != 0)
{
break;
}
}
}
return line_tokens;
}
internal Rng1U64
txt_expr_off_range_from_line_off_range_string_tokens(U64 off, Rng1U64 line_range, String8 line_text, TXT_TokenArray *line_tokens)
{
Rng1U64 result = {0};
Temp scratch = scratch_begin(0, 0);
{
// rjf: unpack line info
TXT_Token *line_tokens_first = line_tokens->v;
TXT_Token *line_tokens_opl = line_tokens->v+line_tokens->count;
// rjf: find token containing `off`
TXT_Token *pt_token = 0;
for(TXT_Token *token = line_tokens_first;
token < line_tokens_opl;
token += 1)
{
if(contains_1u64(token->range, off))
{
Rng1U64 token_range_clamped = intersect_1u64(line_range, token->range);
String8 token_string = str8_substr(line_text, r1u64(token_range_clamped.max - line_range.min, token_range_clamped.max - line_range.min));
B32 token_ender = 0;
switch(token->kind)
{
default:{}break;
case TXT_TokenKind_Symbol:
{
token_ender = (str8_match(token_string, str8_lit("]"), 0));
}break;
case TXT_TokenKind_Identifier:
case TXT_TokenKind_Keyword:
case TXT_TokenKind_Meta:
{
token_ender = 1;
}break;
}
if(token_ender)
{
pt_token = token;
}
break;
}
}
// rjf: found token containing `off`? -> mark that as our initial range
if(pt_token != 0)
{
result = pt_token->range;
}
// rjf: walk back from pt_token - try to find plausible start of expression
if(pt_token != 0)
{
B32 walkback_done = 0;
S32 nest = 0;
for(TXT_Token *wb_token = pt_token;
wb_token >= line_tokens_first && walkback_done == 0;
wb_token -= 1)
{
Rng1U64 wb_token_range_clamped = intersect_1u64(line_range, wb_token->range);
String8 wb_token_string = str8_substr(line_text, r1u64(wb_token_range_clamped.min - line_range.min, wb_token_range_clamped.max - line_range.min));
B32 include_wb_token = 0;
switch(wb_token->kind)
{
default:{}break;
case TXT_TokenKind_Symbol:
{
B32 is_scope_resolution = str8_match(wb_token_string, str8_lit("::"), 0);
B32 is_dot = str8_match(wb_token_string, str8_lit("."), 0);
B32 is_arrow = str8_match(wb_token_string, str8_lit("->"), 0);
B32 is_open_bracket = str8_match(wb_token_string, str8_lit("["), 0);
B32 is_close_bracket = str8_match(wb_token_string, str8_lit("]"), 0);
nest -= !!(is_open_bracket);
nest += !!(is_close_bracket);
if(is_scope_resolution ||
is_dot ||
is_arrow ||
is_open_bracket||
is_close_bracket)
{
include_wb_token = 1;
}
}break;
case TXT_TokenKind_Identifier:
{
include_wb_token = 1;
}break;
}
if(include_wb_token)
{
result = union_1u64(result, wb_token->range);
}
else if(nest == 0)
{
walkback_done = 1;
}
}
}
}
scratch_end(scratch);
return result;
}
internal Rng1U64
txt_expr_off_range_from_info_data_pt(TXT_TextInfo *info, String8 data, TxtPt pt)
{
Rng1U64 result = {0};
Temp scratch = scratch_begin(0, 0);
if(1 <= pt.line && pt.line <= info->lines_count)
{
// rjf: unpack line info
Rng1U64 line_range = info->lines_ranges[pt.line-1];
String8 line_text = str8_substr(data, line_range);
TXT_LineTokensSlice line_tokens_slice = txt_line_tokens_slice_from_info_data_line_range(scratch.arena, info, data, r1s64(pt.line, pt.line));
TXT_TokenArray line_tokens = line_tokens_slice.line_tokens[0];
TXT_Token *line_tokens_first = line_tokens.v;
TXT_Token *line_tokens_opl = line_tokens.v+line_tokens.count;
U64 pt_off = line_range.min + (pt.column-1);
// rjf: grab offset range of expression
result = txt_expr_off_range_from_line_off_range_string_tokens(pt_off, line_range, line_text, &line_tokens);
}
scratch_end(scratch);
return result;
}
internal String8
txt_string_from_info_data_txt_rng(TXT_TextInfo *info, String8 data, TxtRng rng)
{
Rng1U64 rng_off = r1u64(txt_off_from_info_pt(info, rng.min), txt_off_from_info_pt(info, rng.max));
String8 result = str8_substr(data, rng_off);
return result;
}
internal String8
txt_string_from_info_data_line_num(TXT_TextInfo *info, String8 data, S64 line_num)
{
String8 result = {0};
if(1 <= line_num && line_num <= info->lines_count)
{
result = str8_substr(data, info->lines_ranges[line_num-1]);
}
return result;
}
internal TXT_LineTokensSlice
txt_line_tokens_slice_from_info_data_line_range(Arena *arena, TXT_TextInfo *info, String8 data, Rng1S64 line_range)
{
TXT_LineTokensSlice result = {0};
Temp scratch = scratch_begin(&arena, 1);
if(info->lines_count != 0)
{
Rng1S64 line_range_clamped = r1s64(Clamp(1, line_range.min, (S64)info->lines_count), Clamp(1, line_range.max, (S64)info->lines_count));
U64 line_count = (U64)dim_1s64(line_range_clamped)+1;
// rjf: allocate output arrays
result.line_tokens = push_array(arena, TXT_TokenArray, line_count);
// rjf: binary search to find first token
TXT_Token *tokens_first = 0;
ProfScope("binary search to find first token")
{
Rng1U64 slice_range = r1u64(info->lines_ranges[line_range_clamped.min-1].min, info->lines_ranges[line_range_clamped.max-1].max);
U64 min_idx = 0;
U64 opl_idx = info->tokens.count;
for(;;)
{
U64 mid_idx = (opl_idx+min_idx)/2;
if(mid_idx >= opl_idx)
{
break;
}
TXT_Token *mid_token = &info->tokens.v[mid_idx];
if(mid_token->range.min > slice_range.max)
{
opl_idx = mid_idx;
}
else if(mid_token->range.max < slice_range.min)
{
min_idx = mid_idx;
}
else if(tokens_first == 0 || mid_token->range.min < tokens_first->range.min)
{
tokens_first = mid_token;
opl_idx = mid_idx;
}
if(mid_idx == min_idx && mid_idx+1 == opl_idx)
{
break;
}
}
}
// rjf: grab per-line tokens
TXT_TokenList *line_tokens_lists = push_array(scratch.arena, TXT_TokenList, line_count);
if(tokens_first != 0) ProfScope("grab per-line tokens")
{
TXT_Token *tokens_opl = info->tokens.v+info->tokens.count;
U64 line_slice_idx = 0;
for(TXT_Token *token = tokens_first; token < tokens_opl && line_slice_idx < line_count;)
{
if(token->range.min < info->lines_ranges[line_slice_idx+line_range.min-1].max)
{
if(token->range.max > info->lines_ranges[line_slice_idx+line_range.min-1].min)
{
txt_token_list_push(scratch.arena, &line_tokens_lists[line_slice_idx], token);
}
B32 need_token_advance = 0;
B32 need_line_advance = 0;
if(token->range.max >= info->lines_ranges[line_slice_idx+line_range.min-1].max)
{
need_line_advance = 1;
}
if(token->range.max <= info->lines_ranges[line_slice_idx+line_range.min-1].max)
{
need_token_advance += 1;
}
if(need_line_advance) { line_slice_idx += 1; }
if(need_token_advance) { token += 1; }
}
else
{
line_slice_idx += 1;
}
}
}
// rjf: bake per-line tokens to arrays
for(U64 line_slice_idx = 0; line_slice_idx < line_count; line_slice_idx += 1)
{
result.line_tokens[line_slice_idx] = txt_token_array_from_list(arena, &line_tokens_lists[line_slice_idx]);
}
}
scratch_end(scratch);
return result;
}
internal TXT_ScopeNode *
txt_scope_node_from_info_num(TXT_TextInfo *info, U64 num)
{
TXT_ScopeNode *result = &txt_scope_node_nil;
if(1 <= num && num <= info->scope_nodes.count)
{
result = &info->scope_nodes.v[num-1];
}
return result;
}
internal TXT_ScopeNode *
txt_scope_node_from_info_off(TXT_TextInfo *info, U64 off)
{
TXT_ScopeNode *result = &txt_scope_node_nil;
if(info->scope_pts.count != 0)
{
U64 first = 0;
U64 opl = info->scope_pts.count;
for(;;)
{
U64 mid = (first + opl) / 2;
U64 mid_off = info->tokens.v[info->scope_pts.v[mid].token_idx].range.min;
if(mid_off < off)
{
first = mid;
}
else if(off < mid_off)
{
opl = mid;
}
else
{
first = mid;
break;
}
if(opl - first <= 1)
{
break;
}
}
TXT_ScopeNode *closest_node = &info->scope_nodes.v[info->scope_pts.v[first].scope_idx];
for(TXT_ScopeNode *scope_n = closest_node;
scope_n != &txt_scope_node_nil;
scope_n = txt_scope_node_from_info_num(info, scope_n->parent_num))
{
if(info->tokens.v[scope_n->token_idx_range.min].range.min <= off && off < info->tokens.v[scope_n->token_idx_range.max].range.max)
{
result = scope_n;
break;
}
}
}
return result;
}
internal TXT_ScopeNode *
txt_scope_node_from_info_pt(TXT_TextInfo *info, TxtPt pt)
{
U64 off = txt_off_from_info_pt(info, pt);
TXT_ScopeNode *result = txt_scope_node_from_info_off(info, off);
return result;
}
////////////////////////////////
//~ rjf: Artifact Cache Hooks / Lookups
typedef struct TXT_Artifact TXT_Artifact;
struct TXT_Artifact
{
Arena *arena;
U128 data_hash;
TXT_TextInfo info;
};
typedef struct TXT_ArtifactCreateShared TXT_ArtifactCreateShared;
struct TXT_ArtifactCreateShared
{
Arena *arena;
TXT_TextInfo info;
TXT_Artifact *artifact;
};
internal AC_Artifact
txt_artifact_create(String8 key, B32 *cancel_signal, B32 *retry_out, U64 *gen_out)
{
ProfBeginFunction();
Temp scratch = scratch_begin(0, 0);
Access *access = access_open();
//- rjf: get shared state
TXT_ArtifactCreateShared *shared = 0;
if(lane_idx() == 0)
{
shared = push_array(scratch.arena, TXT_ArtifactCreateShared, 1);
}
lane_sync_u64(&shared, 0);
//- rjf: unpack key
U128 hash = {0};
TXT_LangKind lang = TXT_LangKind_Null;
str8_deserial_read_struct(key, 0, &hash);
str8_deserial_read_struct(key, sizeof(hash), &lang);
String8 data = c_data_from_hash(access, hash);
TXT_LangLexFunctionType *lex_function = txt_lex_function_from_lang_kind(lang);
//- rjf: data -> text info
if(!u128_match(hash, u128_zero()))
{
if(lane_idx() == 0)
{
shared->arena = arena_alloc();
}
//- rjf: set # of bytes to process
// (line ending calc) (line counting) (line measuring) (lexing)
set_progress_target(Min(data.size, 1024) + data.size + data.size + data.size*(lang != TXT_LangKind_Null));
//- rjf: detect line end kind
TXT_LineEndKind line_end_kind = TXT_LineEndKind_Null;
if(lane_idx() == 0)
{
U64 lf_count = 0;
U64 cr_count = 0;
for(U64 idx = 0; idx < data.size && idx < 1024; idx += 1)
{
if(data.str[idx] == '\r')
{
cr_count += 1;
}
if(data.str[idx] == '\n')
{
lf_count += 1;
}
}
if(cr_count >= lf_count/2 && lf_count >= 1)
{
line_end_kind = TXT_LineEndKind_CRLF;
}
else if(lf_count >= 1)
{
line_end_kind = TXT_LineEndKind_LF;
}
shared->info.line_end_kind = line_end_kind;
}
lane_sync();
set_progress(Min(data.size, 1024));
//- rjf: count # of lines
U64 lane_line_count = 0;
if(lane_idx() == 0)
{
lane_line_count = 1;
}
{
Rng1U64 range = lane_range(data.size);
for EachInRange(idx, range)
{
if(data.str[idx] == '\n')
{
lane_line_count += 1;
}
if(idx && idx%1000 == 0)
{
add_progress(1000);
}
}
}
ins_atomic_u64_add_eval(&shared->info.lines_count, lane_line_count);
lane_sync();
set_progress(Min(data.size, 1024) + data.size);
//- rjf: allocate & store line ranges
if(lane_idx() == 0)
{
shared->info.lines_ranges = push_array_no_zero(shared->arena, Rng1U64, shared->info.lines_count);
U64 line_idx = 0;
U64 line_start_idx = 0;
for(U64 idx = 0; idx <= data.size; idx += 1)
{
if(idx == data.size || data.str[idx] == '\n' || data.str[idx] == '\r')
{
Rng1U64 line_range = r1u64(line_start_idx, idx);
U64 line_size = dim_1u64(line_range);
shared->info.lines_ranges[line_idx] = line_range;
shared->info.lines_max_size = Max(shared->info.lines_max_size, line_size);
line_idx += 1;
line_start_idx = idx+1;
if(idx < data.size && data.str[idx] == '\r')
{
line_start_idx += 1;
idx += 1;
}
}
if(idx && idx%1000 == 0)
{
add_progress(1000);
}
}
}
lane_sync();
set_progress(Min(data.size, 1024) + data.size + data.size);
//- rjf: lex function * data -> tokens
if(lane_idx() == 0 && lex_function != 0)
{
shared->info.tokens = lex_function(shared->arena, 0, data);
}
lane_sync();
set_progress(Min(data.size, 1024) + data.size + data.size + data.size*(lex_function != 0));
TXT_TokenArray tokens = shared->info.tokens;
//- rjf: count scope points
{
U64 lane_scope_pt_opener_count = 0;
U64 lane_scope_pt_count = 0;
Rng1U64 range = lane_range(tokens.count);
for EachInRange(idx, range)
{
if(tokens.v[idx].kind == TXT_TokenKind_Symbol)
{
String8 token_string = str8_substr(data, tokens.v[idx].range);
B32 is_opener = (token_string.str[0] == '{' ||
token_string.str[0] == '(' ||
token_string.str[0] == '[');
B32 is_closer = (token_string.str[0] == '}' ||
token_string.str[0] == ')' ||
token_string.str[0] == ']');
if(token_string.size == 1 && (is_opener || is_closer))
{
lane_scope_pt_count += 1;
lane_scope_pt_opener_count += !!is_opener;
}
}
}
ins_atomic_u64_add_eval(&shared->info.scope_pts.count, lane_scope_pt_count);
ins_atomic_u64_add_eval(&shared->info.scope_nodes.count, lane_scope_pt_opener_count);
}
lane_sync();
//- rjf: allocate & fill scope data
if(lane_idx() == 0)
{
shared->info.scope_pts.v = push_array_no_zero(shared->arena, TXT_ScopePt, shared->info.scope_pts.count);
shared->info.scope_nodes.v = push_array_no_zero(shared->arena, TXT_ScopeNode, shared->info.scope_nodes.count);
{
typedef struct ScopeTask ScopeTask;
struct ScopeTask
{
ScopeTask *next;
U64 scope_idx;
};
Temp scratch = scratch_begin(0, 0);
ScopeTask *top_scope_task = 0;
ScopeTask *free_scope_task = 0;
U64 pt_idx = 0;
U64 scope_idx = 0;
for EachIndex(token_idx, tokens.count)
{
if(tokens.v[token_idx].kind == TXT_TokenKind_Symbol)
{
String8 token_string = str8_substr(data, tokens.v[token_idx].range);
B32 is_opener = (token_string.str[0] == '{' ||
token_string.str[0] == '(' ||
token_string.str[0] == '[');
B32 is_closer = (token_string.str[0] == '}' ||
token_string.str[0] == ')' ||
token_string.str[0] == ']');
// rjf: opener symbols -> push scope
if(is_opener)
{
// rjf: insert into scope tree
TXT_ScopeNode *new_scope = &shared->info.scope_nodes.v[scope_idx];
new_scope->token_idx_range.min = token_idx;
if(top_scope_task)
{
U64 new_scope_num = scope_idx+1;
TXT_ScopeNode *parent = &shared->info.scope_nodes.v[top_scope_task->scope_idx];
if(parent->first_num == 0)
{
parent->first_num = new_scope_num;
}
if(parent->last_num != 0)
{
TXT_ScopeNode *prev_scope = &shared->info.scope_nodes.v[parent->last_num-1];
prev_scope->next_num = new_scope_num;
}
parent->last_num = new_scope_num;
new_scope->parent_num = top_scope_task->scope_idx+1;
}
// rjf: push onto scope stack
ScopeTask *scope_task = free_scope_task;
if(scope_task)
{
SLLStackPop(free_scope_task);
}
else
{
scope_task = push_array(scratch.arena, ScopeTask, 1);
}
scope_task->scope_idx = scope_idx;
scope_idx += 1;
SLLStackPush(top_scope_task, scope_task);
}
// rjf: opener or closer -> fill endpoint
if(top_scope_task && (is_opener || is_closer))
{
shared->info.scope_pts.v[pt_idx].token_idx = token_idx;
shared->info.scope_pts.v[pt_idx].scope_idx = top_scope_task->scope_idx;
pt_idx += 1;
}
// rjf: closer symbols -> pop
if(is_closer && top_scope_task != 0)
{
ScopeTask *popped = top_scope_task;
shared->info.scope_nodes.v[popped->scope_idx].token_idx_range.max = token_idx;
SLLStackPop(top_scope_task);
SLLStackPush(free_scope_task, popped);
}
}
}
scratch_end(scratch);
}
}
lane_sync();
}
//- rjf: mark dependency on hash
c_hash_downstream_inc(hash);
//- rjf: package as artifact
if(lane_idx() == 0 && shared->arena != 0)
{
shared->artifact = push_array(shared->arena, TXT_Artifact, 1);
shared->artifact->arena = shared->arena;
shared->artifact->data_hash = hash;
shared->artifact->info = shared->info;
}
lane_sync();
access_close(access);
scratch_end(scratch);
ProfEnd();
AC_Artifact result = {0};
result.u64[0] = (U64)shared->artifact;
return result;
}
internal void
txt_artifact_destroy(AC_Artifact artifact)
{
TXT_Artifact *txt_artifact = (TXT_Artifact *)artifact.u64[0];
if(txt_artifact == 0) { return; }
c_hash_downstream_dec(txt_artifact->data_hash);
arena_release(txt_artifact->arena);
}
internal TXT_TextInfo
txt_text_info_from_hash_lang(Access *access, U128 hash, TXT_LangKind lang)
{
#pragma pack(push, 1)
struct
{
U128 hash;
TXT_LangKind lang;
} key = {hash, lang};
#pragma pack(pop)
String8 key_string = str8_struct(&key);
AC_Artifact artifact = ac_artifact_from_key(access, key_string, txt_artifact_create, txt_artifact_destroy, 0, .flags = AC_Flag_Wide);
TXT_Artifact *txt_artifact = (TXT_Artifact *)artifact.u64[0];
TXT_TextInfo info = {0};
if(txt_artifact != 0)
{
info = txt_artifact->info;
}
return info;
}
internal TXT_TextInfo
txt_text_info_from_key_lang(Access *access, C_Key key, TXT_LangKind lang, U128 *hash_out)
{
TXT_TextInfo result = {0};
for(U64 rewind_idx = 0; rewind_idx < C_KEY_HASH_HISTORY_COUNT; rewind_idx += 1)
{
U128 hash = c_hash_from_key(key, rewind_idx);
result = txt_text_info_from_hash_lang(access, hash, lang);
if(result.lines_count != 0)
{
if(hash_out)
{
*hash_out = hash;
}
break;
}
}
return result;
}
Reference in New Issue View Git Blame Copy Permalink