ed/raddebugger
1
0
Fork 0
mirror of https://github.com/Ed94/raddebugger.git synced 2026-08-14 17:28:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

checkpoint on unified lexer

Browse Source
This commit is contained in:
Ryan Fleury
2025-11-06 12:47:05 -08:00
parent 82bc133ba5
commit e9e0036122
1 changed files with 58 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/text/text.c
+58 -38
View File
@@ -215,13 +215,12 @@ txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, Stri
next_escaped = 1;
}
// rjf: take starter, push new token tasks
// rjf: take token starters
B32 new_token_needed = (top_task == 0);
TXT_TokenizerRule *new_rule = nil_rule;
TXT_TokenKind new_token_kind = TXT_TokenKind_Null;
{
TXT_TokenizerRule *new_rule = nil_rule;
TXT_TokenKind new_token_kind = TXT_TokenKind_Null;
// rjf: use next bytes to look up a rule from the table
if(top_task == 0)
{
TXT_TokenizerRule *active_rule = top_task ? top_task->rule : nil_rule;
U64 hash_1byte = u64_hash_from_str8(string_1byte);
@@ -292,37 +291,11 @@ txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, Stri
byte == '>' || byte == '/' ||
byte == '?' || byte == '|') { new_token_kind = TXT_TokenKind_Symbol; }
}
// rjf: start new token
if(new_token_kind != TXT_TokenKind_Null)
{
TokenTask *task = free_task;
if(task != 0)
{
SLLStackPop(free_task);
}
else
{
task = push_array(scratch.arena, TokenTask, 1);
}
SLLStackPush(top_task, task);
top_task->rule = new_rule;
top_task->kind = new_token_kind;
top_task->start_idx = idx;
}
// rjf: invalid token kind -> emit error
else if(top_task == 0)
{
TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)};
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
}
}
// rjf: look for ender based on rule's closing symbol
U64 ender_pad = 0;
B32 ender_found = 0;
B32 task_pop = 0;
if(top_task != 0 && idx > top_task->start_idx)
{
TXT_TokenKind active_token_kind = top_task->kind;
@@ -373,13 +346,33 @@ txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, Stri
}
}
// rjf: next byte is ender => emit token
// rjf: if we have a new token to start, but we have an active token, then
// end the current token (but keep the same stack)
B32 keep_top_task = 0;
if(top_task != 0 && new_token_kind != TXT_TokenKind_Null && idx > top_task->start_idx)
{
keep_top_task = (!ender_found);
ender_found = 1;
}
// rjf: if we have an ender => emit token(s) for current task
if(ender_found)
{
TXT_Token token = {top_task->kind, r1u64(top_task->start_idx, idx+ender_pad)};
TokenTask *popped = top_task;
SLLStackPop(top_task);
SLLStackPush(free_task, popped);
if(!keep_top_task)
{
TokenTask *popped = top_task;
SLLStackPop(top_task);
SLLStackPush(free_task, popped);
if(top_task != 0)
{
top_task->start_idx = idx+ender_pad;
}
}
else
{
top_task->start_idx = idx+ender_pad;
}
// rjf: trim \r's off of end
{
@@ -442,9 +435,31 @@ txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, Stri
{
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
}
// rjf: increment by ender padding
idx += ender_pad;
}
// rjf: start new token
if(new_token_kind != TXT_TokenKind_Null)
{
TokenTask *task = free_task;
if(task != 0)
{
SLLStackPop(free_task);
}
else
{
task = push_array(scratch.arena, TokenTask, 1);
}
SLLStackPush(top_task, task);
top_task->rule = new_rule;
top_task->kind = new_token_kind;
top_task->start_idx = idx;
}
// rjf: invalid token kind -> emit error
else if(new_token_needed)
{
TXT_Token token = {TXT_TokenKind_Error, r1u64(idx, idx+1)};
txt_token_chunk_list_push(scratch.arena, &tokens, chunk_size, &token);
}
// rjf: advance by 1 byte if we haven't found an ender
@@ -452,6 +467,11 @@ txt_token_array_from_lang_kind_string(Arena *arena, TXT_LangKind lang_kind, Stri
{
idx += 1;
}
// rjf: advance by ender padding
idx += ender_pad;
// rjf: advance escaping
escaped = next_escaped;
}
}