From e55fa733fd6ce4a3222090fc9c728f82ca42c8a1 Mon Sep 17 00:00:00 2001 From: Ryan Fleury Date: Mon, 27 Jul 2026 12:20:38 -0700 Subject: [PATCH] mostly restore existing read-only functionality, keep existing patching infrastructure but do not use --- project.raddbg | 1 + src/raddbg/raddbg_core.c | 1 + src/raddbg/raddbg_views.c | 8 +- src/text/text.c | 334 +++++++++++++++++++++++++++++++++----- src/text/text.h | 2 +- 5 files changed, 306 insertions(+), 40 deletions(-) diff --git a/project.raddbg b/project.raddbg index 739e03eb..4adbb44b 100644 --- a/project.raddbg +++ b/project.raddbg @@ -20,6 +20,7 @@ target: { executable: "build/mule_main.exe" working_directory: build + output_label: output_a } target: { diff --git a/src/raddbg/raddbg_core.c b/src/raddbg/raddbg_core.c index 6d3e735e..14f952a3 100644 --- a/src/raddbg/raddbg_core.c +++ b/src/raddbg/raddbg_core.c @@ -9805,6 +9805,7 @@ rd_code_color_slot_from_txt_token_kind(TXT_TokenKind kind) case TXT_TokenKind_Keyword: {color = RD_CodeColorSlot_CodeKeyword;}break; case TXT_TokenKind_Numeric: {color = RD_CodeColorSlot_CodeNumeric;}break; case TXT_TokenKind_String: {color = RD_CodeColorSlot_CodeString;}break; + case TXT_TokenKind_Char: {color = RD_CodeColorSlot_CodeString;}break; case TXT_TokenKind_Meta: {color = RD_CodeColorSlot_CodeMeta;}break; case TXT_TokenKind_LineComment: {color = RD_CodeColorSlot_CodeComment;}break; case TXT_TokenKind_BlockComment:{color = RD_CodeColorSlot_CodeComment;}break; diff --git a/src/raddbg/raddbg_views.c b/src/raddbg/raddbg_views.c index 184a9662..6f6ab47d 100644 --- a/src/raddbg/raddbg_views.c +++ b/src/raddbg/raddbg_views.c @@ -155,6 +155,12 @@ rd_code_view_build(Arena *arena, RD_CodeViewState *cv, RD_CodeViewBuildFlags fla //- rjf: interpret event as single-line text op UI_TxtOp single_line_op = ui_single_line_txt_op_from_event(scratch.arena, evt, line, line_range, *cursor, *mark); + //- TODO(rjf): skip replace-ranges for now + if(single_line_op.replace.size != 0) + { + continue; + } + //- rjf: apply single-line navigations *cursor = single_line_op.cursor; *mark = single_line_op.mark; @@ -394,7 +400,7 @@ rd_code_view_build(Arena *arena, RD_CodeViewState *cv, RD_CodeViewBuildFlags fla // U64 ctx_token_pt_num = txt_token_pt_num_from_off(&text_patched.token_pt_map, visible_byte_range.min); TXT_TokenPt ctx_token_pt = txt_token_pt_from_num(&text_patched.token_pt_map, ctx_token_pt_num); - TXT_TokenArray tokens = txt_token_array_from_data(scratch.arena, ctx_token_pt, visible_data, visible_byte_range.min, max_U64); + TXT_TokenArray tokens = txt_token_array_from_data(scratch.arena, rd_regs()->lang_kind, ctx_token_pt, visible_data, visible_byte_range.min, max_U64); ////////////////////////////// //- rjf: selection on single line, no query? -> set search text diff --git a/src/text/text.c b/src/text/text.c index 70159cc0..55ac73d9 100644 --- a/src/text/text.c +++ b/src/text/text.c @@ -2375,6 +2375,7 @@ txt_token_pt_num_from_off(TXT_TokenPtMap *map, U64 off) U64 last_off = n->pts[num_pts-1].off + n->delta; if(first_off <= off && off <= last_off) { + U64 found_idx = 0; U64 first_idx = 0; U64 last_idx = num_pts-1; for(;first_idx <= last_idx;) @@ -2387,15 +2388,16 @@ txt_token_pt_num_from_off(TXT_TokenPtMap *map, U64 off) } else if(mid_off < off) { - first_idx = mid_idx; + found_idx = mid_idx; + first_idx = mid_idx + 1; } else if(off == mid_off) { - first_idx = mid_idx; + found_idx = mid_idx; break; } } - result = n->num_range.min + first_idx; + result = n->num_range.min + found_idx; break; } } @@ -2423,14 +2425,17 @@ txt_token_pt_from_num(TXT_TokenPtMap *map, U64 num) } internal TXT_TokenArray -txt_token_array_from_data(Arena *arena, TXT_TokenPt ctx_token_pt, String8 data, U64 base_off, U64 limit) +txt_token_array_from_data(Arena *arena, TXT_LangKind lang_kind, TXT_TokenPt ctx_token_pt, String8 data, U64 base_off, U64 limit) { Temp scratch = scratch_begin(&arena, 1); TXT_TokenChunkList tokens = {0}; + if(lang_kind != TXT_LangKind_Null) { + String8Array keywords = txt_keywords_from_lang_kind_table[lang_kind]; + String8Array multichar_symbols = txt_multichar_symbols_from_lang_kind_table[lang_kind]; U64 chunk_size = Clamp(8, data.size/8, 4096); TXT_TokenKind active_token_kind = ctx_token_pt.kind; - U64 active_token_start_off = ctx_token_pt.off; + U64 active_token_start_off = ctx_token_pt.off - base_off; B32 escaped = 0; for(U64 off = 0; off <= data.size;) { @@ -2555,7 +2560,7 @@ txt_token_array_from_data(Arena *arena, TXT_TokenPt ctx_token_pt, String8 data, }break; case TXT_TokenKind_String: { - if(byte == '"') + if(byte == '"' && !escaped) { token_finished = 1; advance = 1; @@ -2563,7 +2568,7 @@ txt_token_array_from_data(Arena *arena, TXT_TokenPt ctx_token_pt, String8 data, }break; case TXT_TokenKind_Char: { - if(byte == '\'') + if(byte == '\'' && !escaped) { token_finished = 1; advance = 1; @@ -2590,7 +2595,7 @@ txt_token_array_from_data(Arena *arena, TXT_TokenPt ctx_token_pt, String8 data, }break; case TXT_TokenKind_LineComment: { - if(byte == '\n') + if(byte == '\n' && !escaped) { token_finished = 1; advance = 1; @@ -2606,7 +2611,7 @@ txt_token_array_from_data(Arena *arena, TXT_TokenPt ctx_token_pt, String8 data, }break; case TXT_TokenKind_Meta: { - if(byte == '\n') + if(byte == '\n' && !escaped) { token_finished = 1; advance = 1; @@ -2621,6 +2626,20 @@ txt_token_array_from_data(Arena *arena, TXT_TokenPt ctx_token_pt, String8 data, advance = 1; } + //- rjf: upgrade identifiers to keywords + if(token_finished && active_token_kind == TXT_TokenKind_Identifier) + { + String8 token_string = str8_substr(data, r1u64(active_token_start_off, off+advance)); + for EachIndex(idx, keywords.count) + { + if(str8_match(keywords.v[idx], token_string, 0)) + { + active_token_kind = TXT_TokenKind_Keyword; + break; + } + } + } + //- rjf: push completed token if(token_finished) { @@ -2646,53 +2665,65 @@ txt_patched_from_info_data_patches(Arena *arena, TXT_TextInfo *info, String8 dat { Temp scratch = scratch_begin(&arena, 1); - // rjf: produce default case, where we just have one range which covers the original data + ////////////////////////////// + //- rjf: produce default case, where we just have one range which covers the original data + // MemoryMap last_memory_map = {0}; U64 last_size = data.size; TXT_LineMap last_line_map = {0}; TXT_TokenPtMap last_token_pt_map = {0}; memory_map_push(scratch.arena, &last_memory_map, r1u64(0, data.size), data.str); txt_line_map_push(scratch.arena, &last_line_map, r1u64(1, info->lines_count+1), info->lines_ranges, 0); - txt_token_pt_map_push(scratch.arena, &last_token_pt_map, r1u64(0, info->big_token_pts_count), info->big_token_pts, 0); + txt_token_pt_map_push(scratch.arena, &last_token_pt_map, r1u64(1, info->big_token_pts_count+1), info->big_token_pts, 0); - // rjf: apply the patches in order, each being able to slice/dice the previous memory map + ////////////////////////////// + //- rjf: apply the patches in order, each being able to slice/dice the previous memory map + // for EachNode(n, TXT_PatchNode, patches->first) { MemoryMap next_memory_map = {0}; TXT_LineMap next_line_map = {0}; TXT_TokenPtMap next_token_pt_map = {0}; - // rjf: compute portion of memory before/after this replace-range + //////////////////////////// + //- rjf: compute portion of memory before/after this replace-range + // Rng1U64 pre_replace_range = r1u64(0, n->v.range.min); Rng1U64 post_replace_range = r1u64(n->v.range.max, last_size); - // rjf: map this replace range -> range of replaced newlines + //////////////////////////// + //- rjf: map this replace range -> range of replaced newlines + // Rng1U64 replace_line_num_range = {0}; { replace_line_num_range.min = txt_line_num_from_off(&last_line_map, n->v.range.min); replace_line_num_range.max = txt_line_num_from_off(&last_line_map, n->v.range.max); } - // rjf: compute ranges of (unchanged) lines before/after this replace-range + //////////////////////////// + //- rjf: compute ranges of (unchanged) lines before/after this replace-range + // Rng1U64 pre_replace_line_num_range = r1u64(1, replace_line_num_range.min); Rng1U64 post_replace_line_num_range = r1u64(replace_line_num_range.max+1, last_line_map.total_line_count+1); - // rjf: map this replace range -> range of token pts + //////////////////////////// + //- rjf: map this replace range -> range of token pts + // Rng1U64 replace_token_pt_range = {0}; { replace_token_pt_range.min = txt_token_pt_num_from_off(&last_token_pt_map, n->v.range.min); replace_token_pt_range.max = txt_token_pt_num_from_off(&last_token_pt_map, n->v.range.max); } - // rjf: compute ranges of unchanged token pts before/after this replace-range - Rng1U64 pre_replace_token_pt_range = r1u64(1, replace_token_pt_range.min); - Rng1U64 post_replace_token_pt_range = r1u64(replace_token_pt_range.max, last_token_pt_map.total_pt_count+1); - - // rjf: compute delta & next size + //////////////////////////// + //- rjf: compute delta & next size + // S64 size_delta = (S64)n->v.replace.size - (S64)dim_1u64(n->v.range); U64 next_size = (U64)((S64)last_size + size_delta); - // rjf: compute line count delta, + list of line ranges inside of replace + //////////////////////////// + //- rjf: compute line count delta, + list of line ranges inside of replace + // S64 line_delta = 0; Rng1U64List replace_line_ranges = {0}; { @@ -2715,13 +2746,9 @@ txt_patched_from_info_data_patches(Arena *arena, TXT_TextInfo *info, String8 dat rng1u64_list_push(scratch.arena, &replace_line_ranges, r1u64(last_line_start_off, n->v.replace.size)); } - // rjf: compute token pt delta - S64 token_pt_delta = 0; - { - // TODO(rjf) - } - - // rjf: push all portions of pre-replace / post-replace ranges in previous memory map + //////////////////////////// + //- rjf: push all portions of pre-replace / post-replace ranges in previous memory map + // { for EachNode(map_n, MemoryMapRangeNode, last_memory_map.first_range) { @@ -2741,13 +2768,17 @@ txt_patched_from_info_data_patches(Arena *arena, TXT_TextInfo *info, String8 dat } } - // rjf: push replaced range + //////////////////////////// + //- rjf: push replaced range + // if(n->v.replace.size != 0) { memory_map_push(scratch.arena, &next_memory_map, r1u64(n->v.range.min, n->v.range.min + n->v.replace.size), n->v.replace.str); } - // rjf: push all portions of pre-replace / post-replace ranges in previous line map + //////////////////////////// + //- rjf: push all portions of pre-replace / post-replace ranges in previous line map + // { for EachNode(map_n, TXT_LineMapRangeNode, last_line_map.first_range) { @@ -2768,7 +2799,9 @@ txt_patched_from_info_data_patches(Arena *arena, TXT_TextInfo *info, String8 dat } } - // rjf: compute affected line ranges + //////////////////////////// + //- rjf: compute affected line ranges + // U64 affected_line_count = replace_line_ranges.count; Rng1U64 *affected_line_ranges = push_array(arena, Rng1U64, affected_line_count); { @@ -2805,10 +2838,223 @@ txt_patched_from_info_data_patches(Arena *arena, TXT_TextInfo *info, String8 dat } } - // rjf: push affected line ranges + //////////////////////////// + //- rjf: push affected line ranges + // txt_line_map_push(scratch.arena, &next_line_map, r1u64(replace_line_num_range.min, replace_line_num_range.min + affected_line_count), affected_line_ranges, 0); - // rjf: push all portions of pre-replace / post-replace ranges in previous token pt map + //////////////////////////// + //- rjf: compute token pt delta, + token pt ranges to keep, + new token pts + // + S64 token_pt_delta = 0; + Rng1U64 pre_replace_token_pt_range = r1u64(1, replace_token_pt_range.min); + Rng1U64 post_replace_token_pt_range = r1u64(replace_token_pt_range.max, last_token_pt_map.total_pt_count+1); + TXT_TokenPt *new_token_pts = 0; + U64 new_token_pts_count = 0; + { + U64 token_pt_lex_start_off = n->v.range.min; + + //- rjf: eliminate starter token pt - range may not cover entire token pt, but we will re-lex this portion + if(replace_token_pt_range.min != 0) + { + TXT_TokenPt starter_token_pt = txt_token_pt_from_num(&last_token_pt_map, replace_token_pt_range.min); + Rng1U64 starter_token_pt_range = r1u64(starter_token_pt.off, starter_token_pt.off + 4); + if(contains_1u64(starter_token_pt_range, n->v.range.min)) + { + token_pt_delta -= 1; + token_pt_lex_start_off = starter_token_pt.off > 0 ? starter_token_pt.off-1 : starter_token_pt.off; + if(replace_token_pt_range.min == replace_token_pt_range.max) + { + post_replace_token_pt_range.min += 1; + } + } + else + { + pre_replace_token_pt_range.max += 1; + } + } + + //- rjf: eliminate ender token pt + if(replace_token_pt_range.max != 0 && replace_token_pt_range.max != replace_token_pt_range.min) + { + TXT_TokenPt ender_token_pt = txt_token_pt_from_num(&last_token_pt_map, replace_token_pt_range.min+1); + if(n->v.range.max > ender_token_pt.off) + { + token_pt_delta -= 1; + post_replace_token_pt_range.min += 1; + } + } + + //- rjf: re-lex region to find new token pts + typedef struct TokenPtChunkNode TokenPtChunkNode; + struct TokenPtChunkNode + { + TokenPtChunkNode *next; + TXT_TokenPt *v; + U64 count; + U64 cap; + }; + TokenPtChunkNode *first_pt_chunk = 0; + TokenPtChunkNode *last_pt_chunk = 0; + U64 total_new_pt_count = 0; + { + U64 ctx_token_pt_num = txt_token_pt_num_from_off(&last_token_pt_map, token_pt_lex_start_off); + TXT_TokenPt ctx_token_pt = txt_token_pt_from_num(&last_token_pt_map, ctx_token_pt_num); + TXT_TokenKind active_token_kind = ctx_token_pt.kind; + U64 active_token_start_off = 0; + String8 herestring_marker = {0}; + Rng1U64 relex_range = r1u64(token_pt_lex_start_off, n->v.range.min + n->v.replace.size); + String8 relex_data = memory_map_data_from_range(scratch.arena, &next_memory_map, relex_range); + B32 escaped = 0; + for(U64 off = 0; off <= relex_data.size; off += 1) + { + U64 extra_advance = 0; + U8 byte = (off+0 < relex_data.size) ? relex_data.str[off+0] : 0; + U8 next_byte = (off+1 < relex_data.size) ? relex_data.str[off+1] : 0; + + //- rjf: adjust escaping state + if(active_token_kind != TXT_TokenKind_Null && byte == '\\') + { + escaped = 1; + } + + //- rjf: no active token -> look for starters + TXT_TokenKind start_active_token_kind = active_token_kind; + U64 active_token_end_off = 0; + if(active_token_kind == TXT_TokenKind_Null) + { + // rjf: " -> start a string literal + if(byte == '"') + { + active_token_kind = TXT_TokenKind_String; + herestring_marker.size = 0; + } + + // rjf: ' -> start a char literal + else if(byte == '\'') + { + active_token_kind = TXT_TokenKind_Char; + herestring_marker.size = 0; + } + + // rjf: R" -> start a C++11+ style herestring + else if(byte == 'R' && next_byte == '"') + { + active_token_kind = TXT_TokenKind_String; + U64 next_paren_pos = str8_find_needle(str8_prefix(data, off+2+256), off+2, s("("), 0); + herestring_marker = str8_substr(data, r1u64(off+2, next_paren_pos)); + extra_advance = 1 + herestring_marker.size + 1; + } + + // rjf: // -> start a single-line comment + else if(byte == '/' && next_byte == '/') + { + active_token_kind = TXT_TokenKind_LineComment; + extra_advance = 1; + } + + // rjf: /* -> start a multi-line comment + else if(byte == '/' && next_byte == '*') + { + active_token_kind = TXT_TokenKind_BlockComment; + extra_advance = 1; + } + + // rjf: # -> start a meta + else if(byte == '#') + { + active_token_kind = TXT_TokenKind_Meta; + } + + // rjf: got a token kind -> remember its starting offset + if(active_token_kind != TXT_TokenKind_Null) + { + active_token_start_off = off; + } + } + + //- rjf: look for enders + else switch(active_token_kind) + { + default:{}break; + case TXT_TokenKind_LineComment: + case TXT_TokenKind_Meta: + if(!escaped && byte == '\n') + { + active_token_end_off = off; + }break; + case TXT_TokenKind_BlockComment: + if(byte == '*' && next_byte == '/') + { + active_token_end_off = off+1; + extra_advance = 1; + }break; + case TXT_TokenKind_String: + { + // TODO(rjf): herestrings - we might not have the right herestring marker, given that it may + // be from an earlier token pt...? + if(byte == '"' && !escaped) + { + active_token_end_off = off+1; + } + }break; + case TXT_TokenKind_Char: + if(!escaped && byte == '\'') + { + active_token_end_off = off+1; + extra_advance = 1; + }break; + } + + //- rjf: found ender -> reset state + if(active_token_end_off > active_token_start_off) + { + active_token_kind = TXT_TokenKind_Null; + } + + //- rjf: state changed -> push new pt + if(active_token_kind != start_active_token_kind) + { + TXT_TokenPt pt = {active_token_kind, relex_range.min + off + ((active_token_kind == TXT_TokenKind_Null) ? extra_advance : 0)}; + TokenPtChunkNode *chunk = last_pt_chunk; + if(chunk == 0 || chunk->count >= chunk->cap) + { + chunk = push_array(scratch.arena, TokenPtChunkNode, 1); + SLLQueuePush(first_pt_chunk, last_pt_chunk, chunk); + chunk->cap = 256; + chunk->v = push_array(scratch.arena, TXT_TokenPt, chunk->cap); + } + chunk->v[chunk->count] = pt; + chunk->count += 1; + total_new_pt_count += 1; + } + + //- rjf: reset escaped state + escaped = 0; + + //- rjf: do extra advance + off += extra_advance; + } + } + + //- rjf: join new token pts + if(total_new_pt_count != 0) + { + token_pt_delta += total_new_pt_count; + new_token_pts_count = total_new_pt_count; + new_token_pts = push_array(arena, TXT_TokenPt, new_token_pts_count); + U64 idx = 0; + for(TokenPtChunkNode *n = first_pt_chunk; n != 0; n = n->next) + { + MemoryCopy(new_token_pts + idx, n->v, sizeof(n->v[0]) * n->count); + idx += n->count; + } + } + } + + //////////////////////////// + //- rjf: push all portions of pre-replace / post-replace ranges in previous token pt map + // { for EachNode(n, TXT_TokenPtMapRangeNode, last_token_pt_map.first_range) { @@ -2824,12 +3070,22 @@ txt_patched_from_info_data_patches(Arena *arena, TXT_TextInfo *info, String8 dat Rng1U64 range_x_post_shifted = range_x_post; range_x_post_shifted.min = (U64)((S64)range_x_post_shifted.min + token_pt_delta); range_x_post_shifted.max = (U64)((S64)range_x_post_shifted.max + token_pt_delta); - txt_token_pt_map_push(scratch.arena, &next_token_pt_map, range_x_post_shifted, n->pts + (range_x_post.min - num_range.min), n->delta); + txt_token_pt_map_push(scratch.arena, &next_token_pt_map, range_x_post_shifted, n->pts + (range_x_post.min - num_range.min), n->delta + size_delta); } } } - // rjf: advance to the next memory map + //////////////////////////// + //- rjf: push new token pts + // + if(new_token_pts_count != 0) + { + txt_token_pt_map_push(scratch.arena, &next_token_pt_map, r1u64(pre_replace_token_pt_range.max, pre_replace_token_pt_range.max + new_token_pts_count), new_token_pts, 0); + } + + //////////////////////////// + //- rjf: advance to the next state + // last_memory_map = next_memory_map; last_size = next_size; last_line_map = next_line_map; @@ -3457,6 +3713,7 @@ txt_artifact_create(String8 key, B32 *cancel_signal, AC_Status *status_out, U64 }; TokenEndpointCandidateChunkNode **lanes_first_token_endpoint_candidate_chunks = 0; TokenEndpointCandidateChunkNode **lanes_last_token_endpoint_candidate_chunks = 0; + if(lang != TXT_LangKind_Null) { if(lane_idx() == 0) { @@ -3526,7 +3783,7 @@ txt_artifact_create(String8 key, B32 *cancel_signal, AC_Status *status_out, U64 //- rjf: join all token endpoint candidates from all lanes TokenEndpointCandidateChunkNode *first_token_endpoint_candidate_chunk = 0; TokenEndpointCandidateChunkNode *last_token_endpoint_candidate_chunk = 0; - if(lane_idx() == 0) + if(lang != TXT_LangKind_Null && lane_idx() == 0) { for EachIndex(l_idx, lane_count()) { @@ -3557,7 +3814,7 @@ txt_artifact_create(String8 key, B32 *cancel_signal, AC_Status *status_out, U64 TokenPtChunkNode *first_token_pt_chunk = 0; TokenPtChunkNode *last_token_pt_chunk = 0; U64 total_token_pt_count = 0; - if(lane_idx() == 0) + if(lang != TXT_LangKind_Null && lane_idx() == 0) { TXT_TokenKind active_token_kind = TXT_TokenKind_Null; U64 active_token_start_off = 0; @@ -3712,6 +3969,7 @@ txt_artifact_create(String8 key, B32 *cancel_signal, AC_Status *status_out, U64 chunk->v[chunk->count] = pts[pt_idx]; chunk->count += 1; total_token_pt_count += 1; + active_token_kind = TXT_TokenKind_Null; } } diff --git a/src/text/text.h b/src/text/text.h index a6b2f043..521a46c8 100644 --- a/src/text/text.h +++ b/src/text/text.h @@ -311,7 +311,7 @@ internal Rng1U64 txt_range_from_line_num(TXT_LineMap *map, U64 num); internal void txt_token_pt_map_push(Arena *arena, TXT_TokenPtMap *map, Rng1U64 num_range, TXT_TokenPt *pts, S64 delta); internal U64 txt_token_pt_num_from_off(TXT_TokenPtMap *map, U64 off); internal TXT_TokenPt txt_token_pt_from_num(TXT_TokenPtMap *map, U64 num); -internal TXT_TokenArray txt_token_array_from_data(Arena *arena, TXT_TokenPt ctx_token_pt, String8 data, U64 base_off, U64 limit); +internal TXT_TokenArray txt_token_array_from_data(Arena *arena, TXT_LangKind lang_kind, TXT_TokenPt ctx_token_pt, String8 data, U64 base_off, U64 limit); internal TXT_Patched txt_patched_from_info_data_patches(Arena *arena, TXT_TextInfo *info, String8 data, TXT_PatchList *patches); //~ TODO(rjf): old unpatched text viz code: