ed/raddebugger
1
0
Fork 0
mirror of https://github.com/Ed94/raddebugger.git synced 2026-08-08 06:48:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

export line and column numbers from MD

Browse Source
This commit is contained in:
Nikita Smith
2026-06-02 12:10:56 -07:00
committed by Ryan Fleury
parent 3715021e0e
commit 3a7586e84b
4 changed files with 253 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mdesk/mdesk.c
+102 -68
View File
@@ -185,7 +185,7 @@ md_node_rec_depth_first(MD_Node *node, MD_Node *subtree_root, U64 child_off, U64
//- rjf: tree building
internal MD_Node *
md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset)
md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset, U64 line_num, U64 col_num)
{
MD_Node *node = push_array(arena, MD_Node, 1);
node->first = node->last = node->parent = node->next = node->prev = node->first_tag = node->last_tag = &md_nil_node;
@@ -194,6 +194,8 @@ md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string,
node->string = string;
node->raw_string = raw_string;
node->src_offset = src_offset;
node->line_num = line_num;
node->col_num = col_num;
return node;
}
@@ -522,10 +524,25 @@ md_tokenize_from_text(Arena *arena, String8 text)
Temp scratch = scratch_begin(&arena, 1);
MD_TokenChunkList tokens = {0};
MD_MsgList msgs = {0};
U8 *byte_first = text.str;
U8 *byte_opl = byte_first + text.size;
U8 *byte = byte_first;
U64 line_num = 1;
U64 col_num = 1;
#define Advance(n) do { \
for(U64 i = 0; i < n && byte < byte_opl; i += 1) { \
if(*byte == '\n') { \
line_num += 1; \
col_num = 0; \
} \
col_num += 1; \
byte += 1; \
} \
} while(0)
//- rjf: scan string & produce tokens
for(;byte < byte_opl;)
{
@@ -533,21 +550,21 @@ md_tokenize_from_text(Arena *arena, String8 text)
U8 *token_start = 0;
U8 *token_opl = 0;
U64 token_line_num = line_num;
U64 token_col_num = col_num;
//- rjf: whitespace
if(token_flags == 0 && (*byte == ' ' || *byte == '\t' || *byte == '\v' || *byte == '\r'))
{
token_flags = MD_TokenFlag_Whitespace;
token_start = byte;
for(; byte < byte_opl && (*byte == ' ' || *byte == '\t' || *byte == '\v' || *byte == '\r');)
{
Advance(1);
}
token_opl = byte;
byte += 1;
for(;byte <= byte_opl; byte += 1)
{
token_opl += 1;
if(byte == byte_opl || (*byte != ' ' && *byte != '\t' && *byte != '\v' && *byte != '\r'))
{
break;
}
}
}
//- rjf: newlines
@@ -555,8 +572,8 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
token_flags = MD_TokenFlag_Newline;
token_start = byte;
token_opl = byte+1;
byte += 1;
Advance(1);
token_opl = byte;
}
//- rjf: single-line comments
@@ -564,16 +581,11 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
token_flags = MD_TokenFlag_Comment;
token_start = byte;
token_opl = byte+2;
byte += 2;
Advance(2);
B32 escaped = 0;
for(;byte <= byte_opl; byte += 1)
for(;byte < byte_opl;)
{
token_opl += 1;
if(byte == byte_opl)
{
break;
}
if(escaped)
{
escaped = 0;
@@ -589,7 +601,10 @@ md_tokenize_from_text(Arena *arena, String8 text)
escaped = 1;
}
}
Advance(1);
}
token_opl = byte;
}
//- rjf: multi-line comments
@@ -597,22 +612,27 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
token_flags = MD_TokenFlag_Comment;
token_start = byte;
token_opl = byte+2;
byte += 2;
for(;byte <= byte_opl; byte += 1)
Advance(2);
B32 missing_close = 1;
for(;byte < byte_opl;)
{
token_opl += 1;
if(byte == byte_opl)
{
token_flags |= MD_TokenFlag_BrokenComment;
break;
}
if(byte+1 < byte_opl && byte[0] == '*' && byte[1] == '/')
{
token_opl += 2;
missing_close = 0;
Advance(2);
break;
}
Advance(1);
}
if(missing_close)
{
token_flags |= MD_TokenFlag_BrokenComment;
}
token_opl = byte;
}
//- rjf: identifiers
@@ -623,11 +643,11 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
token_flags = MD_TokenFlag_Identifier;
token_start = byte;
token_opl = byte;
byte += 1;
for(;byte <= byte_opl; byte += 1)
Advance(1);
for(;byte <= byte_opl; )
{
token_opl += 1;
if(byte == byte_opl ||
(!('A' <= *byte && *byte <= 'Z') &&
!('a' <= *byte && *byte <= 'z') &&
@@ -637,7 +657,10 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
break;
}
Advance(1);
}
token_opl = byte;
}
//- rjf: numerics
@@ -648,11 +671,11 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
token_flags = MD_TokenFlag_Numeric;
token_start = byte;
token_opl = byte;
byte += 1;
for(;byte <= byte_opl; byte += 1)
Advance(1);
for(;byte <= byte_opl;)
{
token_opl += 1;
if(byte == byte_opl ||
(!('A' <= *byte && *byte <= 'Z') &&
!('a' <= *byte && *byte <= 'z') &&
@@ -662,7 +685,10 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
break;
}
Advance(1);
}
token_opl = byte;
}
//- rjf: triplet string literals
@@ -677,9 +703,10 @@ md_tokenize_from_text(Arena *arena, String8 text)
token_flags |= (literal_style == '"')*MD_TokenFlag_StringDoubleQuote;
token_flags |= (literal_style == '`')*MD_TokenFlag_StringTick;
token_start = byte;
token_opl = byte+3;
byte += 3;
for(;byte <= byte_opl; byte += 1)
Advance(3);
for(;byte <= byte_opl;)
{
if(byte == byte_opl)
{
@@ -689,11 +716,14 @@ md_tokenize_from_text(Arena *arena, String8 text)
}
if(byte+2 < byte_opl && (byte[0] == literal_style && byte[1] == literal_style && byte[2] == literal_style))
{
byte += 3;
token_opl = byte;
Advance(3);
break;
}
Advance(1);
}
token_opl = byte;
}
//- rjf: singlet string literals
@@ -705,14 +735,14 @@ md_tokenize_from_text(Arena *arena, String8 text)
token_flags |= (literal_style == '"')*MD_TokenFlag_StringDoubleQuote;
token_flags |= (literal_style == '`')*MD_TokenFlag_StringTick;
token_start = byte;
token_opl = byte+1;
byte += 1;
Advance(1);
B32 escaped = 0;
for(;byte <= byte_opl; byte += 1)
for(;byte <= byte_opl;)
{
if(byte == byte_opl || *byte == '\n')
{
token_opl = byte;
token_flags |= MD_TokenFlag_BrokenStringLiteral;
break;
}
@@ -722,15 +752,17 @@ md_tokenize_from_text(Arena *arena, String8 text)
}
else if(!escaped && byte[0] == literal_style)
{
token_opl = byte+1;
byte += 1;
Advance(1);
break;
}
else if(escaped)
{
escaped = 0;
}
Advance(1);
}
token_opl = byte;
}
//- rjf: non-reserved symbols
@@ -741,11 +773,9 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
token_flags = MD_TokenFlag_Symbol;
token_start = byte;
token_opl = byte;
byte += 1;
for(;byte <= byte_opl; byte += 1)
Advance(1);
for(;byte <= byte_opl;)
{
token_opl += 1;
if(byte == byte_opl ||
(*byte != '~' && *byte != '!' && *byte != '$' && *byte != '%' && *byte != '^' &&
*byte != '&' && *byte != '*' && *byte != '-' && *byte != '=' && *byte != '+' &&
@@ -754,7 +784,9 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
break;
}
Advance(1);
}
token_opl = byte;
}
//- rjf: reserved symbols
@@ -764,8 +796,8 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
token_flags = MD_TokenFlag_Reserved;
token_start = byte;
token_opl = byte+1;
byte += 1;
Advance(1);
token_opl = byte;
}
//- rjf: bad characters in all other cases
@@ -773,21 +805,21 @@ md_tokenize_from_text(Arena *arena, String8 text)
{
token_flags = MD_TokenFlag_BadCharacter;
token_start = byte;
token_opl = byte+1;
byte += 1;
Advance(1);
token_opl = byte;
}
//- rjf; push token if formed
if(token_flags != 0 && token_start != 0 && token_opl > token_start)
{
MD_Token token = {{(U64)(token_start - byte_first), (U64)(token_opl - byte_first)}, token_flags};
MD_Token token = {{(U64)(token_start - byte_first), (U64)(token_opl - byte_first)}, token_flags, token_line_num, token_col_num};
md_token_chunk_list_push(scratch.arena, &tokens, 4096, token);
}
//- rjf: push errors on unterminated comments
if(token_flags & MD_TokenFlag_BrokenComment)
{
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first);
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first, token_line_num, token_col_num);
String8 error_string = str8_lit("Unterminated comment.");
md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
}
@@ -795,12 +827,14 @@ md_tokenize_from_text(Arena *arena, String8 text)
//- rjf: push errors on unterminated strings
if(token_flags & MD_TokenFlag_BrokenStringLiteral)
{
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first);
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first, token_line_num, token_col_num);
String8 error_string = str8_lit("Unterminated string literal.");
md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
}
}
#undef Advance
//- rjf: bake, fill & return
MD_TokenizeResult result = {0};
{
@@ -821,7 +855,7 @@ md_parse_from_text_tokens(Arena *arena, String8 filename, String8 text, MD_Token
//- rjf: set up outputs
MD_MsgList msgs = {0};
MD_Node *root = md_push_node(arena, MD_NodeKind_File, 0, filename, text, 0);
MD_Node *root = md_push_node(arena, MD_NodeKind_File, 0, filename, text, 0, 0, 0);
//- rjf: set up parse rule stack
typedef enum MD_ParseWorkKind
@@ -942,7 +976,7 @@ if(work_top == 0) {work_top = &broken_work;}\
str8_match(token_string, str8_lit("\\"), 0) ||
str8_match(token_string, str8_lit(":"), 0)))
{
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num);
String8 error_string = push_str8f(arena, "Unexpected reserved symbol \"%S\".", token_string);
md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
token += 1;
@@ -956,7 +990,7 @@ if(work_top == 0) {work_top = &broken_work;}\
if(token+1 >= tokens_opl ||
!(token[1].flags & MD_TokenGroup_Label))
{
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num);
String8 error_string = str8_lit("Tag label expected after @ symbol.");
md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
token += 1;
@@ -966,7 +1000,7 @@ if(work_top == 0) {work_top = &broken_work;}\
{
String8 tag_name_raw = str8_substr(text, token[1].range);
String8 tag_name = md_content_string_from_token_flags_str8(token[1].flags, tag_name_raw);
MD_Node *node = md_push_node(arena, MD_NodeKind_Tag, md_node_flags_from_token_flags(token[1].flags), tag_name, tag_name_raw, token[0].range.min);
MD_Node *node = md_push_node(arena, MD_NodeKind_Tag, md_node_flags_from_token_flags(token[1].flags), tag_name, tag_name_raw, token[0].range.min, token[0].line_num, token[0].col_num);
DLLPushBack_NPZ(&md_nil_node, work_top->first_gathered_tag, work_top->last_gathered_tag, node, next, prev);
if(token+2 < tokens_opl && token[2].flags & MD_TokenFlag_Reserved &&
(str8_match(str8_substr(text, token[2].range), str8_lit("("), 0) ||
@@ -992,7 +1026,7 @@ if(work_top == 0) {work_top = &broken_work;}\
String8 node_string = md_content_string_from_token_flags_str8(token->flags, node_string_raw);
MD_NodeFlags flags = md_node_flags_from_token_flags(token->flags)|work_top->gathered_node_flags;
work_top->gathered_node_flags = 0;
MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, node_string, node_string_raw, token[0].range.min);
MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, node_string, node_string_raw, token[0].range.min, token[0].line_num, token[0].col_num);
node->first_tag = work_top->first_gathered_tag;
node->last_tag = work_top->last_gathered_tag;
for(MD_Node *tag = work_top->first_gathered_tag; !md_node_is_nil(tag); tag = tag->next)
@@ -1017,7 +1051,7 @@ if(work_top == 0) {work_top = &broken_work;}\
flags |= MD_NodeFlag_HasBracketLeft*!!str8_match(token_string, str8_lit("["), 0);
flags |= MD_NodeFlag_HasParenLeft*!!str8_match(token_string, str8_lit("("), 0);
work_top->gathered_node_flags = 0;
MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, str8_lit(""), str8_lit(""), token[0].range.min);
MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, str8_lit(""), str8_lit(""), token[0].range.min, token[0].line_num, token[0].col_num);
node->first_tag = work_top->first_gathered_tag;
node->last_tag = work_top->last_gathered_tag;
for(MD_Node *tag = work_top->first_gathered_tag; !md_node_is_nil(tag); tag = tag->next)
@@ -1080,7 +1114,7 @@ if(work_top == 0) {work_top = &broken_work;}\
if(work_top->counted_newlines >= 2)
{
MD_Node *node = work_top->parent;
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num);
String8 error_string = push_str8f(arena, "More than two newlines following \"%S\", which has implicitly-delimited children, resulting in an empty list of children.", node->string);
md_msg_list_push(arena, &msgs, error, MD_MsgKind_Warning, error_string);
MD_ParseWorkPop();
@@ -1121,7 +1155,7 @@ if(work_top == 0) {work_top = &broken_work;}\
//- rjf: no consumption -> unexpected token! we don't know what to do with this.
{
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num);
String8 error_string = push_str8f(arena, "Unexpected \"%S\" token.", token_string);
md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
token += 1;
src/mdesk/mdesk.h
+5 -1
View File
@@ -86,6 +86,8 @@ struct MD_Token
{
Rng1U64 range;
MD_TokenFlags flags;
U64 line_num;
U64 col_num;
};
typedef struct MD_TokenChunkNode MD_TokenChunkNode;
@@ -182,6 +184,8 @@ struct MD_Node
// rjf: source code info
U64 src_offset;
U64 line_num;
U64 col_num;
// rjf: user-controlled generation number
//
@@ -284,7 +288,7 @@ internal MD_NodeRec md_node_rec_depth_first(MD_Node *node, MD_Node *subtree_root
#define md_node_rec_depth_first_pre_rev(node, subtree_root) md_node_rec_depth_first((node), (subtree_root), OffsetOf(MD_Node, last), OffsetOf(MD_Node, prev))
//- rjf: tree building
internal MD_Node *md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset);
internal MD_Node *md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset, U64 line_num, U64 col_num);
internal void md_node_insert_child(MD_Node *parent, MD_Node *prev_child, MD_Node *node);
internal void md_node_insert_tag(MD_Node *parent, MD_Node *prev_child, MD_Node *node);
internal void md_node_push_child(MD_Node *parent, MD_Node *node);
src/torture/torture_main.c
+1
View File
@@ -168,6 +168,7 @@
#include "torture.c"
#include "torture_base.c"
#include "torture_md.c"
#include "torture_radlink.c"
#include "torture_dwarf.c"
#include "torture_d2r.c"
src/torture/torture_md.c
+146
View File
@@ -0,0 +1,146 @@
#define T_Group "MD"
TEST(md_tokenizer)
{
MD_TokenizeResult result;
// empty
result = md_tokenize_from_text(arena, str8_lit(""));
T_Ok(result.tokens.count == 0);
T_Ok(result.msgs.count == 0);
// identifiers
char *ids[] = {
"a",
"abc",
"_",
"_abc",
"abc123",
"abc_def123",
};
for EachElement(i, ids) {
String8 in = str8_cstring(ids[i]);
result = md_tokenize_from_text(arena, in);
T_Ok(result.tokens.count == 1);
T_Ok(result.tokens.v[0].range.min == 0);
T_Ok(result.tokens.v[0].range.max == in.size);
T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Identifier);
T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Identifier) == 0);
T_Ok(result.tokens.v[0].line_num == 1);
T_Ok(result.tokens.v[0].col_num == 1);
T_Ok(result.msgs.count == 0);
}
// numerics
char *numerics[] = {
"0",
"123",
"123.456",
".5",
"-1",
"-123.45"
"123_abc",
"1.2.3",
};
for EachElement(i, numerics) {
String8 in = str8_cstring(numerics[i]);
result = md_tokenize_from_text(arena, in);
T_Ok(result.tokens.count == 1);
T_Ok(result.tokens.v[0].range.min == 0);
T_Ok(result.tokens.v[0].range.max == in.size);
T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Numeric);
T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Numeric) == 0);
T_Ok(result.tokens.v[0].line_num == 1);
T_Ok(result.tokens.v[0].col_num == 1);
T_Ok(result.msgs.count == 0);
}
// comments
char *good_comments[] = {
"/**/",
"/* */",
"/* abc */",
"/* ** */",
"/**/",
"/**************/",
"/*abc*/",
"//foo",
"// bar",
"//"
};
B32 good_comments_passed = 0;
for EachElement(i, good_comments) {
String8 in = str8_cstring(good_comments[i]);
result = md_tokenize_from_text(arena, in);
T_Ok(result.tokens.count == 1);
T_Ok(dim_1u64(result.tokens.v[0].range) == in.size);
T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Comment);
T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Comment) == 0);
T_Ok(result.tokens.v[0].line_num == 1);
T_Ok(result.tokens.v[0].col_num == 1);
T_Ok(result.msgs.count == 0);
}
char *broken_comments[] = {
"/*",
"/*/",
};
for EachElement(i, broken_comments) {
String8 in = str8_cstring(broken_comments[i]);
result = md_tokenize_from_text(arena, in);
T_Ok(result.msgs.worst_message_kind == MD_MsgKind_Error);
T_Ok(result.msgs.count > 0);
}
// unterminated string
result = md_tokenize_from_text(arena, str8_lit("\"abc\nc"));
T_Ok(result.msgs.worst_message_kind == MD_MsgKind_Error);
T_Ok(result.msgs.count > 0);
// terminated string
//
// TODO: strings with \n in the middle are not correctly tokenized
result = md_tokenize_from_text(arena, str8_lit("\"abc\""));
T_Ok(result.msgs.count == 0);
// line column tracking
{
result = md_tokenize_from_text(arena, str8_lit("a\nb"));
T_Ok(result.tokens.count == 3);
// verify 'a'
T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Identifier);
T_Ok(result.tokens.v[0].line_num == 1);
T_Ok(result.tokens.v[0].col_num == 1);
// verify '\n'
T_Ok(result.tokens.v[1].flags & MD_TokenFlag_Newline);
T_Ok(result.tokens.v[1].line_num == 1);
T_Ok(result.tokens.v[1].col_num == 2);
// verify 'b'
T_Ok(result.tokens.v[2].flags & MD_TokenFlag_Identifier);
T_Ok(result.tokens.v[2].line_num == 2);
T_Ok(result.tokens.v[2].col_num == 1);
}
{
result = md_tokenize_from_text(arena, str8_lit(" abc"));
T_Ok(result.tokens.count == 2);
T_Ok(result.msgs.count == 0);
// verify white space
T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Whitespace);
T_Ok(result.tokens.v[0].line_num == 1);
T_Ok(result.tokens.v[0].col_num == 1);
// verify identifier
T_Ok(result.tokens.v[1].flags & MD_TokenFlag_Identifier);
T_Ok(result.tokens.v[1].line_num == 1);
T_Ok(result.tokens.v[1].col_num == 5);
}
}
#undef T_Group