diff --git a/src/mdesk/mdesk.c b/src/mdesk/mdesk.c index efe2a52f..cab21d42 100644 --- a/src/mdesk/mdesk.c +++ b/src/mdesk/mdesk.c @@ -185,7 +185,7 @@ md_node_rec_depth_first(MD_Node *node, MD_Node *subtree_root, U64 child_off, U64 //- rjf: tree building internal MD_Node * -md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset) +md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset, U64 line_num, U64 col_num) { MD_Node *node = push_array(arena, MD_Node, 1); node->first = node->last = node->parent = node->next = node->prev = node->first_tag = node->last_tag = &md_nil_node; @@ -194,6 +194,8 @@ md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, node->string = string; node->raw_string = raw_string; node->src_offset = src_offset; + node->line_num = line_num; + node->col_num = col_num; return node; } @@ -522,9 +524,24 @@ md_tokenize_from_text(Arena *arena, String8 text) Temp scratch = scratch_begin(&arena, 1); MD_TokenChunkList tokens = {0}; MD_MsgList msgs = {0}; + U8 *byte_first = text.str; U8 *byte_opl = byte_first + text.size; U8 *byte = byte_first; + + U64 line_num = 1; + U64 col_num = 1; + +#define Advance(n) do { \ + for(U64 i = 0; i < n && byte < byte_opl; i += 1) { \ + if(*byte == '\n') { \ + line_num += 1; \ + col_num = 0; \ + } \ + col_num += 1; \ + byte += 1; \ + } \ +} while(0) //- rjf: scan string & produce tokens for(;byte < byte_opl;) @@ -532,22 +549,22 @@ md_tokenize_from_text(Arena *arena, String8 text) MD_TokenFlags token_flags = 0; U8 *token_start = 0; U8 *token_opl = 0; + + U64 token_line_num = line_num; + U64 token_col_num = col_num; //- rjf: whitespace if(token_flags == 0 && (*byte == ' ' || *byte == '\t' || *byte == '\v' || *byte == '\r')) { token_flags = MD_TokenFlag_Whitespace; token_start = byte; - token_opl = byte; - byte += 1; - for(;byte <= byte_opl; byte += 1) + + for(; byte < byte_opl && (*byte == ' ' || *byte == '\t' || *byte == '\v' || *byte == '\r');) { - token_opl += 1; - if(byte == byte_opl || (*byte != ' ' && *byte != '\t' && *byte != '\v' && *byte != '\r')) - { - break; - } + Advance(1); } + + token_opl = byte; } //- rjf: newlines @@ -555,8 +572,8 @@ md_tokenize_from_text(Arena *arena, String8 text) { token_flags = MD_TokenFlag_Newline; token_start = byte; - token_opl = byte+1; - byte += 1; + Advance(1); + token_opl = byte; } //- rjf: single-line comments @@ -564,16 +581,11 @@ md_tokenize_from_text(Arena *arena, String8 text) { token_flags = MD_TokenFlag_Comment; token_start = byte; - token_opl = byte+2; - byte += 2; + Advance(2); + B32 escaped = 0; - for(;byte <= byte_opl; byte += 1) + for(;byte < byte_opl;) { - token_opl += 1; - if(byte == byte_opl) - { - break; - } if(escaped) { escaped = 0; @@ -589,7 +601,10 @@ md_tokenize_from_text(Arena *arena, String8 text) escaped = 1; } } + Advance(1); } + + token_opl = byte; } //- rjf: multi-line comments @@ -597,22 +612,27 @@ md_tokenize_from_text(Arena *arena, String8 text) { token_flags = MD_TokenFlag_Comment; token_start = byte; - token_opl = byte+2; - byte += 2; - for(;byte <= byte_opl; byte += 1) + + Advance(2); + + B32 missing_close = 1; + for(;byte < byte_opl;) { - token_opl += 1; - if(byte == byte_opl) - { - token_flags |= MD_TokenFlag_BrokenComment; - break; - } if(byte+1 < byte_opl && byte[0] == '*' && byte[1] == '/') { - token_opl += 2; + missing_close = 0; + Advance(2); break; } + Advance(1); } + + if(missing_close) + { + token_flags |= MD_TokenFlag_BrokenComment; + } + + token_opl = byte; } //- rjf: identifiers @@ -623,11 +643,11 @@ md_tokenize_from_text(Arena *arena, String8 text) { token_flags = MD_TokenFlag_Identifier; token_start = byte; - token_opl = byte; - byte += 1; - for(;byte <= byte_opl; byte += 1) + + Advance(1); + + for(;byte <= byte_opl; ) { - token_opl += 1; if(byte == byte_opl || (!('A' <= *byte && *byte <= 'Z') && !('a' <= *byte && *byte <= 'z') && @@ -637,7 +657,10 @@ md_tokenize_from_text(Arena *arena, String8 text) { break; } + Advance(1); } + + token_opl = byte; } //- rjf: numerics @@ -648,11 +671,11 @@ md_tokenize_from_text(Arena *arena, String8 text) { token_flags = MD_TokenFlag_Numeric; token_start = byte; - token_opl = byte; - byte += 1; - for(;byte <= byte_opl; byte += 1) + + Advance(1); + + for(;byte <= byte_opl;) { - token_opl += 1; if(byte == byte_opl || (!('A' <= *byte && *byte <= 'Z') && !('a' <= *byte && *byte <= 'z') && @@ -662,7 +685,10 @@ md_tokenize_from_text(Arena *arena, String8 text) { break; } + Advance(1); } + + token_opl = byte; } //- rjf: triplet string literals @@ -677,9 +703,10 @@ md_tokenize_from_text(Arena *arena, String8 text) token_flags |= (literal_style == '"')*MD_TokenFlag_StringDoubleQuote; token_flags |= (literal_style == '`')*MD_TokenFlag_StringTick; token_start = byte; - token_opl = byte+3; - byte += 3; - for(;byte <= byte_opl; byte += 1) + + Advance(3); + + for(;byte <= byte_opl;) { if(byte == byte_opl) { @@ -689,11 +716,14 @@ md_tokenize_from_text(Arena *arena, String8 text) } if(byte+2 < byte_opl && (byte[0] == literal_style && byte[1] == literal_style && byte[2] == literal_style)) { - byte += 3; - token_opl = byte; + Advance(3); break; } + + Advance(1); } + + token_opl = byte; } //- rjf: singlet string literals @@ -705,14 +735,14 @@ md_tokenize_from_text(Arena *arena, String8 text) token_flags |= (literal_style == '"')*MD_TokenFlag_StringDoubleQuote; token_flags |= (literal_style == '`')*MD_TokenFlag_StringTick; token_start = byte; - token_opl = byte+1; - byte += 1; + + Advance(1); + B32 escaped = 0; - for(;byte <= byte_opl; byte += 1) + for(;byte <= byte_opl;) { if(byte == byte_opl || *byte == '\n') { - token_opl = byte; token_flags |= MD_TokenFlag_BrokenStringLiteral; break; } @@ -722,15 +752,17 @@ md_tokenize_from_text(Arena *arena, String8 text) } else if(!escaped && byte[0] == literal_style) { - token_opl = byte+1; - byte += 1; + Advance(1); break; } else if(escaped) { escaped = 0; } + Advance(1); } + + token_opl = byte; } //- rjf: non-reserved symbols @@ -741,11 +773,9 @@ md_tokenize_from_text(Arena *arena, String8 text) { token_flags = MD_TokenFlag_Symbol; token_start = byte; - token_opl = byte; - byte += 1; - for(;byte <= byte_opl; byte += 1) + Advance(1); + for(;byte <= byte_opl;) { - token_opl += 1; if(byte == byte_opl || (*byte != '~' && *byte != '!' && *byte != '$' && *byte != '%' && *byte != '^' && *byte != '&' && *byte != '*' && *byte != '-' && *byte != '=' && *byte != '+' && @@ -754,7 +784,9 @@ md_tokenize_from_text(Arena *arena, String8 text) { break; } + Advance(1); } + token_opl = byte; } //- rjf: reserved symbols @@ -764,8 +796,8 @@ md_tokenize_from_text(Arena *arena, String8 text) { token_flags = MD_TokenFlag_Reserved; token_start = byte; - token_opl = byte+1; - byte += 1; + Advance(1); + token_opl = byte; } //- rjf: bad characters in all other cases @@ -773,21 +805,21 @@ md_tokenize_from_text(Arena *arena, String8 text) { token_flags = MD_TokenFlag_BadCharacter; token_start = byte; - token_opl = byte+1; - byte += 1; + Advance(1); + token_opl = byte; } //- rjf; push token if formed if(token_flags != 0 && token_start != 0 && token_opl > token_start) { - MD_Token token = {{(U64)(token_start - byte_first), (U64)(token_opl - byte_first)}, token_flags}; + MD_Token token = {{(U64)(token_start - byte_first), (U64)(token_opl - byte_first)}, token_flags, token_line_num, token_col_num}; md_token_chunk_list_push(scratch.arena, &tokens, 4096, token); } //- rjf: push errors on unterminated comments if(token_flags & MD_TokenFlag_BrokenComment) { - MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first); + MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first, token_line_num, token_col_num); String8 error_string = str8_lit("Unterminated comment."); md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string); } @@ -795,11 +827,13 @@ md_tokenize_from_text(Arena *arena, String8 text) //- rjf: push errors on unterminated strings if(token_flags & MD_TokenFlag_BrokenStringLiteral) { - MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first); + MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first, token_line_num, token_col_num); String8 error_string = str8_lit("Unterminated string literal."); md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string); } } + +#undef Advance //- rjf: bake, fill & return MD_TokenizeResult result = {0}; @@ -821,7 +855,7 @@ md_parse_from_text_tokens(Arena *arena, String8 filename, String8 text, MD_Token //- rjf: set up outputs MD_MsgList msgs = {0}; - MD_Node *root = md_push_node(arena, MD_NodeKind_File, 0, filename, text, 0); + MD_Node *root = md_push_node(arena, MD_NodeKind_File, 0, filename, text, 0, 0, 0); //- rjf: set up parse rule stack typedef enum MD_ParseWorkKind @@ -942,7 +976,7 @@ if(work_top == 0) {work_top = &broken_work;}\ str8_match(token_string, str8_lit("\\"), 0) || str8_match(token_string, str8_lit(":"), 0))) { - MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min); + MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num); String8 error_string = push_str8f(arena, "Unexpected reserved symbol \"%S\".", token_string); md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string); token += 1; @@ -956,7 +990,7 @@ if(work_top == 0) {work_top = &broken_work;}\ if(token+1 >= tokens_opl || !(token[1].flags & MD_TokenGroup_Label)) { - MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min); + MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num); String8 error_string = str8_lit("Tag label expected after @ symbol."); md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string); token += 1; @@ -966,7 +1000,7 @@ if(work_top == 0) {work_top = &broken_work;}\ { String8 tag_name_raw = str8_substr(text, token[1].range); String8 tag_name = md_content_string_from_token_flags_str8(token[1].flags, tag_name_raw); - MD_Node *node = md_push_node(arena, MD_NodeKind_Tag, md_node_flags_from_token_flags(token[1].flags), tag_name, tag_name_raw, token[0].range.min); + MD_Node *node = md_push_node(arena, MD_NodeKind_Tag, md_node_flags_from_token_flags(token[1].flags), tag_name, tag_name_raw, token[0].range.min, token[0].line_num, token[0].col_num); DLLPushBack_NPZ(&md_nil_node, work_top->first_gathered_tag, work_top->last_gathered_tag, node, next, prev); if(token+2 < tokens_opl && token[2].flags & MD_TokenFlag_Reserved && (str8_match(str8_substr(text, token[2].range), str8_lit("("), 0) || @@ -992,7 +1026,7 @@ if(work_top == 0) {work_top = &broken_work;}\ String8 node_string = md_content_string_from_token_flags_str8(token->flags, node_string_raw); MD_NodeFlags flags = md_node_flags_from_token_flags(token->flags)|work_top->gathered_node_flags; work_top->gathered_node_flags = 0; - MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, node_string, node_string_raw, token[0].range.min); + MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, node_string, node_string_raw, token[0].range.min, token[0].line_num, token[0].col_num); node->first_tag = work_top->first_gathered_tag; node->last_tag = work_top->last_gathered_tag; for(MD_Node *tag = work_top->first_gathered_tag; !md_node_is_nil(tag); tag = tag->next) @@ -1017,7 +1051,7 @@ if(work_top == 0) {work_top = &broken_work;}\ flags |= MD_NodeFlag_HasBracketLeft*!!str8_match(token_string, str8_lit("["), 0); flags |= MD_NodeFlag_HasParenLeft*!!str8_match(token_string, str8_lit("("), 0); work_top->gathered_node_flags = 0; - MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, str8_lit(""), str8_lit(""), token[0].range.min); + MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, str8_lit(""), str8_lit(""), token[0].range.min, token[0].line_num, token[0].col_num); node->first_tag = work_top->first_gathered_tag; node->last_tag = work_top->last_gathered_tag; for(MD_Node *tag = work_top->first_gathered_tag; !md_node_is_nil(tag); tag = tag->next) @@ -1080,7 +1114,7 @@ if(work_top == 0) {work_top = &broken_work;}\ if(work_top->counted_newlines >= 2) { MD_Node *node = work_top->parent; - MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min); + MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num); String8 error_string = push_str8f(arena, "More than two newlines following \"%S\", which has implicitly-delimited children, resulting in an empty list of children.", node->string); md_msg_list_push(arena, &msgs, error, MD_MsgKind_Warning, error_string); MD_ParseWorkPop(); @@ -1121,7 +1155,7 @@ if(work_top == 0) {work_top = &broken_work;}\ //- rjf: no consumption -> unexpected token! we don't know what to do with this. { - MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min); + MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num); String8 error_string = push_str8f(arena, "Unexpected \"%S\" token.", token_string); md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string); token += 1; diff --git a/src/mdesk/mdesk.h b/src/mdesk/mdesk.h index e7a55473..984c471b 100644 --- a/src/mdesk/mdesk.h +++ b/src/mdesk/mdesk.h @@ -86,6 +86,8 @@ struct MD_Token { Rng1U64 range; MD_TokenFlags flags; + U64 line_num; + U64 col_num; }; typedef struct MD_TokenChunkNode MD_TokenChunkNode; @@ -182,6 +184,8 @@ struct MD_Node // rjf: source code info U64 src_offset; + U64 line_num; + U64 col_num; // rjf: user-controlled generation number // @@ -284,7 +288,7 @@ internal MD_NodeRec md_node_rec_depth_first(MD_Node *node, MD_Node *subtree_root #define md_node_rec_depth_first_pre_rev(node, subtree_root) md_node_rec_depth_first((node), (subtree_root), OffsetOf(MD_Node, last), OffsetOf(MD_Node, prev)) //- rjf: tree building -internal MD_Node *md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset); +internal MD_Node *md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset, U64 line_num, U64 col_num); internal void md_node_insert_child(MD_Node *parent, MD_Node *prev_child, MD_Node *node); internal void md_node_insert_tag(MD_Node *parent, MD_Node *prev_child, MD_Node *node); internal void md_node_push_child(MD_Node *parent, MD_Node *node); diff --git a/src/torture/torture_main.c b/src/torture/torture_main.c index 04f3ba3e..f6f505b9 100644 --- a/src/torture/torture_main.c +++ b/src/torture/torture_main.c @@ -168,6 +168,7 @@ #include "torture.c" #include "torture_base.c" +#include "torture_md.c" #include "torture_radlink.c" #include "torture_dwarf.c" #include "torture_d2r.c" diff --git a/src/torture/torture_md.c b/src/torture/torture_md.c new file mode 100644 index 00000000..58eac35c --- /dev/null +++ b/src/torture/torture_md.c @@ -0,0 +1,146 @@ +#define T_Group "MD" + +TEST(md_tokenizer) +{ + MD_TokenizeResult result; + + // empty + result = md_tokenize_from_text(arena, str8_lit("")); + T_Ok(result.tokens.count == 0); + T_Ok(result.msgs.count == 0); + + // identifiers + char *ids[] = { + "a", + "abc", + "_", + "_abc", + "abc123", + "abc_def123", + }; + for EachElement(i, ids) { + String8 in = str8_cstring(ids[i]); + result = md_tokenize_from_text(arena, in); + + T_Ok(result.tokens.count == 1); + T_Ok(result.tokens.v[0].range.min == 0); + T_Ok(result.tokens.v[0].range.max == in.size); + T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Identifier); + T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Identifier) == 0); + T_Ok(result.tokens.v[0].line_num == 1); + T_Ok(result.tokens.v[0].col_num == 1); + T_Ok(result.msgs.count == 0); + } + + // numerics + char *numerics[] = { + "0", + "123", + "123.456", + ".5", + "-1", + "-123.45" + "123_abc", + "1.2.3", + }; + for EachElement(i, numerics) { + String8 in = str8_cstring(numerics[i]); + result = md_tokenize_from_text(arena, in); + + T_Ok(result.tokens.count == 1); + T_Ok(result.tokens.v[0].range.min == 0); + T_Ok(result.tokens.v[0].range.max == in.size); + T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Numeric); + T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Numeric) == 0); + T_Ok(result.tokens.v[0].line_num == 1); + T_Ok(result.tokens.v[0].col_num == 1); + T_Ok(result.msgs.count == 0); + } + + // comments + char *good_comments[] = { + "/**/", + "/* */", + "/* abc */", + "/* ** */", + "/**/", + "/**************/", + "/*abc*/", + "//foo", + "// bar", + "//" + }; + B32 good_comments_passed = 0; + for EachElement(i, good_comments) { + String8 in = str8_cstring(good_comments[i]); + result = md_tokenize_from_text(arena, in); + + T_Ok(result.tokens.count == 1); + T_Ok(dim_1u64(result.tokens.v[0].range) == in.size); + T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Comment); + T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Comment) == 0); + T_Ok(result.tokens.v[0].line_num == 1); + T_Ok(result.tokens.v[0].col_num == 1); + T_Ok(result.msgs.count == 0); + } + + char *broken_comments[] = { + "/*", + "/*/", + }; + for EachElement(i, broken_comments) { + String8 in = str8_cstring(broken_comments[i]); + result = md_tokenize_from_text(arena, in); + T_Ok(result.msgs.worst_message_kind == MD_MsgKind_Error); + T_Ok(result.msgs.count > 0); + } + + // unterminated string + result = md_tokenize_from_text(arena, str8_lit("\"abc\nc")); + T_Ok(result.msgs.worst_message_kind == MD_MsgKind_Error); + T_Ok(result.msgs.count > 0); + + // terminated string + // + // TODO: strings with \n in the middle are not correctly tokenized + result = md_tokenize_from_text(arena, str8_lit("\"abc\"")); + T_Ok(result.msgs.count == 0); + + // line column tracking + { + result = md_tokenize_from_text(arena, str8_lit("a\nb")); + T_Ok(result.tokens.count == 3); + + // verify 'a' + T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Identifier); + T_Ok(result.tokens.v[0].line_num == 1); + T_Ok(result.tokens.v[0].col_num == 1); + + // verify '\n' + T_Ok(result.tokens.v[1].flags & MD_TokenFlag_Newline); + T_Ok(result.tokens.v[1].line_num == 1); + T_Ok(result.tokens.v[1].col_num == 2); + + // verify 'b' + T_Ok(result.tokens.v[2].flags & MD_TokenFlag_Identifier); + T_Ok(result.tokens.v[2].line_num == 2); + T_Ok(result.tokens.v[2].col_num == 1); + } + { + result = md_tokenize_from_text(arena, str8_lit(" abc")); + T_Ok(result.tokens.count == 2); + T_Ok(result.msgs.count == 0); + + // verify white space + T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Whitespace); + T_Ok(result.tokens.v[0].line_num == 1); + T_Ok(result.tokens.v[0].col_num == 1); + + // verify identifier + T_Ok(result.tokens.v[1].flags & MD_TokenFlag_Identifier); + T_Ok(result.tokens.v[1].line_num == 1); + T_Ok(result.tokens.v[1].col_num == 5); + } +} + +#undef T_Group