export line and column numbers from MD

src/mdesk/mdesk.c

@@ -185,7 +185,7 @@ md_node_rec_depth_first(MD_Node *node, MD_Node *subtree_root, U64 child_off, U64
 //- rjf: tree building
 
 internal MD_Node *
-md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset)
+md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset, U64 line_num, U64 col_num)
 {
   MD_Node *node = push_array(arena, MD_Node, 1);
   node->first = node->last = node->parent = node->next = node->prev = node->first_tag = node->last_tag = &md_nil_node;
@@ -194,6 +194,8 @@ md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string,
   node->string     = string;
   node->raw_string = raw_string;
   node->src_offset = src_offset;
+  node->line_num   = line_num;
+  node->col_num    = col_num;
   return node;
 }
 
@@ -522,10 +524,25 @@ md_tokenize_from_text(Arena *arena, String8 text)
   Temp scratch = scratch_begin(&arena, 1);
   MD_TokenChunkList tokens = {0};
   MD_MsgList msgs = {0};
+
   U8 *byte_first = text.str;
   U8 *byte_opl = byte_first + text.size;
   U8 *byte = byte_first;
 
+  U64 line_num = 1;
+  U64 col_num = 1;
+
+#define Advance(n) do {                              \
+  for(U64 i = 0; i < n && byte < byte_opl; i += 1) { \
+    if(*byte == '\n') {                              \
+      line_num += 1;                                 \
+      col_num   = 0;                                 \
+    }                                                \
+    col_num += 1;                                    \
+    byte    += 1;                                    \
+  }                                                  \
+} while(0)
+  
   //- rjf: scan string & produce tokens
   for(;byte < byte_opl;)
   {
@@ -533,21 +550,21 @@ md_tokenize_from_text(Arena *arena, String8 text)
     U8 *token_start = 0;
     U8 *token_opl = 0;
 
+    U64 token_line_num = line_num;
+    U64 token_col_num = col_num;
+    
     //- rjf: whitespace
     if(token_flags == 0 && (*byte == ' ' || *byte == '\t' || *byte == '\v' || *byte == '\r'))
     {
       token_flags = MD_TokenFlag_Whitespace;
       token_start = byte;
+
+      for(; byte < byte_opl && (*byte == ' ' || *byte == '\t' || *byte == '\v' || *byte == '\r');)
+      {
+        Advance(1);
+      }
+
       token_opl = byte;
-      byte += 1;
-      for(;byte <= byte_opl; byte += 1)
-      {
-        token_opl += 1;
-        if(byte == byte_opl || (*byte != ' ' && *byte != '\t' && *byte != '\v' && *byte != '\r'))
-        {
-          break;
-        }
-      }
     }
     
     //- rjf: newlines
@@ -555,8 +572,8 @@ md_tokenize_from_text(Arena *arena, String8 text)
     {
       token_flags = MD_TokenFlag_Newline;
       token_start = byte;
-      token_opl = byte+1;
+      Advance(1);
-      byte += 1;
+      token_opl = byte;
     }
     
     //- rjf: single-line comments
@@ -564,16 +581,11 @@ md_tokenize_from_text(Arena *arena, String8 text)
     {
       token_flags = MD_TokenFlag_Comment;
       token_start = byte;
-      token_opl = byte+2;
+      Advance(2);
-      byte += 2;
+
       B32 escaped = 0;
-      for(;byte <= byte_opl; byte += 1)
+      for(;byte < byte_opl;)
       {
-        token_opl += 1;
-        if(byte == byte_opl)
-        {
-          break;
-        }
         if(escaped)
         {
           escaped = 0;
@@ -589,7 +601,10 @@ md_tokenize_from_text(Arena *arena, String8 text)
             escaped = 1;
           }
         }
+        Advance(1);
       }
+
+      token_opl = byte;
     }
     
     //- rjf: multi-line comments
@@ -597,22 +612,27 @@ md_tokenize_from_text(Arena *arena, String8 text)
     {
       token_flags = MD_TokenFlag_Comment;
       token_start = byte;
-      token_opl = byte+2;
+      
-      byte += 2;
+      Advance(2);
-      for(;byte <= byte_opl; byte += 1)
+
+      B32 missing_close = 1;
+      for(;byte < byte_opl;)
       {
-        token_opl += 1;
-        if(byte == byte_opl)
-        {
-          token_flags |= MD_TokenFlag_BrokenComment;
-          break;
-        }
         if(byte+1 < byte_opl && byte[0] == '*' && byte[1] == '/')
         {
-          token_opl += 2;
+          missing_close = 0;
+          Advance(2);
           break;
         }
+        Advance(1);
       }
+
+      if(missing_close)
+      {
+        token_flags |= MD_TokenFlag_BrokenComment;
+      }
+
+      token_opl = byte;
     }
     
     //- rjf: identifiers
@@ -623,11 +643,11 @@ md_tokenize_from_text(Arena *arena, String8 text)
     {
       token_flags = MD_TokenFlag_Identifier;
       token_start = byte;
-      token_opl = byte;
+
-      byte += 1;
+      Advance(1);
-      for(;byte <= byte_opl; byte += 1)
+
+      for(;byte <= byte_opl; )
       {
-        token_opl += 1;
         if(byte == byte_opl ||
            (!('A' <= *byte && *byte <= 'Z') &&
             !('a' <= *byte && *byte <= 'z') &&
@@ -637,7 +657,10 @@ md_tokenize_from_text(Arena *arena, String8 text)
         {
           break;
         }
+        Advance(1);
       }
+
+      token_opl = byte;
     }
     
     //- rjf: numerics
@@ -648,11 +671,11 @@ md_tokenize_from_text(Arena *arena, String8 text)
     {
       token_flags = MD_TokenFlag_Numeric;
       token_start = byte;
-      token_opl = byte;
+
-      byte += 1;
+      Advance(1);
-      for(;byte <= byte_opl; byte += 1)
+
+      for(;byte <= byte_opl;)
       {
-        token_opl += 1;
         if(byte == byte_opl ||
            (!('A' <= *byte && *byte <= 'Z') &&
             !('a' <= *byte && *byte <= 'z') &&
@@ -662,7 +685,10 @@ md_tokenize_from_text(Arena *arena, String8 text)
         {
           break;
         }
+        Advance(1);
       }
+
+      token_opl = byte;
     }
     
     //- rjf: triplet string literals
@@ -677,9 +703,10 @@ md_tokenize_from_text(Arena *arena, String8 text)
       token_flags |= (literal_style ==  '"')*MD_TokenFlag_StringDoubleQuote;
       token_flags |= (literal_style ==  '`')*MD_TokenFlag_StringTick;
       token_start = byte;
-      token_opl = byte+3;
+
-      byte += 3;
+      Advance(3);
-      for(;byte <= byte_opl; byte += 1)
+
+      for(;byte <= byte_opl;)
       {
         if(byte == byte_opl)
         {
@@ -689,11 +716,14 @@ md_tokenize_from_text(Arena *arena, String8 text)
         }
         if(byte+2 < byte_opl && (byte[0] == literal_style && byte[1] == literal_style && byte[2] == literal_style))
         {
-          byte += 3;
+          Advance(3);
-          token_opl = byte;
           break;
         }
+
+        Advance(1);
       }
+
+      token_opl = byte;
     }
     
     //- rjf: singlet string literals
@@ -705,14 +735,14 @@ md_tokenize_from_text(Arena *arena, String8 text)
       token_flags |= (literal_style ==  '"')*MD_TokenFlag_StringDoubleQuote;
       token_flags |= (literal_style ==  '`')*MD_TokenFlag_StringTick;
       token_start = byte;
-      token_opl = byte+1;
+
-      byte += 1;
+      Advance(1);
+
       B32 escaped = 0;
-      for(;byte <= byte_opl; byte += 1)
+      for(;byte <= byte_opl;)
       {
         if(byte == byte_opl || *byte == '\n')
         {
-          token_opl = byte;
           token_flags |= MD_TokenFlag_BrokenStringLiteral;
           break;
         }
@@ -722,15 +752,17 @@ md_tokenize_from_text(Arena *arena, String8 text)
         }
         else if(!escaped && byte[0] == literal_style)
         {
-          token_opl = byte+1;
+          Advance(1);
-          byte += 1;
           break;
         }
         else if(escaped)
         {
           escaped = 0;
         }
+        Advance(1);
       }
+
+      token_opl = byte;
     }
     
     //- rjf: non-reserved symbols
@@ -741,11 +773,9 @@ md_tokenize_from_text(Arena *arena, String8 text)
     {
       token_flags = MD_TokenFlag_Symbol;
       token_start = byte;
-      token_opl = byte;
+      Advance(1);
-      byte += 1;
+      for(;byte <= byte_opl;)
-      for(;byte <= byte_opl; byte += 1)
       {
-        token_opl += 1;
         if(byte == byte_opl ||
            (*byte != '~' && *byte != '!' && *byte != '$' && *byte != '%' && *byte != '^' &&
             *byte != '&' && *byte != '*' && *byte != '-' && *byte != '=' && *byte != '+' &&
@@ -754,7 +784,9 @@ md_tokenize_from_text(Arena *arena, String8 text)
         {
           break;
         }
+        Advance(1);
       }
+      token_opl = byte;
     }
     
     //- rjf: reserved symbols
@@ -764,8 +796,8 @@ md_tokenize_from_text(Arena *arena, String8 text)
     {
       token_flags = MD_TokenFlag_Reserved;
       token_start = byte;
-      token_opl = byte+1;
+      Advance(1);
-      byte += 1;
+      token_opl = byte;
     }
     
     //- rjf: bad characters in all other cases
@@ -773,21 +805,21 @@ md_tokenize_from_text(Arena *arena, String8 text)
     {
       token_flags = MD_TokenFlag_BadCharacter;
       token_start = byte;
-      token_opl = byte+1;
+      Advance(1);
-      byte += 1;
+      token_opl = byte;
     }
     
     //- rjf; push token if formed
     if(token_flags != 0 && token_start != 0 && token_opl > token_start)
     {
-      MD_Token token = {{(U64)(token_start - byte_first), (U64)(token_opl - byte_first)}, token_flags};
+      MD_Token token = {{(U64)(token_start - byte_first), (U64)(token_opl - byte_first)}, token_flags, token_line_num, token_col_num};
       md_token_chunk_list_push(scratch.arena, &tokens, 4096, token);
     }
     
     //- rjf: push errors on unterminated comments
     if(token_flags & MD_TokenFlag_BrokenComment)
     {
-      MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first);
+      MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first, token_line_num, token_col_num);
       String8 error_string = str8_lit("Unterminated comment.");
       md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
     }
@@ -795,12 +827,14 @@ md_tokenize_from_text(Arena *arena, String8 text)
     //- rjf: push errors on unterminated strings
     if(token_flags & MD_TokenFlag_BrokenStringLiteral)
     {
-      MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first);
+      MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first, token_line_num, token_col_num);
       String8 error_string = str8_lit("Unterminated string literal.");
       md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
     }
   }
 
+#undef Advance
+  
   //- rjf: bake, fill & return
   MD_TokenizeResult result = {0};
   {
@@ -821,7 +855,7 @@ md_parse_from_text_tokens(Arena *arena, String8 filename, String8 text, MD_Token
   
   //- rjf: set up outputs
   MD_MsgList msgs = {0};
-  MD_Node *root = md_push_node(arena, MD_NodeKind_File, 0, filename, text, 0);
+  MD_Node *root = md_push_node(arena, MD_NodeKind_File, 0, filename, text, 0, 0, 0);
   
   //- rjf: set up parse rule stack
   typedef enum MD_ParseWorkKind
@@ -942,7 +976,7 @@ if(work_top == 0) {work_top = &broken_work;}\
         str8_match(token_string, str8_lit("\\"), 0) ||
         str8_match(token_string, str8_lit(":"), 0)))
     {
-      MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
+      MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num);
       String8 error_string = push_str8f(arena, "Unexpected reserved symbol \"%S\".", token_string);
       md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
       token += 1;
@@ -956,7 +990,7 @@ if(work_top == 0) {work_top = &broken_work;}\
       if(token+1 >= tokens_opl ||
          !(token[1].flags & MD_TokenGroup_Label))
       {
-        MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
+        MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num);
         String8 error_string = str8_lit("Tag label expected after @ symbol.");
         md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
         token += 1;
@@ -966,7 +1000,7 @@ if(work_top == 0) {work_top = &broken_work;}\
       {
         String8 tag_name_raw = str8_substr(text, token[1].range);
         String8 tag_name = md_content_string_from_token_flags_str8(token[1].flags, tag_name_raw);
-        MD_Node *node = md_push_node(arena, MD_NodeKind_Tag, md_node_flags_from_token_flags(token[1].flags), tag_name, tag_name_raw, token[0].range.min);
+        MD_Node *node = md_push_node(arena, MD_NodeKind_Tag, md_node_flags_from_token_flags(token[1].flags), tag_name, tag_name_raw, token[0].range.min, token[0].line_num, token[0].col_num);
         DLLPushBack_NPZ(&md_nil_node, work_top->first_gathered_tag, work_top->last_gathered_tag, node, next, prev);
         if(token+2 < tokens_opl && token[2].flags & MD_TokenFlag_Reserved &&
            (str8_match(str8_substr(text, token[2].range), str8_lit("("), 0) ||
@@ -992,7 +1026,7 @@ if(work_top == 0) {work_top = &broken_work;}\
       String8 node_string = md_content_string_from_token_flags_str8(token->flags, node_string_raw);
       MD_NodeFlags flags = md_node_flags_from_token_flags(token->flags)|work_top->gathered_node_flags;
       work_top->gathered_node_flags = 0;
-      MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, node_string, node_string_raw, token[0].range.min);
+      MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, node_string, node_string_raw, token[0].range.min, token[0].line_num, token[0].col_num);
       node->first_tag = work_top->first_gathered_tag;
       node->last_tag = work_top->last_gathered_tag;
       for(MD_Node *tag = work_top->first_gathered_tag; !md_node_is_nil(tag); tag = tag->next)
@@ -1017,7 +1051,7 @@ if(work_top == 0) {work_top = &broken_work;}\
       flags |= MD_NodeFlag_HasBracketLeft*!!str8_match(token_string, str8_lit("["), 0);
       flags |=   MD_NodeFlag_HasParenLeft*!!str8_match(token_string, str8_lit("("), 0);
       work_top->gathered_node_flags = 0;
-      MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, str8_lit(""), str8_lit(""), token[0].range.min);
+      MD_Node *node = md_push_node(arena, MD_NodeKind_Main, flags, str8_lit(""), str8_lit(""), token[0].range.min, token[0].line_num, token[0].col_num);
       node->first_tag = work_top->first_gathered_tag;
       node->last_tag = work_top->last_gathered_tag;
       for(MD_Node *tag = work_top->first_gathered_tag; !md_node_is_nil(tag); tag = tag->next)
@@ -1080,7 +1114,7 @@ if(work_top == 0) {work_top = &broken_work;}\
       if(work_top->counted_newlines >= 2)
       {
         MD_Node *node = work_top->parent;
-        MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
+        MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num);
         String8 error_string = push_str8f(arena, "More than two newlines following \"%S\", which has implicitly-delimited children, resulting in an empty list of children.", node->string);
         md_msg_list_push(arena, &msgs, error, MD_MsgKind_Warning, error_string);
         MD_ParseWorkPop();
@@ -1121,7 +1155,7 @@ if(work_top == 0) {work_top = &broken_work;}\
     
     //- rjf: no consumption -> unexpected token! we don't know what to do with this.
     {
-      MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
+      MD_Node *error = md_push_node(arena, MD_NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min, token->line_num, token->col_num);
       String8 error_string = push_str8f(arena, "Unexpected \"%S\" token.", token_string);
       md_msg_list_push(arena, &msgs, error, MD_MsgKind_Error, error_string);
       token += 1;

src/mdesk/mdesk.h

@@ -86,6 +86,8 @@ struct MD_Token
 {
   Rng1U64 range;
   MD_TokenFlags flags;
+  U64 line_num;
+  U64 col_num;
 };
 
 typedef struct MD_TokenChunkNode MD_TokenChunkNode;
@@ -182,6 +184,8 @@ struct MD_Node
   
   // rjf: source code info
   U64 src_offset;
+  U64 line_num;
+  U64 col_num;
   
   // rjf: user-controlled generation number
   //
@@ -284,7 +288,7 @@ internal MD_NodeRec md_node_rec_depth_first(MD_Node *node, MD_Node *subtree_root
 #define md_node_rec_depth_first_pre_rev(node, subtree_root) md_node_rec_depth_first((node), (subtree_root), OffsetOf(MD_Node, last), OffsetOf(MD_Node, prev))
 
 //- rjf: tree building
-internal MD_Node *md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset);
+internal MD_Node *md_push_node(Arena *arena, MD_NodeKind kind, MD_NodeFlags flags, String8 string, String8 raw_string, U64 src_offset, U64 line_num, U64 col_num);
 internal void md_node_insert_child(MD_Node *parent, MD_Node *prev_child, MD_Node *node);
 internal void md_node_insert_tag(MD_Node *parent, MD_Node *prev_child, MD_Node *node);
 internal void md_node_push_child(MD_Node *parent, MD_Node *node);

src/torture/torture_main.c

@@ -168,6 +168,7 @@
 
 #include "torture.c"
 #include "torture_base.c"
+#include "torture_md.c"
 #include "torture_radlink.c"
 #include "torture_dwarf.c"
 #include "torture_d2r.c"

src/torture/torture_md.c

@@ -0,0 +1,146 @@
+#define T_Group "MD" 
+
+TEST(md_tokenizer)
+{
+  MD_TokenizeResult result;
+
+  // empty
+  result = md_tokenize_from_text(arena, str8_lit(""));
+  T_Ok(result.tokens.count == 0);
+  T_Ok(result.msgs.count == 0);
+
+  // identifiers
+  char *ids[] = {
+    "a",
+    "abc",
+    "_",
+    "_abc",
+    "abc123",
+    "abc_def123",
+  };
+  for EachElement(i, ids) {
+    String8 in = str8_cstring(ids[i]);
+    result = md_tokenize_from_text(arena, in);
+
+    T_Ok(result.tokens.count == 1);
+    T_Ok(result.tokens.v[0].range.min == 0);
+    T_Ok(result.tokens.v[0].range.max == in.size);
+    T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Identifier);
+    T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Identifier) == 0);
+    T_Ok(result.tokens.v[0].line_num == 1);
+    T_Ok(result.tokens.v[0].col_num == 1);
+    T_Ok(result.msgs.count == 0);
+  }
+
+  // numerics
+  char *numerics[] = {
+    "0",
+    "123",
+    "123.456",
+    ".5",
+    "-1",
+    "-123.45"
+    "123_abc",
+    "1.2.3",
+  };
+  for EachElement(i, numerics) {
+    String8 in = str8_cstring(numerics[i]);
+    result = md_tokenize_from_text(arena, in);
+
+    T_Ok(result.tokens.count == 1);
+    T_Ok(result.tokens.v[0].range.min == 0);
+    T_Ok(result.tokens.v[0].range.max == in.size);
+    T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Numeric);
+    T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Numeric) == 0);
+    T_Ok(result.tokens.v[0].line_num == 1);
+    T_Ok(result.tokens.v[0].col_num == 1);
+    T_Ok(result.msgs.count == 0);
+  }
+
+  // comments
+  char *good_comments[] = {
+    "/**/",
+    "/* */",
+    "/* abc */",
+    "/* ** */",
+    "/**/",
+    "/**************/",
+    "/*abc*/",
+    "//foo",
+    "// bar",
+    "//"
+  };
+  B32 good_comments_passed = 0;
+  for EachElement(i, good_comments) {
+    String8 in = str8_cstring(good_comments[i]);
+    result = md_tokenize_from_text(arena, in);
+
+    T_Ok(result.tokens.count == 1);
+    T_Ok(dim_1u64(result.tokens.v[0].range) == in.size);
+    T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Comment);
+    T_Ok((result.tokens.v[0].flags & ~MD_TokenFlag_Comment) == 0);
+    T_Ok(result.tokens.v[0].line_num == 1);
+    T_Ok(result.tokens.v[0].col_num == 1);
+    T_Ok(result.msgs.count == 0);
+  }
+
+  char *broken_comments[] = {
+    "/*",
+    "/*/",
+  };
+  for EachElement(i, broken_comments) {
+    String8 in = str8_cstring(broken_comments[i]);
+    result = md_tokenize_from_text(arena, in);
+    T_Ok(result.msgs.worst_message_kind == MD_MsgKind_Error);
+    T_Ok(result.msgs.count > 0);
+  }
+
+  // unterminated string
+  result = md_tokenize_from_text(arena, str8_lit("\"abc\nc"));
+  T_Ok(result.msgs.worst_message_kind == MD_MsgKind_Error);
+  T_Ok(result.msgs.count > 0);
+
+  // terminated string
+  //
+  // TODO: strings with \n in the middle are not correctly tokenized
+  result = md_tokenize_from_text(arena, str8_lit("\"abc\""));
+  T_Ok(result.msgs.count == 0);
+
+  // line column tracking
+  {
+    result = md_tokenize_from_text(arena, str8_lit("a\nb"));
+    T_Ok(result.tokens.count == 3);
+
+    // verify 'a'
+    T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Identifier);
+    T_Ok(result.tokens.v[0].line_num == 1);
+    T_Ok(result.tokens.v[0].col_num == 1);
+
+    // verify '\n'
+    T_Ok(result.tokens.v[1].flags & MD_TokenFlag_Newline);
+    T_Ok(result.tokens.v[1].line_num == 1);
+    T_Ok(result.tokens.v[1].col_num == 2);
+
+    // verify 'b'
+    T_Ok(result.tokens.v[2].flags & MD_TokenFlag_Identifier);
+    T_Ok(result.tokens.v[2].line_num == 2);
+    T_Ok(result.tokens.v[2].col_num == 1);
+  }
+  {
+    result = md_tokenize_from_text(arena, str8_lit("    abc"));
+    T_Ok(result.tokens.count == 2);
+    T_Ok(result.msgs.count == 0);
+
+    // verify white space
+    T_Ok(result.tokens.v[0].flags & MD_TokenFlag_Whitespace);
+    T_Ok(result.tokens.v[0].line_num == 1);
+    T_Ok(result.tokens.v[0].col_num == 1);
+
+    // verify identifier
+    T_Ok(result.tokens.v[1].flags & MD_TokenFlag_Identifier);
+    T_Ok(result.tokens.v[1].line_num == 1);
+    T_Ok(result.tokens.v[1].col_num == 5);
+  }
+}
+
+#undef T_Group