[impl] test for string contents; reduce all string lexing; notes about delimiter error tokens

docs/metadesk_reference.md

@@ -977,6 +977,40 @@ main:
     return: MD_b32,
 };
 
+@send(Parsing)
+@func MD_PushNodeError: {
+    ctx: *MD_ParseCtx,
+    node: *MD_Node,
+    kind: MD_MessageKind,
+    str: MD_String8,
+};
+
+@send(Parsing)
+@func MD_PushNodeErrorF: {
+    ctx: *MD_ParseCtx,
+    node: *MD_Node,
+    kind: MD_MessageKind,
+    fmt: *char,
+    "..."
+};
+
+@send(Parsing)
+@func MD_PushTokenError: {
+    ctx: *MD_ParseCtx,
+    token: MD_Token,
+    kind: MD_MessageKind,
+    str: MD_String8,
+};
+
+@send(Parsing)
+@func MD_PushTokenErrorF: {
+    ctx: *MD_ParseCtx,
+    token: MD_Token,
+    kind: MD_MessageKind,
+    fmt: *char,
+    "..."
+};
+
 @send(Parsing)
 @func MD_Parse_InitializeCtx: {
     filename: MD_String8,

source/md_impl.c

@@ -1151,6 +1151,8 @@ _MD_StringLiteralIsBalanced(MD_Token token)
     return result;
 }
 
+// TODO(allen): wouldn't the lexer have already figured this out? Why not just make
+// it a flag or a kind on the token?
 MD_PRIVATE_FUNCTION_IMPL MD_b32
 _MD_CommentIsSyntacticallyCorrect(MD_Token comment_token)
 {
@@ -1161,6 +1163,7 @@ _MD_CommentIsSyntacticallyCorrect(MD_Token comment_token)
     MD_b32 result = !incorrect;
     return result;
 }
+
 MD_PRIVATE_FUNCTION_IMPL void
 _MD_ParseTagList(MD_ParseCtx *ctx, MD_Node **first_out, MD_Node **last_out)
 {
@@ -1326,60 +1329,6 @@ MD_Parse_BumpNext(MD_ParseCtx *ctx)
     MD_Parse_Bump(ctx, MD_Parse_LexNext(ctx));
 }
 
-MD_PRIVATE_FUNCTION_IMPL MD_u8 *
-_MD_TokenizerScanEscaped(MD_u8 *at, MD_u8 *one_past_last, MD_u8 c)
-{
-    while(at < one_past_last)
-    {
-        if(*at == c || *at == '\n') break;
-        else if(at[0] == '\\' && at + 1 < one_past_last && (at[1] == c || at[1] == '\\')) at += 2;
-        else at += 1;
-    }
-    
-    if (at < one_past_last && *at == c) at += 1;
-    
-    return at;
-}
-
-MD_PRIVATE_FUNCTION_IMPL MD_u8 *
-_MD_TokenizerScanTripletEscaped(MD_u8 *at, MD_u8 *one_past_last, MD_u8 c)
-{
-    MD_u32 consecutive_c = 0;
-    while(at < one_past_last && consecutive_c < 3)
-    {
-        if(at[0] == c)
-        {
-            consecutive_c += 1;
-        }
-        else
-        {
-            consecutive_c = 0;
-            if(at[0] == '\\' && at + 1 < one_past_last)
-            {
-                if(at[1] == c || at[1] == '\\')
-                {
-                    at += 1;
-                }
-            }
-        }
-        at += 1;
-    }
-    return at;
-}
-
-MD_PRIVATE_FUNCTION_IMPL MD_u32
-_MD_ComputeTextLiteralChop(MD_u32 bytes_to_skip, MD_u8 *beg, MD_u8 *end)
-{
-    MD_u32 result = 0;
-    MD_u8 *skip_end = beg + bytes_to_skip;
-    MD_u8 *chop_beg = end - bytes_to_skip;
-    if(skip_end <= chop_beg && MD_StringMatch(MD_S8(beg, bytes_to_skip), MD_S8(chop_beg, bytes_to_skip), 0))
-    {
-        result = bytes_to_skip;
-    }
-    return result;
-}
-
 MD_FUNCTION_IMPL MD_Token
 MD_Parse_LexNext(MD_ParseCtx *ctx)
 {
@@ -1420,25 +1369,37 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
                     if (at[1] == '/')
                     {
                         
+                        // TODO(allen): This seems like an odd choice to me. What about two spaces!?
+                        // What about an extra /? I'm wondering if there are other places where we make
+                        // this kind of judgement call a lot, or is this the only one? Maybe the user
+                        // should just always skip-chop whitespace if they want to clean this kind of
+                        // thing up? They're going to have to if they ever use two spaces anyways, right?
+                        
                         // NOTE(rjf): Trim off the first //, and a space after it if one is there.
+                        if(at+2 < one_past_last &&
+                           at[2] == ' ')
                         {
-                            if(at+2 < one_past_last &&
+                            skip_n = 3;
-                               at[2] == ' ')
+                        }
-                            {
+                        else
-                                skip_n = 3;
+                        {
-                            }
+                            skip_n = 2;
-                            else
-                            {
-                                skip_n = 2;
-                            }
                         }
                         
-                        at += 2;
+                        at += skip_n;
                         token.kind = MD_TokenKind_Comment;
                         MD_TokenizerScan(*at != '\n' && *at != '\r');
                     }
                     else if (at[1] == '*')
                     {
+                        // TODO(allen): proposal:
+                        // 1. only set `kind = Comment` in the `counter == 0` case
+                        // 2. otherwise set `kind = RunOnComment` (or something)
+                        //    maybe also emit an error *from here* or signal to a system that
+                        //    runs later that there are token errors to emit.
+                        // Or: keep the `kind = Comment` but in the `counter != 0` case
+                        //     set some kind of error/unclosed/run-on flag on the token.
+                        
                         at += 2;
                         token.kind = MD_TokenKind_Comment;
                         skip_n = 2;
@@ -1468,60 +1429,106 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
                 if (token.kind == MD_TokenKind_Nil) goto symbol_lex;
             }break;
             
+            // NOTE(allen): Strings
+            case '"':
+            case '\'':
+            
             // NOTE(rjf): "Bundle-of-tokens" strings (`stuff` or ```stuff```)
             // In practice no different than a regular string, but provides an
             // alternate syntax which will allow tools like 4coder to treat the
             // contents as regular tokens.
             case '`':
             {
-                token.kind = MD_TokenKind_StringLiteral;
+                // TODO(allen): proposal:
-                if (at + 2 < one_past_last && at[1] == '`' && at[2] == '`')
+                // go see the proposal in the block comment lexer, same idea here?
-                {
-                    skip_n = 3;
-                    at = _MD_TokenizerScanTripletEscaped(at+3, one_past_last, '`');
-                }
-                else
-                {
-                    skip_n = 1;
-                    at = _MD_TokenizerScanEscaped(at+1, one_past_last, '`');
-                }
-                chop_n = _MD_ComputeTextLiteralChop(skip_n, first, at);
-            }break;
                 
-            // NOTE(allen): Strings
+                // determine delimiter setup
-            case '"':
+                MD_u8 d = *at;
-            {
+                MD_b32 is_triplet = (at + 2 < one_past_last && at[1] == d && at[2] == d);
-                token.kind = MD_TokenKind_StringLiteral;
+                
-                if (at + 2 < one_past_last && at[1] == '"' && at[2] == '"')
+                // lex triple-delimiter string
+                if (is_triplet)
                 {
                     skip_n = 3;
-                    at = _MD_TokenizerScanTripletEscaped(at+3, one_past_last, '"');
+                    at += 3;
+                    MD_u32 consecutive_d = 0;
+                    for (;;)
+                    {
+                        // fail condition
+                        if (at >= one_past_last){
+                            break;
+                        }
+                        
+                        if(at[0] == d)
+                        {
+                            consecutive_d += 1;
+                            at += 1;
+                            // close condition
+                            if (consecutive_d == 3){
+                                chop_n = 3;
+                                break;
+                            }
+                        }
+                        else
+                        {
+                            consecutive_d = 0;
+                            
+                            // escaping rule
+                            if(at[0] == '\\')
+                            {
+                                at += 1;
+                                if(at < one_past_last && (at[0] == d || at[0] == '\\'))
+                                {
+                                    at += 1;
+                                }
+                            }
+                            else{
+                                at += 1;
+                            }
+                        }
+                    }
                 }
-                else
+                
+                // lex single-delimiter string
+                if (!is_triplet)
                 {
                     skip_n = 1;
                     at += 1;
-                    at = _MD_TokenizerScanEscaped(at, one_past_last, '"');
+                    for (;at < one_past_last;)
-                }
+                    {
-                chop_n = _MD_ComputeTextLiteralChop(skip_n, first, at);
+                        // close condition
-            }break;
+                        if (*at == d){
+                            at += 1;
+                            chop_n = 1;
+                            break;
+                        }
                         
-            case '\'':
+                        // fail condition
-            {
+                        if (*at == '\n'){
-                if (at + 2 < one_past_last && at[1] == '\'' && at[2] == '\'')
+                            break;
-                {
+                        }
-                    token.kind = MD_TokenKind_StringLiteral;
+                        
-                    skip_n = 3;
+                        // escaping rule
-                    at = _MD_TokenizerScanTripletEscaped(at+3, one_past_last, '\'');
+                        if (at[0] == '\\'){
+                            at += 1;
+                            if (at < one_past_last && (at[0] == d || at[0] == '\\')){
+                                at += 1;
+                            }
+                        }
+                        else{
+                            at += 1;
+                        }
+                    }
                 }
-                else
+                
-                {
+                // set token kind
+                token.kind = MD_TokenKind_StringLiteral;
+                // TODO(allen): I don't see any place where this actually proves useful.
+                // I think it'd tidy things up to drop it. we already use this as a string
+                // in a lot of usages of metadesk.
+                if (d == '\'' && !is_triplet){
                     token.kind = MD_TokenKind_CharLiteral;
-                    skip_n = 1;
-                    at += 1;
-                    at = _MD_TokenizerScanEscaped(at, one_past_last, '\'');
                 }
-                chop_n = _MD_ComputeTextLiteralChop(skip_n, first, at);
             }break;
             
             // NOTE(allen): Identifiers, Numbers, Operators
@@ -1841,6 +1848,8 @@ MD_ParseOneNodeFromCtx(MD_ParseCtx *ctx)
             }
         }
         comment_before = comment_token.string;
+        // TODO(allen): I find this odd. Wouldn't it have been easier to generate this
+        // durring or right after the lexing phase?
         if(!_MD_CommentIsSyntacticallyCorrect(comment_token))
         {
             MD_String8 capped = MD_StringPrefix(comment_token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP);
@@ -1972,6 +1981,8 @@ MD_ParseOneNodeFromCtx(MD_ParseCtx *ctx)
             }
         }
         comment_after = comment_token.string;
+        // TODO(allen): I find this odd. Wouldn't it have been easier to generate this
+        // durring or right after the lexing phase?
         if(!_MD_CommentIsSyntacticallyCorrect(comment_token))
         {
             MD_String8 capped = MD_StringPrefix(comment_token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP);

tests/sanity_tests.c

@@ -551,6 +551,38 @@ int main(void)
         }
     }
     
+    Test("String Inner & Outer")
+    {
+        MD_String8 samples[6] = {
+            MD_S8Lit("'foo-bar'"),
+            MD_S8Lit("'''foo-bar'''"),
+            MD_S8Lit("\"foo-bar\""),
+            MD_S8Lit("\"\"\"foo-bar\"\"\""),
+            MD_S8Lit("`foo-bar`"),
+            MD_S8Lit("```foo-bar```"),
+        };
+        
+        MD_Node *nodes[MD_ArrayCount(samples)];
+        for (int i = 0; i < MD_ArrayCount(samples); i += 1){
+            MD_ParseResult result = MD_ParseOneNode(MD_S8Lit(""), samples[i]);
+            nodes[i] = result.node;
+        }
+        
+        TestResult(MD_StringMatch(nodes[0]->string, MD_S8Lit("foo-bar"), 0));
+        TestResult(MD_StringMatch(nodes[1]->string, MD_S8Lit("foo-bar"), 0));
+        TestResult(MD_StringMatch(nodes[2]->string, MD_S8Lit("foo-bar"), 0));
+        TestResult(MD_StringMatch(nodes[3]->string, MD_S8Lit("foo-bar"), 0));
+        TestResult(MD_StringMatch(nodes[4]->string, MD_S8Lit("foo-bar"), 0));
+        TestResult(MD_StringMatch(nodes[5]->string, MD_S8Lit("foo-bar"), 0));
+        
+        TestResult(MD_StringMatch(nodes[0]->whole_string, samples[0], 0));
+        TestResult(MD_StringMatch(nodes[1]->whole_string, samples[1], 0));
+        TestResult(MD_StringMatch(nodes[2]->whole_string, samples[2], 0));
+        TestResult(MD_StringMatch(nodes[3]->whole_string, samples[3], 0));
+        TestResult(MD_StringMatch(nodes[4]->whole_string, samples[4], 0));
+        TestResult(MD_StringMatch(nodes[5]->whole_string, samples[5], 0));
+    }
+    
     Test("String escaping")
     {
         {