split up string literal token kinds to encode which syntax they used; translate these to node flags

source/md.h

@@ -298,6 +298,8 @@ MD_WordStyle;
 
 typedef enum MD_NodeKind
 {
+    // NOTE(rjf): Must be kept in sync with MD_StringFromNodeKind.
+    
     MD_NodeKind_Nil,
     MD_NodeKind_File,
     MD_NodeKind_List,
@@ -312,31 +314,40 @@ typedef enum MD_NodeKind
 }
 MD_NodeKind;
 
-typedef MD_u32 MD_NodeFlags;
+typedef MD_u64 MD_NodeFlags;
 #define MD_NodeFlag_AfterFromBefore(f) ((f) << 1)
 enum
 {
+    // NOTE(rjf): Must be kept in sync with MD_StringListFromNodeFlags.
+    
     // NOTE(rjf): Because of MD_NodeFlag_AfterFromBefore, it is *required* that
     // every single pair of "Before*" or "After*" flags be in the correct order
     // which is that the Before* flag comes first, and the After* flag comes
     // immediately after (After* being the more significant bit).
     
-    MD_NodeFlag_ParenLeft        = (1<<0),
+    MD_NodeFlag_ParenLeft               = (1<<0),
-    MD_NodeFlag_ParenRight       = (1<<1),
+    MD_NodeFlag_ParenRight              = (1<<1),
-    MD_NodeFlag_BracketLeft      = (1<<2),
+    MD_NodeFlag_BracketLeft             = (1<<2),
-    MD_NodeFlag_BracketRight     = (1<<3),
+    MD_NodeFlag_BracketRight            = (1<<3),
-    MD_NodeFlag_BraceLeft        = (1<<4),
+    MD_NodeFlag_BraceLeft               = (1<<4),
-    MD_NodeFlag_BraceRight       = (1<<5),
+    MD_NodeFlag_BraceRight              = (1<<5),
     
-    MD_NodeFlag_BeforeSemicolon  = (1<<6),
+    MD_NodeFlag_BeforeSemicolon         = (1<<6),
-    MD_NodeFlag_AfterSemicolon   = (1<<7),
+    MD_NodeFlag_AfterSemicolon          = (1<<7),
     
-    MD_NodeFlag_BeforeComma      = (1<<8),
+    MD_NodeFlag_BeforeComma             = (1<<8),
-    MD_NodeFlag_AfterComma       = (1<<9),
+    MD_NodeFlag_AfterComma              = (1<<9),
     
-    MD_NodeFlag_Numeric          = (1<<10),
+    MD_NodeFlag_StringSingleQuote       = (1<<10),
-    MD_NodeFlag_Identifier       = (1<<11),
+    MD_NodeFlag_StringDoubleQuote       = (1<<13),
-    MD_NodeFlag_StringLiteral    = (1<<12),
+    MD_NodeFlag_StringTick              = (1<<15),
+    MD_NodeFlag_StringTripletSingleQuote= (1<<16),
+    MD_NodeFlag_StringTripletDoubleQuote= (1<<18),
+    MD_NodeFlag_StringTripletTick       = (1<<20),
+    
+    MD_NodeFlag_Numeric                 = (1<<22),
+    MD_NodeFlag_Identifier              = (1<<23),
+    MD_NodeFlag_StringLiteral           = (1<<24),
 };
 
 typedef struct MD_Node MD_Node;
@@ -420,30 +431,15 @@ typedef enum MD_TokenKind
     MD_TokenKind_Nil,
     
     MD_TokenKind_RegularMin,
-    
-    // A group of characters that begins with an underscore or alphabetic character,
-    // and consists of numbers, alphabetic characters, or underscores after that.
     MD_TokenKind_Identifier,
-    
-    // A group of characters beginning with a numeric character or a '-', and then
-    // consisting of only numbers, alphabetic characters, or '.'s after that.
     MD_TokenKind_NumericLiteral,
-    
+    MD_TokenKind_StringLiteralSingleQuote,
-    // A group of arbitrary characters, grouped together by a " character, OR by a
+    MD_TokenKind_StringLiteralSingleQuoteTriplet,
-    // """ symbol at the beginning and end of the group. String literals beginning with
+    MD_TokenKind_StringLiteralDoubleQuote,
-    // " are to only be specified on a single line, but """ strings can exist across
+    MD_TokenKind_StringLiteralDoubleQuoteTriplet,
-    // many lines.
+    MD_TokenKind_StringLiteralTick,
-    MD_TokenKind_StringLiteral,
+    MD_TokenKind_StringLiteralTickTriplet,
-    
-    // A group of symbolic characters, where symbolic characters means any of the following:
-    // ~!@#$%^&*()-+=[{]}:;<>,./?|\
-   //
-    // Groups of multiple characters are only allowed in specific circumstances. Most of these
-    // are only 1 character long, but some groups are allowed:
-    //
-    // "<<", ">>", "<=", ">=", "+=", "-=", "*=", "/=", "::", ":=", "==", "&=", "|=", "->"
     MD_TokenKind_Symbol,
-    
     MD_TokenKind_RegularMax,
     
     MD_TokenKind_Comment,
@@ -453,8 +449,8 @@ typedef enum MD_TokenKind
     MD_TokenKind_Newline,
     MD_TokenKind_WhitespaceMax,
     
-    MD_TokenKind_BadCharacter,
     // Character outside currently supported encodings
+    MD_TokenKind_BadCharacter,
     
     MD_TokenKind_COUNT,
 }

source/md_impl.c

@@ -4,13 +4,8 @@
 #define MD_PRIVATE_FUNCTION_IMPL MD_FUNCTION_IMPL
 #define MD_UNTERMINATED_TOKEN_LEN_CAP 20
 
-//~
+//~ Nil Node Definition
 
-// NOTE(allen): Review @rjf; Building in C++
-// While very latest version of C++ have designated initializers
-// I would like to be able to build on more simple versions, so I
-// ditched the designated initializers in favor of the extra work
-// of maintaining order based initializers.
 static MD_Node _md_nil_node =
 {
     &_md_nil_node,         // next
@@ -32,6 +27,7 @@ static MD_Node _md_nil_node =
 };
 
 //~ Memory Operations
+
 MD_FUNCTION_IMPL void
 MD_MemoryZero(void *memory, MD_u64 size)
 {
@@ -654,11 +650,18 @@ MD_StringListFromNodeFlags(MD_NodeFlags flags)
         "BraceRight",
         
         "BeforeSemicolon",
-        "BeforeComma",
-        
         "AfterSemicolon",
+        
+        "BeforeComma",
         "AfterComma",
         
+        "StringSingleQuote",
+        "StringDoubleQuote",
+        "StringTick",
+        "StringTripletSingleQuote",
+        "StringTripletDoubleQuote",
+        "StringTripletTick",
+        
         "Numeric",
         "Identifier",
         "StringLiteral",
@@ -1058,9 +1061,18 @@ MD_NodeFlagsFromTokenKind(MD_TokenKind kind)
 {
     MD_NodeFlags result = 0;
     switch (kind){
-        case MD_TokenKind_Identifier:     result = MD_NodeFlag_Identifier;    break;
+        case MD_TokenKind_Identifier:                      result = MD_NodeFlag_Identifier;    break;
-        case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric;       break;
+        case MD_TokenKind_NumericLiteral:                  result = MD_NodeFlag_Numeric;       break;
-        case MD_TokenKind_StringLiteral:  result = MD_NodeFlag_StringLiteral; break;
+        case MD_TokenKind_StringLiteralSingleQuote:        result |= MD_NodeFlag_StringSingleQuote;        goto string_lit;
+        case MD_TokenKind_StringLiteralDoubleQuote:        result |= MD_NodeFlag_StringDoubleQuote;        goto string_lit;
+        case MD_TokenKind_StringLiteralTick:               result |= MD_NodeFlag_StringTick;               goto string_lit;
+        case MD_TokenKind_StringLiteralSingleQuoteTriplet: result |= MD_NodeFlag_StringTripletSingleQuote; goto string_lit;
+        case MD_TokenKind_StringLiteralDoubleQuoteTriplet: result |= MD_NodeFlag_StringTripletDoubleQuote; goto string_lit;
+        case MD_TokenKind_StringLiteralTickTriplet:        result |= MD_NodeFlag_StringTripletTick;        goto string_lit;
+        string_lit:;
+        {
+            result |= MD_NodeFlag_StringLiteral;
+        }break;
     }
     return(result);
 }
@@ -1357,11 +1369,6 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
             // NOTE(allen): Strings
             case '"':
             case '\'':
-            
-            // NOTE(rjf): "Bundle-of-tokens" strings (`stuff` or ```stuff```)
-            // In practice no different than a regular string, but provides an
-            // alternate syntax which will allow tools like 4coder to treat the
-            // contents as regular tokens.
             case '`':
             {
                 // TODO(allen): proposal:
@@ -1447,7 +1454,27 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
                 }
                 
                 // set token kind
-                token.kind = MD_TokenKind_StringLiteral;
+                if(is_triplet)
+                {
+                    switch(d)
+                    {
+                        case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuoteTriplet; break;
+                        case '"':  token.kind = MD_TokenKind_StringLiteralDoubleQuoteTriplet; break;
+                        case '`':  token.kind = MD_TokenKind_StringLiteralTickTriplet; break;
+                        default: break;
+                    }
+                }
+                else
+                {
+                    switch(d)
+                    {
+                        case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuote; break;
+                        case '"':  token.kind = MD_TokenKind_StringLiteralDoubleQuote; break;
+                        case '`':  token.kind = MD_TokenKind_StringLiteralTick; break;
+                        default: break;
+                    }
+                }
+                
             }break;
             
             // NOTE(allen): Identifiers, Numbers, Operators
@@ -1821,33 +1848,57 @@ MD_ParseOneNodeFromCtx(MD_ParseCtx *ctx)
     }
     
     // NOTE(rjf): Labels
-    else if(MD_Parse_RequireKind(ctx, MD_TokenKind_Identifier,     &token) ||
+    else if(next_token.kind == MD_TokenKind_Identifier                        ||
-            MD_Parse_RequireKind(ctx, MD_TokenKind_NumericLiteral, &token) ||
+            next_token.kind == MD_TokenKind_NumericLiteral                    ||
-            MD_Parse_RequireKind(ctx, MD_TokenKind_StringLiteral,  &token) ||
+            next_token.kind == MD_TokenKind_StringLiteralTick                 ||
-            MD_Parse_RequireKind(ctx, MD_TokenKind_Symbol,         &token))
+            next_token.kind == MD_TokenKind_StringLiteralSingleQuote          ||
+            next_token.kind == MD_TokenKind_StringLiteralDoubleQuote          ||
+            next_token.kind == MD_TokenKind_StringLiteralTickTriplet          ||
+            next_token.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet   ||
+            next_token.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet   ||
+            next_token.kind == MD_TokenKind_Symbol                           )
     {
-        result.node = MD_MakeNode(MD_NodeKind_Label, token.string, token.outer_string, token.outer_string.str);
+        MD_Parse_Bump(ctx, next_token);
-        result.node->flags |= MD_NodeFlagsFromTokenKind(token.kind);
+        result.node = MD_MakeNode(MD_NodeKind_Label, next_token.string, next_token.outer_string, next_token.outer_string.str);
+        result.node->flags |= MD_NodeFlagsFromTokenKind(next_token.kind);
         
-        if(token.kind == MD_TokenKind_StringLiteral)
+        // TODO(rjf): Before we were just able to check one kind. I think preserving
+        // which kind of string literal was used is very important, for the same reason
+        // that preserving which symbols were used to delimit a set is important.
+        // But, having to manage this "group of kinds" is a little bit annoying.
+        // Maybe we should define the set of legal kinds for certain syntactic
+        // contexts somewhere unified, so that the parser is never duplicating this?
+        //
+        // It's also possible that it never matters and we only ever use this group
+        // in one place, but I just got that "we're duplicating stuff" allergy that
+        // I usually get.
+        //
+        // If that turned out to be a good idea, maybe we could do something like
+        // MD_TokenKindIsLegalLabelHead, MD_TokenKindNeedsBalancing?? I don't know.
+        if(next_token.kind == MD_TokenKind_StringLiteralTick                 ||
+           next_token.kind == MD_TokenKind_StringLiteralSingleQuote          ||
+           next_token.kind == MD_TokenKind_StringLiteralDoubleQuote          ||
+           next_token.kind == MD_TokenKind_StringLiteralTickTriplet          ||
+           next_token.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet   ||
+           next_token.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet)
         {
-            if(!_MD_StringLiteralIsBalanced(token))
+            if(!_MD_StringLiteralIsBalanced(next_token))
             {
-                MD_String8 capped = MD_StringPrefix(token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP);
+                MD_String8 capped = MD_StringPrefix(next_token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP);
                 MD_PushNodeErrorF(ctx, result.node, MD_MessageKind_CatastrophicError,
                                   "Unterminated text literal \"%.*s\"", MD_StringExpand(capped));
             }
         }
-        else if(token.kind == MD_TokenKind_Symbol && token.string.size == 1 && MD_CharIsReservedSymbol(token.string.str[0]))
+        else if(next_token.kind == MD_TokenKind_Symbol && next_token.string.size == 1 && MD_CharIsReservedSymbol(next_token.string.str[0]))
         {
-            MD_u8 c = token.string.str[0];
+            MD_u8 c = next_token.string.str[0];
             if(c == '}' || c == ']' || c == ')')
             {
-                MD_PushTokenErrorF(ctx, token, MD_MessageKind_CatastrophicError, "Unbalanced \"%c\"", c);
+                MD_PushTokenErrorF(ctx, next_token, MD_MessageKind_CatastrophicError, "Unbalanced \"%c\"", c);
             }
             else
             {
-                MD_PushTokenErrorF(ctx, token, MD_MessageKind_Error, "Unexpected reserved symbol \"%c\"",
+                MD_PushTokenErrorF(ctx, next_token, MD_MessageKind_Error, "Unexpected reserved symbol \"%c\"",
                                    c);
             }
         }