diff --git a/source/md.h b/source/md.h index 2a0ef9a..2206ee6 100644 --- a/source/md.h +++ b/source/md.h @@ -298,6 +298,8 @@ MD_WordStyle; typedef enum MD_NodeKind { + // NOTE(rjf): Must be kept in sync with MD_StringFromNodeKind. + MD_NodeKind_Nil, MD_NodeKind_File, MD_NodeKind_List, @@ -312,31 +314,40 @@ typedef enum MD_NodeKind } MD_NodeKind; -typedef MD_u32 MD_NodeFlags; +typedef MD_u64 MD_NodeFlags; #define MD_NodeFlag_AfterFromBefore(f) ((f) << 1) enum { + // NOTE(rjf): Must be kept in sync with MD_StringListFromNodeFlags. + // NOTE(rjf): Because of MD_NodeFlag_AfterFromBefore, it is *required* that // every single pair of "Before*" or "After*" flags be in the correct order // which is that the Before* flag comes first, and the After* flag comes // immediately after (After* being the more significant bit). - MD_NodeFlag_ParenLeft = (1<<0), - MD_NodeFlag_ParenRight = (1<<1), - MD_NodeFlag_BracketLeft = (1<<2), - MD_NodeFlag_BracketRight = (1<<3), - MD_NodeFlag_BraceLeft = (1<<4), - MD_NodeFlag_BraceRight = (1<<5), + MD_NodeFlag_ParenLeft = (1<<0), + MD_NodeFlag_ParenRight = (1<<1), + MD_NodeFlag_BracketLeft = (1<<2), + MD_NodeFlag_BracketRight = (1<<3), + MD_NodeFlag_BraceLeft = (1<<4), + MD_NodeFlag_BraceRight = (1<<5), - MD_NodeFlag_BeforeSemicolon = (1<<6), - MD_NodeFlag_AfterSemicolon = (1<<7), + MD_NodeFlag_BeforeSemicolon = (1<<6), + MD_NodeFlag_AfterSemicolon = (1<<7), - MD_NodeFlag_BeforeComma = (1<<8), - MD_NodeFlag_AfterComma = (1<<9), + MD_NodeFlag_BeforeComma = (1<<8), + MD_NodeFlag_AfterComma = (1<<9), - MD_NodeFlag_Numeric = (1<<10), - MD_NodeFlag_Identifier = (1<<11), - MD_NodeFlag_StringLiteral = (1<<12), + MD_NodeFlag_StringSingleQuote = (1<<10), + MD_NodeFlag_StringDoubleQuote = (1<<13), + MD_NodeFlag_StringTick = (1<<15), + MD_NodeFlag_StringTripletSingleQuote= (1<<16), + MD_NodeFlag_StringTripletDoubleQuote= (1<<18), + MD_NodeFlag_StringTripletTick = (1<<20), + + MD_NodeFlag_Numeric = (1<<22), + MD_NodeFlag_Identifier = (1<<23), + MD_NodeFlag_StringLiteral = (1<<24), }; typedef struct MD_Node MD_Node; @@ -420,30 +431,15 @@ typedef enum MD_TokenKind MD_TokenKind_Nil, MD_TokenKind_RegularMin, - - // A group of characters that begins with an underscore or alphabetic character, - // and consists of numbers, alphabetic characters, or underscores after that. MD_TokenKind_Identifier, - - // A group of characters beginning with a numeric character or a '-', and then - // consisting of only numbers, alphabetic characters, or '.'s after that. MD_TokenKind_NumericLiteral, - - // A group of arbitrary characters, grouped together by a " character, OR by a - // """ symbol at the beginning and end of the group. String literals beginning with - // " are to only be specified on a single line, but """ strings can exist across - // many lines. - MD_TokenKind_StringLiteral, - - // A group of symbolic characters, where symbolic characters means any of the following: - // ~!@#$%^&*()-+=[{]}:;<>,./?|\ - // - // Groups of multiple characters are only allowed in specific circumstances. Most of these - // are only 1 character long, but some groups are allowed: - // - // "<<", ">>", "<=", ">=", "+=", "-=", "*=", "/=", "::", ":=", "==", "&=", "|=", "->" + MD_TokenKind_StringLiteralSingleQuote, + MD_TokenKind_StringLiteralSingleQuoteTriplet, + MD_TokenKind_StringLiteralDoubleQuote, + MD_TokenKind_StringLiteralDoubleQuoteTriplet, + MD_TokenKind_StringLiteralTick, + MD_TokenKind_StringLiteralTickTriplet, MD_TokenKind_Symbol, - MD_TokenKind_RegularMax, MD_TokenKind_Comment, @@ -453,8 +449,8 @@ typedef enum MD_TokenKind MD_TokenKind_Newline, MD_TokenKind_WhitespaceMax, - MD_TokenKind_BadCharacter, // Character outside currently supported encodings + MD_TokenKind_BadCharacter, MD_TokenKind_COUNT, } diff --git a/source/md_impl.c b/source/md_impl.c index a159007..d2f0a0f 100644 --- a/source/md_impl.c +++ b/source/md_impl.c @@ -4,13 +4,8 @@ #define MD_PRIVATE_FUNCTION_IMPL MD_FUNCTION_IMPL #define MD_UNTERMINATED_TOKEN_LEN_CAP 20 -//~ +//~ Nil Node Definition -// NOTE(allen): Review @rjf; Building in C++ -// While very latest version of C++ have designated initializers -// I would like to be able to build on more simple versions, so I -// ditched the designated initializers in favor of the extra work -// of maintaining order based initializers. static MD_Node _md_nil_node = { &_md_nil_node, // next @@ -32,6 +27,7 @@ static MD_Node _md_nil_node = }; //~ Memory Operations + MD_FUNCTION_IMPL void MD_MemoryZero(void *memory, MD_u64 size) { @@ -654,11 +650,18 @@ MD_StringListFromNodeFlags(MD_NodeFlags flags) "BraceRight", "BeforeSemicolon", - "BeforeComma", - "AfterSemicolon", + + "BeforeComma", "AfterComma", + "StringSingleQuote", + "StringDoubleQuote", + "StringTick", + "StringTripletSingleQuote", + "StringTripletDoubleQuote", + "StringTripletTick", + "Numeric", "Identifier", "StringLiteral", @@ -1058,9 +1061,18 @@ MD_NodeFlagsFromTokenKind(MD_TokenKind kind) { MD_NodeFlags result = 0; switch (kind){ - case MD_TokenKind_Identifier: result = MD_NodeFlag_Identifier; break; - case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric; break; - case MD_TokenKind_StringLiteral: result = MD_NodeFlag_StringLiteral; break; + case MD_TokenKind_Identifier: result = MD_NodeFlag_Identifier; break; + case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric; break; + case MD_TokenKind_StringLiteralSingleQuote: result |= MD_NodeFlag_StringSingleQuote; goto string_lit; + case MD_TokenKind_StringLiteralDoubleQuote: result |= MD_NodeFlag_StringDoubleQuote; goto string_lit; + case MD_TokenKind_StringLiteralTick: result |= MD_NodeFlag_StringTick; goto string_lit; + case MD_TokenKind_StringLiteralSingleQuoteTriplet: result |= MD_NodeFlag_StringTripletSingleQuote; goto string_lit; + case MD_TokenKind_StringLiteralDoubleQuoteTriplet: result |= MD_NodeFlag_StringTripletDoubleQuote; goto string_lit; + case MD_TokenKind_StringLiteralTickTriplet: result |= MD_NodeFlag_StringTripletTick; goto string_lit; + string_lit:; + { + result |= MD_NodeFlag_StringLiteral; + }break; } return(result); } @@ -1357,11 +1369,6 @@ MD_Parse_LexNext(MD_ParseCtx *ctx) // NOTE(allen): Strings case '"': case '\'': - - // NOTE(rjf): "Bundle-of-tokens" strings (`stuff` or ```stuff```) - // In practice no different than a regular string, but provides an - // alternate syntax which will allow tools like 4coder to treat the - // contents as regular tokens. case '`': { // TODO(allen): proposal: @@ -1447,7 +1454,27 @@ MD_Parse_LexNext(MD_ParseCtx *ctx) } // set token kind - token.kind = MD_TokenKind_StringLiteral; + if(is_triplet) + { + switch(d) + { + case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuoteTriplet; break; + case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuoteTriplet; break; + case '`': token.kind = MD_TokenKind_StringLiteralTickTriplet; break; + default: break; + } + } + else + { + switch(d) + { + case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuote; break; + case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuote; break; + case '`': token.kind = MD_TokenKind_StringLiteralTick; break; + default: break; + } + } + }break; // NOTE(allen): Identifiers, Numbers, Operators @@ -1821,33 +1848,57 @@ MD_ParseOneNodeFromCtx(MD_ParseCtx *ctx) } // NOTE(rjf): Labels - else if(MD_Parse_RequireKind(ctx, MD_TokenKind_Identifier, &token) || - MD_Parse_RequireKind(ctx, MD_TokenKind_NumericLiteral, &token) || - MD_Parse_RequireKind(ctx, MD_TokenKind_StringLiteral, &token) || - MD_Parse_RequireKind(ctx, MD_TokenKind_Symbol, &token)) + else if(next_token.kind == MD_TokenKind_Identifier || + next_token.kind == MD_TokenKind_NumericLiteral || + next_token.kind == MD_TokenKind_StringLiteralTick || + next_token.kind == MD_TokenKind_StringLiteralSingleQuote || + next_token.kind == MD_TokenKind_StringLiteralDoubleQuote || + next_token.kind == MD_TokenKind_StringLiteralTickTriplet || + next_token.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet || + next_token.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet || + next_token.kind == MD_TokenKind_Symbol ) { - result.node = MD_MakeNode(MD_NodeKind_Label, token.string, token.outer_string, token.outer_string.str); - result.node->flags |= MD_NodeFlagsFromTokenKind(token.kind); + MD_Parse_Bump(ctx, next_token); + result.node = MD_MakeNode(MD_NodeKind_Label, next_token.string, next_token.outer_string, next_token.outer_string.str); + result.node->flags |= MD_NodeFlagsFromTokenKind(next_token.kind); - if(token.kind == MD_TokenKind_StringLiteral) + // TODO(rjf): Before we were just able to check one kind. I think preserving + // which kind of string literal was used is very important, for the same reason + // that preserving which symbols were used to delimit a set is important. + // But, having to manage this "group of kinds" is a little bit annoying. + // Maybe we should define the set of legal kinds for certain syntactic + // contexts somewhere unified, so that the parser is never duplicating this? + // + // It's also possible that it never matters and we only ever use this group + // in one place, but I just got that "we're duplicating stuff" allergy that + // I usually get. + // + // If that turned out to be a good idea, maybe we could do something like + // MD_TokenKindIsLegalLabelHead, MD_TokenKindNeedsBalancing?? I don't know. + if(next_token.kind == MD_TokenKind_StringLiteralTick || + next_token.kind == MD_TokenKind_StringLiteralSingleQuote || + next_token.kind == MD_TokenKind_StringLiteralDoubleQuote || + next_token.kind == MD_TokenKind_StringLiteralTickTriplet || + next_token.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet || + next_token.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet) { - if(!_MD_StringLiteralIsBalanced(token)) + if(!_MD_StringLiteralIsBalanced(next_token)) { - MD_String8 capped = MD_StringPrefix(token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP); + MD_String8 capped = MD_StringPrefix(next_token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP); MD_PushNodeErrorF(ctx, result.node, MD_MessageKind_CatastrophicError, "Unterminated text literal \"%.*s\"", MD_StringExpand(capped)); } } - else if(token.kind == MD_TokenKind_Symbol && token.string.size == 1 && MD_CharIsReservedSymbol(token.string.str[0])) + else if(next_token.kind == MD_TokenKind_Symbol && next_token.string.size == 1 && MD_CharIsReservedSymbol(next_token.string.str[0])) { - MD_u8 c = token.string.str[0]; + MD_u8 c = next_token.string.str[0]; if(c == '}' || c == ']' || c == ')') { - MD_PushTokenErrorF(ctx, token, MD_MessageKind_CatastrophicError, "Unbalanced \"%c\"", c); + MD_PushTokenErrorF(ctx, next_token, MD_MessageKind_CatastrophicError, "Unbalanced \"%c\"", c); } else { - MD_PushTokenErrorF(ctx, token, MD_MessageKind_Error, "Unexpected reserved symbol \"%c\"", + MD_PushTokenErrorF(ctx, next_token, MD_MessageKind_Error, "Unexpected reserved symbol \"%c\"", c); } }