ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-07-09 11:21:37 -07:00
Code Issues Packages Projects Releases Wiki Activity

split up string literal token kinds to encode which syntax they used; translate these to node flags

Browse Source
This commit is contained in:
ryanfleury
2021-06-24 23:30:01 -06:00
parent f7e40e224f
commit 4ec674600d
2 changed files with 113 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/md.h
+32 -36
View File
@@ -298,6 +298,8 @@ MD_WordStyle;
typedef enum MD_NodeKind
{
// NOTE(rjf): Must be kept in sync with MD_StringFromNodeKind.
MD_NodeKind_Nil,
MD_NodeKind_File,
MD_NodeKind_List,
@@ -312,31 +314,40 @@ typedef enum MD_NodeKind
}
MD_NodeKind;
typedef MD_u32 MD_NodeFlags;
typedef MD_u64 MD_NodeFlags;
#define MD_NodeFlag_AfterFromBefore(f) ((f) << 1)
enum
{
// NOTE(rjf): Must be kept in sync with MD_StringListFromNodeFlags.
// NOTE(rjf): Because of MD_NodeFlag_AfterFromBefore, it is *required* that
// every single pair of "Before*" or "After*" flags be in the correct order
// which is that the Before* flag comes first, and the After* flag comes
// immediately after (After* being the more significant bit).
MD_NodeFlag_ParenLeft = (1<<0),
MD_NodeFlag_ParenRight = (1<<1),
MD_NodeFlag_BracketLeft = (1<<2),
MD_NodeFlag_BracketRight = (1<<3),
MD_NodeFlag_BraceLeft = (1<<4),
MD_NodeFlag_BraceRight = (1<<5),
MD_NodeFlag_ParenLeft = (1<<0),
MD_NodeFlag_ParenRight = (1<<1),
MD_NodeFlag_BracketLeft = (1<<2),
MD_NodeFlag_BracketRight = (1<<3),
MD_NodeFlag_BraceLeft = (1<<4),
MD_NodeFlag_BraceRight = (1<<5),
MD_NodeFlag_BeforeSemicolon = (1<<6),
MD_NodeFlag_AfterSemicolon = (1<<7),
MD_NodeFlag_BeforeSemicolon = (1<<6),
MD_NodeFlag_AfterSemicolon = (1<<7),
MD_NodeFlag_BeforeComma = (1<<8),
MD_NodeFlag_AfterComma = (1<<9),
MD_NodeFlag_BeforeComma = (1<<8),
MD_NodeFlag_AfterComma = (1<<9),
MD_NodeFlag_Numeric = (1<<10),
MD_NodeFlag_Identifier = (1<<11),
MD_NodeFlag_StringLiteral = (1<<12),
MD_NodeFlag_StringSingleQuote = (1<<10),
MD_NodeFlag_StringDoubleQuote = (1<<13),
MD_NodeFlag_StringTick = (1<<15),
MD_NodeFlag_StringTripletSingleQuote= (1<<16),
MD_NodeFlag_StringTripletDoubleQuote= (1<<18),
MD_NodeFlag_StringTripletTick = (1<<20),
MD_NodeFlag_Numeric = (1<<22),
MD_NodeFlag_Identifier = (1<<23),
MD_NodeFlag_StringLiteral = (1<<24),
};
typedef struct MD_Node MD_Node;
@@ -420,30 +431,15 @@ typedef enum MD_TokenKind
MD_TokenKind_Nil,
MD_TokenKind_RegularMin,
// A group of characters that begins with an underscore or alphabetic character,
// and consists of numbers, alphabetic characters, or underscores after that.
MD_TokenKind_Identifier,
// A group of characters beginning with a numeric character or a '-', and then
// consisting of only numbers, alphabetic characters, or '.'s after that.
MD_TokenKind_NumericLiteral,
// A group of arbitrary characters, grouped together by a " character, OR by a
// """ symbol at the beginning and end of the group. String literals beginning with
// " are to only be specified on a single line, but """ strings can exist across
// many lines.
MD_TokenKind_StringLiteral,
// A group of symbolic characters, where symbolic characters means any of the following:
// ~!@#$%^&*()-+=[{]}:;<>,./?|\
//
// Groups of multiple characters are only allowed in specific circumstances. Most of these
// are only 1 character long, but some groups are allowed:
//
// "<<", ">>", "<=", ">=", "+=", "-=", "*=", "/=", "::", ":=", "==", "&=", "|=", "->"
MD_TokenKind_StringLiteralSingleQuote,
MD_TokenKind_StringLiteralSingleQuoteTriplet,
MD_TokenKind_StringLiteralDoubleQuote,
MD_TokenKind_StringLiteralDoubleQuoteTriplet,
MD_TokenKind_StringLiteralTick,
MD_TokenKind_StringLiteralTickTriplet,
MD_TokenKind_Symbol,
MD_TokenKind_RegularMax,
MD_TokenKind_Comment,
@@ -453,8 +449,8 @@ typedef enum MD_TokenKind
MD_TokenKind_Newline,
MD_TokenKind_WhitespaceMax,
MD_TokenKind_BadCharacter,
// Character outside currently supported encodings
MD_TokenKind_BadCharacter,
MD_TokenKind_COUNT,
}
source/md_impl.c
+81 -30
View File
@@ -4,13 +4,8 @@
#define MD_PRIVATE_FUNCTION_IMPL MD_FUNCTION_IMPL
#define MD_UNTERMINATED_TOKEN_LEN_CAP 20
//~
//~ Nil Node Definition
// NOTE(allen): Review @rjf; Building in C++
// While very latest version of C++ have designated initializers
// I would like to be able to build on more simple versions, so I
// ditched the designated initializers in favor of the extra work
// of maintaining order based initializers.
static MD_Node _md_nil_node =
{
&_md_nil_node, // next
@@ -32,6 +27,7 @@ static MD_Node _md_nil_node =
};
//~ Memory Operations
MD_FUNCTION_IMPL void
MD_MemoryZero(void *memory, MD_u64 size)
{
@@ -654,11 +650,18 @@ MD_StringListFromNodeFlags(MD_NodeFlags flags)
"BraceRight",
"BeforeSemicolon",
"BeforeComma",
"AfterSemicolon",
"BeforeComma",
"AfterComma",
"StringSingleQuote",
"StringDoubleQuote",
"StringTick",
"StringTripletSingleQuote",
"StringTripletDoubleQuote",
"StringTripletTick",
"Numeric",
"Identifier",
"StringLiteral",
@@ -1058,9 +1061,18 @@ MD_NodeFlagsFromTokenKind(MD_TokenKind kind)
{
MD_NodeFlags result = 0;
switch (kind){
case MD_TokenKind_Identifier: result = MD_NodeFlag_Identifier; break;
case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric; break;
case MD_TokenKind_StringLiteral: result = MD_NodeFlag_StringLiteral; break;
case MD_TokenKind_Identifier: result = MD_NodeFlag_Identifier; break;
case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric; break;
case MD_TokenKind_StringLiteralSingleQuote: result |= MD_NodeFlag_StringSingleQuote; goto string_lit;
case MD_TokenKind_StringLiteralDoubleQuote: result |= MD_NodeFlag_StringDoubleQuote; goto string_lit;
case MD_TokenKind_StringLiteralTick: result |= MD_NodeFlag_StringTick; goto string_lit;
case MD_TokenKind_StringLiteralSingleQuoteTriplet: result |= MD_NodeFlag_StringTripletSingleQuote; goto string_lit;
case MD_TokenKind_StringLiteralDoubleQuoteTriplet: result |= MD_NodeFlag_StringTripletDoubleQuote; goto string_lit;
case MD_TokenKind_StringLiteralTickTriplet: result |= MD_NodeFlag_StringTripletTick; goto string_lit;
string_lit:;
{
result |= MD_NodeFlag_StringLiteral;
}break;
}
return(result);
}
@@ -1357,11 +1369,6 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
// NOTE(allen): Strings
case '"':
case '\'':
// NOTE(rjf): "Bundle-of-tokens" strings (`stuff` or ```stuff```)
// In practice no different than a regular string, but provides an
// alternate syntax which will allow tools like 4coder to treat the
// contents as regular tokens.
case '`':
{
// TODO(allen): proposal:
@@ -1447,7 +1454,27 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
}
// set token kind
token.kind = MD_TokenKind_StringLiteral;
if(is_triplet)
{
switch(d)
{
case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuoteTriplet; break;
case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuoteTriplet; break;
case '`': token.kind = MD_TokenKind_StringLiteralTickTriplet; break;
default: break;
}
}
else
{
switch(d)
{
case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuote; break;
case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuote; break;
case '`': token.kind = MD_TokenKind_StringLiteralTick; break;
default: break;
}
}
}break;
// NOTE(allen): Identifiers, Numbers, Operators
@@ -1821,33 +1848,57 @@ MD_ParseOneNodeFromCtx(MD_ParseCtx *ctx)
}
// NOTE(rjf): Labels
else if(MD_Parse_RequireKind(ctx, MD_TokenKind_Identifier, &token) ||
MD_Parse_RequireKind(ctx, MD_TokenKind_NumericLiteral, &token) ||
MD_Parse_RequireKind(ctx, MD_TokenKind_StringLiteral, &token) ||
MD_Parse_RequireKind(ctx, MD_TokenKind_Symbol, &token))
else if(next_token.kind == MD_TokenKind_Identifier ||
next_token.kind == MD_TokenKind_NumericLiteral ||
next_token.kind == MD_TokenKind_StringLiteralTick ||
next_token.kind == MD_TokenKind_StringLiteralSingleQuote ||
next_token.kind == MD_TokenKind_StringLiteralDoubleQuote ||
next_token.kind == MD_TokenKind_StringLiteralTickTriplet ||
next_token.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet ||
next_token.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet ||
next_token.kind == MD_TokenKind_Symbol )
{
result.node = MD_MakeNode(MD_NodeKind_Label, token.string, token.outer_string, token.outer_string.str);
result.node->flags |= MD_NodeFlagsFromTokenKind(token.kind);
MD_Parse_Bump(ctx, next_token);
result.node = MD_MakeNode(MD_NodeKind_Label, next_token.string, next_token.outer_string, next_token.outer_string.str);
result.node->flags |= MD_NodeFlagsFromTokenKind(next_token.kind);
if(token.kind == MD_TokenKind_StringLiteral)
// TODO(rjf): Before we were just able to check one kind. I think preserving
// which kind of string literal was used is very important, for the same reason
// that preserving which symbols were used to delimit a set is important.
// But, having to manage this "group of kinds" is a little bit annoying.
// Maybe we should define the set of legal kinds for certain syntactic
// contexts somewhere unified, so that the parser is never duplicating this?
//
// It's also possible that it never matters and we only ever use this group
// in one place, but I just got that "we're duplicating stuff" allergy that
// I usually get.
//
// If that turned out to be a good idea, maybe we could do something like
// MD_TokenKindIsLegalLabelHead, MD_TokenKindNeedsBalancing?? I don't know.
if(next_token.kind == MD_TokenKind_StringLiteralTick ||
next_token.kind == MD_TokenKind_StringLiteralSingleQuote ||
next_token.kind == MD_TokenKind_StringLiteralDoubleQuote ||
next_token.kind == MD_TokenKind_StringLiteralTickTriplet ||
next_token.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet ||
next_token.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet)
{
if(!_MD_StringLiteralIsBalanced(token))
if(!_MD_StringLiteralIsBalanced(next_token))
{
MD_String8 capped = MD_StringPrefix(token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP);
MD_String8 capped = MD_StringPrefix(next_token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP);
MD_PushNodeErrorF(ctx, result.node, MD_MessageKind_CatastrophicError,
"Unterminated text literal \"%.*s\"", MD_StringExpand(capped));
}
}
else if(token.kind == MD_TokenKind_Symbol && token.string.size == 1 && MD_CharIsReservedSymbol(token.string.str[0]))
else if(next_token.kind == MD_TokenKind_Symbol && next_token.string.size == 1 && MD_CharIsReservedSymbol(next_token.string.str[0]))
{
MD_u8 c = token.string.str[0];
MD_u8 c = next_token.string.str[0];
if(c == '}' || c == ']' || c == ')')
{
MD_PushTokenErrorF(ctx, token, MD_MessageKind_CatastrophicError, "Unbalanced \"%c\"", c);
MD_PushTokenErrorF(ctx, next_token, MD_MessageKind_CatastrophicError, "Unbalanced \"%c\"", c);
}
else
{
MD_PushTokenErrorF(ctx, token, MD_MessageKind_Error, "Unexpected reserved symbol \"%c\"",
MD_PushTokenErrorF(ctx, next_token, MD_MessageKind_Error, "Unexpected reserved symbol \"%c\"",
c);
}
}