ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-07-24 00:17:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

trim down token kinds; return node flags from lexer

Browse Source
This commit is contained in:
ryanfleury
2021-06-29 22:55:57 -06:00
parent 0dea16c4c1
commit 5c99b2a737
2 changed files with 53 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/md.h
+9 -25
View File
@@ -422,26 +422,19 @@ struct MD_Map
typedef enum MD_TokenKind
{
// NOTE(rjf): *MUST* stay up-to-date with MD_TokenGroupsFromTokenKind.
MD_TokenKind_Nil,
MD_TokenKind_RegularMin,
MD_TokenKind_Identifier,
MD_TokenKind_NumericLiteral,
MD_TokenKind_StringLiteralSingleQuote,
MD_TokenKind_StringLiteralSingleQuoteTriplet,
MD_TokenKind_StringLiteralDoubleQuote,
MD_TokenKind_StringLiteralDoubleQuoteTriplet,
MD_TokenKind_StringLiteralTick,
MD_TokenKind_StringLiteralTickTriplet,
MD_TokenKind_StringLiteral,
MD_TokenKind_Symbol,
MD_TokenKind_RegularMax,
MD_TokenKind_Comment,
MD_TokenKind_WhitespaceMin,
MD_TokenKind_Whitespace,
MD_TokenKind_Newline,
MD_TokenKind_WhitespaceMax,
// Character outside currently supported encodings
MD_TokenKind_BadCharacter,
@@ -450,24 +443,19 @@ typedef enum MD_TokenKind
}
MD_TokenKind;
typedef MD_u32 MD_TokenFlags;
enum
{
MD_TokenFlag_ErrorUnterminated = (1<<0),
};
typedef MD_u32 MD_TokenGroups;
enum{
MD_TokenGroup_Comment = (1 << 0),
MD_TokenGroup_Whitespace = (1 << 1),
MD_TokenGroup_Regular = (1 << 2)
MD_TokenGroup_Comment = (1 << 0),
MD_TokenGroup_Whitespace = (1 << 1),
MD_TokenGroup_Regular = (1 << 2),
MD_TokenGroup_LabelString = (1 << 3),
};
typedef struct MD_Token MD_Token;
struct MD_Token
{
MD_TokenKind kind;
MD_TokenFlags flags;
MD_NodeFlags node_flags;
MD_String8 string;
MD_String8 outer_string;
};
@@ -731,11 +719,7 @@ MD_FUNCTION MD_MapSlot* MD_MapOverwrite(MD_Map *map, MD_MapKey key, void *val);
//~ Parsing
MD_FUNCTION MD_NodeFlags MD_NodeFlagsFromTokenKind(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsWhitespace(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsComment(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsRegular(MD_TokenKind kind);
MD_FUNCTION MD_TokenGroups MD_TokenGroupsFromTokenKind(MD_TokenKind kind);
MD_FUNCTION MD_Token MD_TokenFromString(MD_String8 string);
MD_FUNCTION MD_u64 MD_BytesFromStringTokenGroupRun(MD_String8 string, MD_TokenGroups groups);
source/md_impl.c
+44 -64
View File
@@ -1070,27 +1070,6 @@ MD_MapOverwrite(MD_Map *map, MD_MapKey key, void *val){
//~ Parsing
MD_FUNCTION_IMPL MD_NodeFlags
MD_NodeFlagsFromTokenKind(MD_TokenKind kind)
{
MD_NodeFlags result = 0;
switch (kind){
case MD_TokenKind_Identifier: result = MD_NodeFlag_Identifier; break;
case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric; break;
case MD_TokenKind_StringLiteralSingleQuote: result |= MD_NodeFlag_StringSingleQuote; goto string_lit;
case MD_TokenKind_StringLiteralDoubleQuote: result |= MD_NodeFlag_StringDoubleQuote; goto string_lit;
case MD_TokenKind_StringLiteralTick: result |= MD_NodeFlag_StringTick; goto string_lit;
case MD_TokenKind_StringLiteralSingleQuoteTriplet: result |= MD_NodeFlag_StringTripletSingleQuote; goto string_lit;
case MD_TokenKind_StringLiteralDoubleQuoteTriplet: result |= MD_NodeFlag_StringTripletDoubleQuote; goto string_lit;
case MD_TokenKind_StringLiteralTickTriplet: result |= MD_NodeFlag_StringTripletTick; goto string_lit;
string_lit:;
{
result |= MD_NodeFlag_StringLiteral;
}break;
}
return(result);
}
MD_PRIVATE_FUNCTION_IMPL MD_b32
_MD_TokenBoundariesAreBalanced(MD_Token token)
{
@@ -1113,22 +1092,30 @@ _MD_CommentIsSyntacticallyCorrect(MD_Token comment_token)
return result;
}
MD_FUNCTION_IMPL MD_b32
MD_TokenKindIsWhitespace(MD_TokenKind kind)
MD_FUNCTION MD_TokenGroups
MD_TokenGroupsFromTokenKind(MD_TokenKind kind)
{
return MD_TokenKind_WhitespaceMin < kind && kind < MD_TokenKind_WhitespaceMax;
}
MD_FUNCTION_IMPL MD_b32
MD_TokenKindIsComment(MD_TokenKind kind)
{
return(kind == MD_TokenKind_Comment);
}
MD_FUNCTION_IMPL MD_b32
MD_TokenKindIsRegular(MD_TokenKind kind)
{
return(MD_TokenKind_RegularMin < kind && kind < MD_TokenKind_RegularMax);
MD_TokenGroups groups = 0;
switch(kind)
{
//- rjf: nil
default:
case MD_TokenKind_Nil: break;
//- rjf: regular
case MD_TokenKind_Identifier: groups |= MD_TokenGroup_Regular|MD_TokenGroup_LabelString; break;
case MD_TokenKind_NumericLiteral: groups |= MD_TokenGroup_Regular|MD_TokenGroup_LabelString; break;
case MD_TokenKind_StringLiteral: groups |= MD_TokenGroup_Regular|MD_TokenGroup_LabelString; break;
case MD_TokenKind_Symbol: groups |= MD_TokenGroup_Regular|MD_TokenGroup_LabelString; break;
//- rjf: comments
case MD_TokenKind_Comment: groups |= MD_TokenGroup_Comment; break;
//- rjf: whitespace
case MD_TokenKind_Whitespace: groups |= MD_TokenGroup_Whitespace; break;
case MD_TokenKind_Newline: groups |= MD_TokenGroup_Whitespace; break;
}
return groups;
}
MD_FUNCTION MD_Token
@@ -1236,6 +1223,8 @@ MD_TokenFromString(MD_String8 string)
case '\'':
case '`':
{
token.kind = MD_TokenKind_StringLiteral;
// TODO(allen): proposal:
// go see the proposal in the block comment lexer, same idea here?
@@ -1318,14 +1307,15 @@ MD_TokenFromString(MD_String8 string)
}
}
// set token kind
//- rjf: set relevant node flags on token
token.node_flags |= MD_NodeFlag_StringLiteral;
if(is_triplet)
{
switch(d)
{
case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuoteTriplet; break;
case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuoteTriplet; break;
case '`': token.kind = MD_TokenKind_StringLiteralTickTriplet; break;
case '\'': token.node_flags |= MD_NodeFlag_StringTripletSingleQuote; break;
case '"': token.node_flags |= MD_NodeFlag_StringTripletDoubleQuote; break;
case '`': token.node_flags |= MD_NodeFlag_StringTripletTick; break;
default: break;
}
}
@@ -1333,9 +1323,9 @@ MD_TokenFromString(MD_String8 string)
{
switch(d)
{
case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuote; break;
case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuote; break;
case '`': token.kind = MD_TokenKind_StringLiteralTick; break;
case '\'': token.node_flags |= MD_NodeFlag_StringSingleQuote; break;
case '"': token.node_flags |= MD_NodeFlag_StringDoubleQuote; break;
case '`': token.node_flags |= MD_NodeFlag_StringTick; break;
default: break;
}
}
@@ -1347,6 +1337,7 @@ MD_TokenFromString(MD_String8 string)
{
if (MD_CharIsAlpha(*at) || *at == '_')
{
token.node_flags |= MD_NodeFlag_Identifier;
token.kind = MD_TokenKind_Identifier;
at += 1;
MD_TokenizerScan(MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '_');
@@ -1355,6 +1346,7 @@ MD_TokenFromString(MD_String8 string)
else if (MD_CharIsDigit(*at) ||
(at + 1 < one_past_last && at[0] == '-' && MD_CharIsDigit(at[1])))
{
token.node_flags |= MD_NodeFlag_Numeric;
token.kind = MD_TokenKind_NumericLiteral;
at += 1;
MD_TokenizerScan(MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '.');
@@ -1397,9 +1389,10 @@ MD_BytesFromStringTokenGroupRun(MD_String8 string, MD_TokenGroups groups)
loop:
{
MD_Token token = MD_TokenFromString(MD_StringSkip(string, result));
if((skip_comment && MD_TokenKindIsComment(token.kind)) ||
(skip_whitespace && MD_TokenKindIsWhitespace(token.kind)) ||
(skip_regular && MD_TokenKindIsRegular(token.kind)))
MD_TokenGroups groups = MD_TokenGroupsFromTokenKind(token.kind);
if((skip_comment && groups & MD_TokenGroup_Comment) ||
(skip_whitespace && groups & MD_TokenGroup_Whitespace) ||
(skip_regular && groups & MD_TokenGroup_Regular))
{
result += token.outer_string.size;
goto loop;
@@ -1791,7 +1784,7 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
MD_MemoryZero(&comment_token, sizeof(comment_token));
}
}
else if(MD_TokenKindIsWhitespace(token.kind))
else if(MD_TokenGroupsFromTokenKind(token.kind) & MD_TokenGroup_Whitespace)
{
off += token.outer_string.size;
}
@@ -1842,20 +1835,12 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
//- rjf: try to parse regular node, with/without children
off += MD_BytesFromStringTokenGroupRun(MD_StringSkip(string, off), MD_TokenGroup_Comment|MD_TokenGroup_Whitespace);
MD_Token label_name = MD_TokenFromString(MD_StringSkip(string, off));
if(label_name.kind == MD_TokenKind_Identifier ||
label_name.kind == MD_TokenKind_NumericLiteral ||
label_name.kind == MD_TokenKind_StringLiteralTick ||
label_name.kind == MD_TokenKind_StringLiteralSingleQuote ||
label_name.kind == MD_TokenKind_StringLiteralDoubleQuote ||
label_name.kind == MD_TokenKind_StringLiteralTickTriplet ||
label_name.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet ||
label_name.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet ||
label_name.kind == MD_TokenKind_Symbol )
if(MD_TokenGroupsFromTokenKind(label_name.kind) & MD_TokenGroup_LabelString)
{
off += label_name.outer_string.size;
parsed_node = MD_MakeNode(MD_NodeKind_Label, label_name.string, label_name.outer_string,
label_name.outer_string.str - string.str);
parsed_node->flags |= MD_NodeFlagsFromTokenKind(label_name.kind);
parsed_node->flags |= label_name.node_flags;
//- rjf: check for string literal errors
// TODO(rjf): Before we were just able to check one kind. I think preserving
@@ -1871,12 +1856,7 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
//
// If that turned out to be a good idea, maybe we could do something like
// MD_TokenKindIsLegalLabelHead, MD_TokenKindNeedsBalancing?? I don't know.
if(label_name.kind == MD_TokenKind_StringLiteralTick ||
label_name.kind == MD_TokenKind_StringLiteralSingleQuote ||
label_name.kind == MD_TokenKind_StringLiteralDoubleQuote ||
label_name.kind == MD_TokenKind_StringLiteralTickTriplet ||
label_name.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet ||
label_name.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet)
if(label_name.kind == MD_TokenKind_StringLiteral)
{
if(!_MD_TokenBoundariesAreBalanced(label_name))
{
@@ -1989,7 +1969,7 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
{
break;
}
else if(MD_TokenKindIsWhitespace(token.kind))
else if(MD_TokenGroupsFromTokenKind(token.kind) & MD_TokenGroup_Whitespace)
{
off += token.outer_string.size;
}