ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-08-07 16:18:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

trim down token kinds; return node flags from lexer

Browse Source
This commit is contained in:
ryanfleury
2021-06-29 22:55:57 -06:00
parent 0dea16c4c1
commit 5c99b2a737
2 changed files with 53 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/md.h
+9 -25
View File
@@ -422,26 +422,19 @@ struct MD_Map
typedef enum MD_TokenKind typedef enum MD_TokenKind
{ {
// NOTE(rjf): *MUST* stay up-to-date with MD_TokenGroupsFromTokenKind.
MD_TokenKind_Nil, MD_TokenKind_Nil,
MD_TokenKind_RegularMin,
MD_TokenKind_Identifier, MD_TokenKind_Identifier,
MD_TokenKind_NumericLiteral, MD_TokenKind_NumericLiteral,
MD_TokenKind_StringLiteralSingleQuote, MD_TokenKind_StringLiteral,
MD_TokenKind_StringLiteralSingleQuoteTriplet,
MD_TokenKind_StringLiteralDoubleQuote,
MD_TokenKind_StringLiteralDoubleQuoteTriplet,
MD_TokenKind_StringLiteralTick,
MD_TokenKind_StringLiteralTickTriplet,
MD_TokenKind_Symbol, MD_TokenKind_Symbol,
MD_TokenKind_RegularMax,
MD_TokenKind_Comment, MD_TokenKind_Comment,
MD_TokenKind_WhitespaceMin,
MD_TokenKind_Whitespace, MD_TokenKind_Whitespace,
MD_TokenKind_Newline, MD_TokenKind_Newline,
MD_TokenKind_WhitespaceMax,
// Character outside currently supported encodings // Character outside currently supported encodings
MD_TokenKind_BadCharacter, MD_TokenKind_BadCharacter,
@@ -450,24 +443,19 @@ typedef enum MD_TokenKind
} }
MD_TokenKind; MD_TokenKind;
typedef MD_u32 MD_TokenFlags;
enum
{
MD_TokenFlag_ErrorUnterminated = (1<<0),
};
typedef MD_u32 MD_TokenGroups; typedef MD_u32 MD_TokenGroups;
enum{ enum{
MD_TokenGroup_Comment = (1 << 0), MD_TokenGroup_Comment = (1 << 0),
MD_TokenGroup_Whitespace = (1 << 1), MD_TokenGroup_Whitespace = (1 << 1),
MD_TokenGroup_Regular = (1 << 2) MD_TokenGroup_Regular = (1 << 2),
MD_TokenGroup_LabelString = (1 << 3),
}; };
typedef struct MD_Token MD_Token; typedef struct MD_Token MD_Token;
struct MD_Token struct MD_Token
{ {
MD_TokenKind kind; MD_TokenKind kind;
MD_TokenFlags flags; MD_NodeFlags node_flags;
MD_String8 string; MD_String8 string;
MD_String8 outer_string; MD_String8 outer_string;
}; };
@@ -731,11 +719,7 @@ MD_FUNCTION MD_MapSlot* MD_MapOverwrite(MD_Map *map, MD_MapKey key, void *val);
//~ Parsing //~ Parsing
MD_FUNCTION MD_NodeFlags MD_NodeFlagsFromTokenKind(MD_TokenKind kind); MD_FUNCTION MD_TokenGroups MD_TokenGroupsFromTokenKind(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsWhitespace(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsComment(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsRegular(MD_TokenKind kind);
MD_FUNCTION MD_Token MD_TokenFromString(MD_String8 string); MD_FUNCTION MD_Token MD_TokenFromString(MD_String8 string);
MD_FUNCTION MD_u64 MD_BytesFromStringTokenGroupRun(MD_String8 string, MD_TokenGroups groups); MD_FUNCTION MD_u64 MD_BytesFromStringTokenGroupRun(MD_String8 string, MD_TokenGroups groups);
source/md_impl.c
+44 -64
View File
@@ -1070,27 +1070,6 @@ MD_MapOverwrite(MD_Map *map, MD_MapKey key, void *val){
//~ Parsing //~ Parsing
MD_FUNCTION_IMPL MD_NodeFlags
MD_NodeFlagsFromTokenKind(MD_TokenKind kind)
{
MD_NodeFlags result = 0;
switch (kind){
case MD_TokenKind_Identifier: result = MD_NodeFlag_Identifier; break;
case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric; break;
case MD_TokenKind_StringLiteralSingleQuote: result |= MD_NodeFlag_StringSingleQuote; goto string_lit;
case MD_TokenKind_StringLiteralDoubleQuote: result |= MD_NodeFlag_StringDoubleQuote; goto string_lit;
case MD_TokenKind_StringLiteralTick: result |= MD_NodeFlag_StringTick; goto string_lit;
case MD_TokenKind_StringLiteralSingleQuoteTriplet: result |= MD_NodeFlag_StringTripletSingleQuote; goto string_lit;
case MD_TokenKind_StringLiteralDoubleQuoteTriplet: result |= MD_NodeFlag_StringTripletDoubleQuote; goto string_lit;
case MD_TokenKind_StringLiteralTickTriplet: result |= MD_NodeFlag_StringTripletTick; goto string_lit;
string_lit:;
{
result |= MD_NodeFlag_StringLiteral;
}break;
}
return(result);
}
MD_PRIVATE_FUNCTION_IMPL MD_b32 MD_PRIVATE_FUNCTION_IMPL MD_b32
_MD_TokenBoundariesAreBalanced(MD_Token token) _MD_TokenBoundariesAreBalanced(MD_Token token)
{ {
@@ -1113,22 +1092,30 @@ _MD_CommentIsSyntacticallyCorrect(MD_Token comment_token)
return result; return result;
} }
MD_FUNCTION_IMPL MD_b32 MD_FUNCTION MD_TokenGroups
MD_TokenKindIsWhitespace(MD_TokenKind kind) MD_TokenGroupsFromTokenKind(MD_TokenKind kind)
{ {
return MD_TokenKind_WhitespaceMin < kind && kind < MD_TokenKind_WhitespaceMax; MD_TokenGroups groups = 0;
} switch(kind)
{
MD_FUNCTION_IMPL MD_b32 //- rjf: nil
MD_TokenKindIsComment(MD_TokenKind kind) default:
{ case MD_TokenKind_Nil: break;
return(kind == MD_TokenKind_Comment);
} //- rjf: regular
case MD_TokenKind_Identifier: groups |= MD_TokenGroup_Regular|MD_TokenGroup_LabelString; break;
MD_FUNCTION_IMPL MD_b32 case MD_TokenKind_NumericLiteral: groups |= MD_TokenGroup_Regular|MD_TokenGroup_LabelString; break;
MD_TokenKindIsRegular(MD_TokenKind kind) case MD_TokenKind_StringLiteral: groups |= MD_TokenGroup_Regular|MD_TokenGroup_LabelString; break;
{ case MD_TokenKind_Symbol: groups |= MD_TokenGroup_Regular|MD_TokenGroup_LabelString; break;
return(MD_TokenKind_RegularMin < kind && kind < MD_TokenKind_RegularMax);
//- rjf: comments
case MD_TokenKind_Comment: groups |= MD_TokenGroup_Comment; break;
//- rjf: whitespace
case MD_TokenKind_Whitespace: groups |= MD_TokenGroup_Whitespace; break;
case MD_TokenKind_Newline: groups |= MD_TokenGroup_Whitespace; break;
}
return groups;
} }
MD_FUNCTION MD_Token MD_FUNCTION MD_Token
@@ -1236,6 +1223,8 @@ MD_TokenFromString(MD_String8 string)
case '\'': case '\'':
case '`': case '`':
{ {
token.kind = MD_TokenKind_StringLiteral;
// TODO(allen): proposal: // TODO(allen): proposal:
// go see the proposal in the block comment lexer, same idea here? // go see the proposal in the block comment lexer, same idea here?
@@ -1318,14 +1307,15 @@ MD_TokenFromString(MD_String8 string)
} }
} }
// set token kind //- rjf: set relevant node flags on token
token.node_flags |= MD_NodeFlag_StringLiteral;
if(is_triplet) if(is_triplet)
{ {
switch(d) switch(d)
{ {
case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuoteTriplet; break; case '\'': token.node_flags |= MD_NodeFlag_StringTripletSingleQuote; break;
case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuoteTriplet; break; case '"': token.node_flags |= MD_NodeFlag_StringTripletDoubleQuote; break;
case '`': token.kind = MD_TokenKind_StringLiteralTickTriplet; break; case '`': token.node_flags |= MD_NodeFlag_StringTripletTick; break;
default: break; default: break;
} }
} }
@@ -1333,9 +1323,9 @@ MD_TokenFromString(MD_String8 string)
{ {
switch(d) switch(d)
{ {
case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuote; break; case '\'': token.node_flags |= MD_NodeFlag_StringSingleQuote; break;
case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuote; break; case '"': token.node_flags |= MD_NodeFlag_StringDoubleQuote; break;
case '`': token.kind = MD_TokenKind_StringLiteralTick; break; case '`': token.node_flags |= MD_NodeFlag_StringTick; break;
default: break; default: break;
} }
} }
@@ -1347,6 +1337,7 @@ MD_TokenFromString(MD_String8 string)
{ {
if (MD_CharIsAlpha(*at) || *at == '_') if (MD_CharIsAlpha(*at) || *at == '_')
{ {
token.node_flags |= MD_NodeFlag_Identifier;
token.kind = MD_TokenKind_Identifier; token.kind = MD_TokenKind_Identifier;
at += 1; at += 1;
MD_TokenizerScan(MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '_'); MD_TokenizerScan(MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '_');
@@ -1355,6 +1346,7 @@ MD_TokenFromString(MD_String8 string)
else if (MD_CharIsDigit(*at) || else if (MD_CharIsDigit(*at) ||
(at + 1 < one_past_last && at[0] == '-' && MD_CharIsDigit(at[1]))) (at + 1 < one_past_last && at[0] == '-' && MD_CharIsDigit(at[1])))
{ {
token.node_flags |= MD_NodeFlag_Numeric;
token.kind = MD_TokenKind_NumericLiteral; token.kind = MD_TokenKind_NumericLiteral;
at += 1; at += 1;
MD_TokenizerScan(MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '.'); MD_TokenizerScan(MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '.');
@@ -1397,9 +1389,10 @@ MD_BytesFromStringTokenGroupRun(MD_String8 string, MD_TokenGroups groups)
loop: loop:
{ {
MD_Token token = MD_TokenFromString(MD_StringSkip(string, result)); MD_Token token = MD_TokenFromString(MD_StringSkip(string, result));
if((skip_comment && MD_TokenKindIsComment(token.kind)) || MD_TokenGroups groups = MD_TokenGroupsFromTokenKind(token.kind);
(skip_whitespace && MD_TokenKindIsWhitespace(token.kind)) || if((skip_comment && groups & MD_TokenGroup_Comment) ||
(skip_regular && MD_TokenKindIsRegular(token.kind))) (skip_whitespace && groups & MD_TokenGroup_Whitespace) ||
(skip_regular && groups & MD_TokenGroup_Regular))
{ {
result += token.outer_string.size; result += token.outer_string.size;
goto loop; goto loop;
@@ -1791,7 +1784,7 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
MD_MemoryZero(&comment_token, sizeof(comment_token)); MD_MemoryZero(&comment_token, sizeof(comment_token));
} }
} }
else if(MD_TokenKindIsWhitespace(token.kind)) else if(MD_TokenGroupsFromTokenKind(token.kind) & MD_TokenGroup_Whitespace)
{ {
off += token.outer_string.size; off += token.outer_string.size;
} }
@@ -1842,20 +1835,12 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
//- rjf: try to parse regular node, with/without children //- rjf: try to parse regular node, with/without children
off += MD_BytesFromStringTokenGroupRun(MD_StringSkip(string, off), MD_TokenGroup_Comment|MD_TokenGroup_Whitespace); off += MD_BytesFromStringTokenGroupRun(MD_StringSkip(string, off), MD_TokenGroup_Comment|MD_TokenGroup_Whitespace);
MD_Token label_name = MD_TokenFromString(MD_StringSkip(string, off)); MD_Token label_name = MD_TokenFromString(MD_StringSkip(string, off));
if(label_name.kind == MD_TokenKind_Identifier || if(MD_TokenGroupsFromTokenKind(label_name.kind) & MD_TokenGroup_LabelString)
label_name.kind == MD_TokenKind_NumericLiteral ||
label_name.kind == MD_TokenKind_StringLiteralTick ||
label_name.kind == MD_TokenKind_StringLiteralSingleQuote ||
label_name.kind == MD_TokenKind_StringLiteralDoubleQuote ||
label_name.kind == MD_TokenKind_StringLiteralTickTriplet ||
label_name.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet ||
label_name.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet ||
label_name.kind == MD_TokenKind_Symbol )
{ {
off += label_name.outer_string.size; off += label_name.outer_string.size;
parsed_node = MD_MakeNode(MD_NodeKind_Label, label_name.string, label_name.outer_string, parsed_node = MD_MakeNode(MD_NodeKind_Label, label_name.string, label_name.outer_string,
label_name.outer_string.str - string.str); label_name.outer_string.str - string.str);
parsed_node->flags |= MD_NodeFlagsFromTokenKind(label_name.kind); parsed_node->flags |= label_name.node_flags;
//- rjf: check for string literal errors //- rjf: check for string literal errors
// TODO(rjf): Before we were just able to check one kind. I think preserving // TODO(rjf): Before we were just able to check one kind. I think preserving
@@ -1871,12 +1856,7 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
// //
// If that turned out to be a good idea, maybe we could do something like // If that turned out to be a good idea, maybe we could do something like
// MD_TokenKindIsLegalLabelHead, MD_TokenKindNeedsBalancing?? I don't know. // MD_TokenKindIsLegalLabelHead, MD_TokenKindNeedsBalancing?? I don't know.
if(label_name.kind == MD_TokenKind_StringLiteralTick || if(label_name.kind == MD_TokenKind_StringLiteral)
label_name.kind == MD_TokenKind_StringLiteralSingleQuote ||
label_name.kind == MD_TokenKind_StringLiteralDoubleQuote ||
label_name.kind == MD_TokenKind_StringLiteralTickTriplet ||
label_name.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet ||
label_name.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet)
{ {
if(!_MD_TokenBoundariesAreBalanced(label_name)) if(!_MD_TokenBoundariesAreBalanced(label_name))
{ {
@@ -1989,7 +1969,7 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
{ {
break; break;
} }
else if(MD_TokenKindIsWhitespace(token.kind)) else if(MD_TokenGroupsFromTokenKind(token.kind) & MD_TokenGroup_Whitespace)
{ {
off += token.outer_string.size; off += token.outer_string.size;
} }