ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-08-06 23:58:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

introduce Reserved token kind; get parser (mostly) working with new token kind

Browse Source
This commit is contained in:
Allen Webster
2021-06-30 16:49:37 -04:00
parent dbfd86d558
commit 297b1e9259
3 changed files with 80 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.bat
+3
View File
@@ -22,6 +22,9 @@ cl %compile_flags% ..\tests\cpp_build_test.cpp
cl %compile_flags% ..\tests\grammar.c cl %compile_flags% ..\tests\grammar.c
popd popd
rem Stop wasting time getting hung on broken parser
rem exit
echo. echo.
echo ~~~ Running Sanity Tests ~~~ echo ~~~ Running Sanity Tests ~~~
pushd build pushd build
source/md.h
+1 -1
View File
@@ -630,7 +630,7 @@ MD_FUNCTION MD_b32 MD_CharIsAlpha(MD_u8 c);
MD_FUNCTION MD_b32 MD_CharIsAlphaUpper(MD_u8 c); MD_FUNCTION MD_b32 MD_CharIsAlphaUpper(MD_u8 c);
MD_FUNCTION MD_b32 MD_CharIsAlphaLower(MD_u8 c); MD_FUNCTION MD_b32 MD_CharIsAlphaLower(MD_u8 c);
MD_FUNCTION MD_b32 MD_CharIsDigit(MD_u8 c); MD_FUNCTION MD_b32 MD_CharIsDigit(MD_u8 c);
MD_FUNCTION MD_b32 MD_CharIsSymbol(MD_u8 c); MD_FUNCTION MD_b32 MD_CharIsUnreservedSymbol(MD_u8 c);
MD_FUNCTION MD_b32 MD_CharIsReservedSymbol(MD_u8 c); MD_FUNCTION MD_b32 MD_CharIsReservedSymbol(MD_u8 c);
MD_FUNCTION MD_b32 MD_CharIsSpace(MD_u8 c); MD_FUNCTION MD_b32 MD_CharIsSpace(MD_u8 c);
MD_FUNCTION MD_u8 MD_CharToUpper(MD_u8 c); MD_FUNCTION MD_u8 MD_CharToUpper(MD_u8 c);
source/md_impl.c
+76 -75
View File
@@ -83,14 +83,12 @@ MD_CharIsDigit(MD_u8 c)
} }
MD_FUNCTION_IMPL MD_b32 MD_FUNCTION_IMPL MD_b32
MD_CharIsSymbol(MD_u8 c) MD_CharIsUnreservedSymbol(MD_u8 c)
{ {
return (c == '~' || c == '!' || c == '@' || c == '#' || c == '$' || return (c == '~' || c == '!' || c == '$' || c == '%' || c == '^' ||
c == '%' || c == '^' || c == '&' || c == '*' || c == '(' || c == '&' || c == '*' || c == '-' || c == '=' || c == '+' ||
c == ')' || c == '-' || c == '=' || c == '+' || c == '[' || c == '<' || c == '.' || c == '>' || c == '/' || c == '?' ||
c == ']' || c == '{' || c == '}' || c == ':' || c == ';' || c == '|' || c == '\\');
c == ',' || c == '<' || c == '.' || c == '>' || c == '/' ||
c == '?' || c == '|' || c == '\\');
} }
MD_FUNCTION_IMPL MD_b32 MD_FUNCTION_IMPL MD_b32
@@ -1283,13 +1281,19 @@ MD_TokenFromString(MD_String8 string)
MD_TokenizerScan(MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '.'); MD_TokenizerScan(MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '.');
} }
else if (MD_CharIsSymbol(*at)) else if (MD_CharIsUnreservedSymbol(*at))
{ {
symbol_lex: symbol_lex:
token.kind = MD_TokenKind_Symbol; token.kind = MD_TokenKind_Symbol;
at += 1; at += 1;
} }
else if (MD_CharIsReservedSymbol(*at))
{
token.kind = MD_TokenKind_Reserved;
at += 1;
}
else else
{ {
token.kind = MD_TokenKind_BadCharacter; token.kind = MD_TokenKind_BadCharacter;
@@ -1409,9 +1413,10 @@ MD_ParseNodeSet(MD_String8 string, MD_u64 offset, MD_Node *parent, MD_ParseSetRu
MD_u64 opener_check_off = off; MD_u64 opener_check_off = off;
opener_check_off += MD_LexAdvanceFromSkips(MD_StringSkip(string, opener_check_off), MD_TokenGroup_Irregular); opener_check_off += MD_LexAdvanceFromSkips(MD_StringSkip(string, opener_check_off), MD_TokenGroup_Irregular);
initial_token = MD_TokenFromString(MD_StringSkip(string, opener_check_off)); initial_token = MD_TokenFromString(MD_StringSkip(string, opener_check_off));
if(initial_token.kind == MD_TokenKind_Symbol) if(initial_token.kind == MD_TokenKind_Reserved)
{ {
if(MD_StringMatch(initial_token.outer_string, MD_S8Lit("{"), 0)) MD_u8 c = initial_token.outer_string.str[0];
if(c == '{')
{ {
set_opener = '{'; set_opener = '{';
set_opener_flags |= MD_NodeFlag_BraceLeft; set_opener_flags |= MD_NodeFlag_BraceLeft;
@@ -1419,7 +1424,7 @@ MD_ParseNodeSet(MD_String8 string, MD_u64 offset, MD_Node *parent, MD_ParseSetRu
off = opener_check_off; off = opener_check_off;
close_with_brace = 1; close_with_brace = 1;
} }
else if(MD_StringMatch(initial_token.outer_string, MD_S8Lit("("), 0)) else if(c == '(')
{ {
set_opener = '('; set_opener = '(';
set_opener_flags |= MD_NodeFlag_ParenLeft; set_opener_flags |= MD_NodeFlag_ParenLeft;
@@ -1427,7 +1432,7 @@ MD_ParseNodeSet(MD_String8 string, MD_u64 offset, MD_Node *parent, MD_ParseSetRu
off = opener_check_off; off = opener_check_off;
close_with_paren = 1; close_with_paren = 1;
} }
else if(MD_StringMatch(initial_token.outer_string, MD_S8Lit("["), 0)) else if(c == '[')
{ {
set_opener = '['; set_opener = '[';
set_opener_flags |= MD_NodeFlag_BracketLeft; set_opener_flags |= MD_NodeFlag_BracketLeft;
@@ -1501,20 +1506,19 @@ MD_ParseNodeSet(MD_String8 string, MD_u64 offset, MD_Node *parent, MD_ParseSetRu
{ {
closer_check_off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular); closer_check_off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular);
MD_Token potential_closer = MD_TokenFromString(MD_StringSkip(string, closer_check_off)); MD_Token potential_closer = MD_TokenFromString(MD_StringSkip(string, closer_check_off));
if(potential_closer.kind == MD_TokenKind_Symbol && if(potential_closer.kind == MD_TokenKind_Reserved)
(MD_StringMatch(potential_closer.outer_string, MD_S8Lit(","), 0) ||
MD_StringMatch(potential_closer.outer_string, MD_S8Lit(";"), 0)))
{ {
closer_check_off += potential_closer.outer_string.size; MD_u8 c = potential_closer.outer_string.str[0];
off = closer_check_off; if (c == ',' || c == ';')
break; {
} closer_check_off += potential_closer.outer_string.size;
else if(potential_closer.kind == MD_TokenKind_Symbol && off = closer_check_off;
(MD_StringMatch(potential_closer.string, MD_S8Lit("}"), 0) || break;
MD_StringMatch(potential_closer.string, MD_S8Lit("]"), 0) || }
MD_StringMatch(potential_closer.string, MD_S8Lit(")"), 0))) else if(c == '}' || c == ']'|| c == ')')
{ {
goto end_parse; goto end_parse;
}
} }
} }
@@ -1526,9 +1530,10 @@ MD_ParseNodeSet(MD_String8 string, MD_u64 offset, MD_Node *parent, MD_ParseSetRu
MD_u64 closer_check_off = off; MD_u64 closer_check_off = off;
closer_check_off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular); closer_check_off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular);
MD_Token potential_closer = MD_TokenFromString(MD_StringSkip(string, closer_check_off)); MD_Token potential_closer = MD_TokenFromString(MD_StringSkip(string, closer_check_off));
if(potential_closer.kind == MD_TokenKind_Symbol) if(potential_closer.kind == MD_TokenKind_Reserved)
{ {
if(close_with_brace && MD_StringMatch(potential_closer.outer_string, MD_S8Lit("}"), 0)) MD_u8 c = potential_closer.outer_string.str[0];
if(close_with_brace && c == '}')
{ {
closer_check_off += potential_closer.outer_string.size; closer_check_off += potential_closer.outer_string.size;
off = closer_check_off; off = closer_check_off;
@@ -1536,7 +1541,7 @@ MD_ParseNodeSet(MD_String8 string, MD_u64 offset, MD_Node *parent, MD_ParseSetRu
got_closer = 1; got_closer = 1;
break; break;
} }
else if(close_with_paren && MD_StringMatch(potential_closer.outer_string, MD_S8Lit("]"), 0)) else if(close_with_paren && c == ']')
{ {
closer_check_off += potential_closer.outer_string.size; closer_check_off += potential_closer.outer_string.size;
off = closer_check_off; off = closer_check_off;
@@ -1544,7 +1549,7 @@ MD_ParseNodeSet(MD_String8 string, MD_u64 offset, MD_Node *parent, MD_ParseSetRu
got_closer = 1; got_closer = 1;
break; break;
} }
else if(close_with_paren && MD_StringMatch(potential_closer.outer_string, MD_S8Lit(")"), 0)) else if(close_with_paren && c == ')')
{ {
closer_check_off += potential_closer.outer_string.size; closer_check_off += potential_closer.outer_string.size;
off = closer_check_off; off = closer_check_off;
@@ -1573,17 +1578,18 @@ MD_ParseNodeSet(MD_String8 string, MD_u64 offset, MD_Node *parent, MD_ParseSetRu
{ {
off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular); off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular);
MD_Token trailing_separator = MD_TokenFromString(MD_StringSkip(string, off)); MD_Token trailing_separator = MD_TokenFromString(MD_StringSkip(string, off));
if(MD_StringMatch(trailing_separator.string, MD_S8Lit(","), 0) && if (trailing_separator.kind == MD_TokenKind_Reserved){
trailing_separator.kind == MD_TokenKind_Symbol) MD_u8 c = trailing_separator.string.str[0];
{ if(c == ',')
trailing_separator_flags |= MD_NodeFlag_BeforeComma; {
off += trailing_separator.outer_string.size; trailing_separator_flags |= MD_NodeFlag_BeforeComma;
} off += trailing_separator.outer_string.size;
else if(MD_StringMatch(trailing_separator.string, MD_S8Lit(";"), 0) && }
trailing_separator.kind == MD_TokenKind_Symbol) else if(c == ';')
{ {
trailing_separator_flags |= MD_NodeFlag_BeforeSemicolon; trailing_separator_flags |= MD_NodeFlag_BeforeSemicolon;
off += trailing_separator.outer_string.size; off += trailing_separator.outer_string.size;
}
} }
} }
@@ -1631,8 +1637,8 @@ MD_ParseTagList(MD_String8 string, MD_u64 offset)
//- rjf: parse @ symbol, signifying start of tag //- rjf: parse @ symbol, signifying start of tag
off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular); off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular);
MD_Token next_token = MD_TokenFromString(MD_StringSkip(string, off)); MD_Token next_token = MD_TokenFromString(MD_StringSkip(string, off));
if(!MD_StringMatch(next_token.string, MD_S8Lit("@"), 0) || if(next_token.kind != MD_TokenKind_Reserved ||
next_token.kind != MD_TokenKind_Symbol) next_token.string.str[0] != '@')
{ {
break; break;
} }
@@ -1657,8 +1663,8 @@ MD_ParseTagList(MD_String8 string, MD_u64 offset)
//- rjf: parse tag arguments //- rjf: parse tag arguments
MD_Token open_paren = MD_TokenFromString(MD_StringSkip(string, off)); MD_Token open_paren = MD_TokenFromString(MD_StringSkip(string, off));
MD_ParseResult args_parse = MD_ParseResultZero(); MD_ParseResult args_parse = MD_ParseResultZero();
if(MD_StringMatch(open_paren.string, MD_S8Lit("("), 0) && if(open_paren.kind == MD_TokenKind_Reserved &&
open_paren.kind == MD_TokenKind_Symbol) open_paren.string.str[0] == '(')
{ {
args_parse = MD_ParseNodeSet(string, off, tag, MD_ParseSetRule_EndOnDelimiter); args_parse = MD_ParseNodeSet(string, off, tag, MD_ParseSetRule_EndOnDelimiter);
MD_PushErrorListToList(&result.errors, &args_parse.errors); MD_PushErrorListToList(&result.errors, &args_parse.errors);
@@ -1732,17 +1738,30 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
//- rjf: try to parse an unnamed set //- rjf: try to parse an unnamed set
off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular); off += MD_LexAdvanceFromSkips(MD_StringSkip(string, off), MD_TokenGroup_Irregular);
MD_Token unnamed_set_opener = MD_TokenFromString(MD_StringSkip(string, off)); MD_Token unnamed_set_opener = MD_TokenFromString(MD_StringSkip(string, off));
if(unnamed_set_opener.kind == MD_TokenKind_Symbol && if(unnamed_set_opener.kind == MD_TokenKind_Reserved)
(MD_StringMatch(unnamed_set_opener.string, MD_S8Lit("("), 0) ||
MD_StringMatch(unnamed_set_opener.string, MD_S8Lit("{"), 0) ||
MD_StringMatch(unnamed_set_opener.string, MD_S8Lit("["), 0)))
{ {
parsed_node = MD_MakeNode(MD_NodeKind_Label, MD_S8Lit(""), MD_S8Lit(""), MD_u8 c = unnamed_set_opener.string.str[0];
unnamed_set_opener.outer_string.str - string.str); if (c == '(' || c == '{' || c == '['){
children_parse = MD_ParseNodeSet(string, off, parsed_node, MD_ParseSetRule_EndOnDelimiter); parsed_node = MD_MakeNode(MD_NodeKind_Label, MD_S8Lit(""), MD_S8Lit(""),
off += children_parse.bytes_parsed; unnamed_set_opener.outer_string.str - string.str);
MD_PushErrorListToList(&result.errors, &children_parse.errors); children_parse = MD_ParseNodeSet(string, off, parsed_node, MD_ParseSetRule_EndOnDelimiter);
off += children_parse.bytes_parsed;
MD_PushErrorListToList(&result.errors, &children_parse.errors);
}
else if (c == ')' || c == '}' || c == ']')
{
MD_Error *error = MD_MakeNodeError(parsed_node, MD_MessageKind_CatastrophicError, MD_PushStringF("Unbalanced \"%c\"", c));
MD_PushErrorToList(&result.errors, error);
off += unnamed_set_opener.outer_string.size;
}
else
{
MD_Error *error = MD_MakeNodeError(parsed_node, MD_MessageKind_Error, MD_PushStringF("Unexpected reserved symbol \"%c\"", c));
MD_PushErrorToList(&result.errors, error);
off += unnamed_set_opener.outer_string.size;
}
goto end_parse; goto end_parse;
} }
//- rjf: try to parse regular node, with/without children //- rjf: try to parse regular node, with/without children
@@ -1755,31 +1774,12 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
label_name.outer_string.str - string.str); label_name.outer_string.str - string.str);
parsed_node->flags |= label_name.node_flags; parsed_node->flags |= label_name.node_flags;
if(label_name.kind == MD_TokenKind_StringLiteral)
{
}
//- rjf: check for unexpected reserved symbols
else if(label_name.kind == MD_TokenKind_Symbol && label_name.string.size == 1 && MD_CharIsReservedSymbol(label_name.string.str[0]))
{
MD_u8 c = label_name.string.str[0];
if(c == '}' || c == ']' || c == ')')
{
MD_Error *error = MD_MakeNodeError(parsed_node, MD_MessageKind_CatastrophicError, MD_PushStringF("Unbalanced \"%c\"", c));
MD_PushErrorToList(&result.errors, error);
}
else
{
MD_Error *error = MD_MakeNodeError(parsed_node, MD_MessageKind_Error, MD_PushStringF("Unexpected reserved symbol \"%c\"", c));
MD_PushErrorToList(&result.errors, error);
}
}
//- rjf: try to parse children for this node //- rjf: try to parse children for this node
MD_u64 colon_check_off = off; MD_u64 colon_check_off = off;
colon_check_off += MD_LexAdvanceFromSkips(MD_StringSkip(string, colon_check_off), MD_TokenGroup_Irregular); colon_check_off += MD_LexAdvanceFromSkips(MD_StringSkip(string, colon_check_off), MD_TokenGroup_Irregular);
MD_Token colon = MD_TokenFromString(MD_StringSkip(string, colon_check_off)); MD_Token colon = MD_TokenFromString(MD_StringSkip(string, colon_check_off));
if(MD_StringMatch(colon.string, MD_S8Lit(":"), 0) && colon.kind == MD_TokenKind_Symbol) if(colon.kind == MD_TokenKind_Reserved &&
colon.string.str[0] == ':')
{ {
colon_check_off += colon.outer_string.size; colon_check_off += colon.outer_string.size;
off = colon_check_off; off = colon_check_off;
@@ -1799,7 +1799,8 @@ MD_ParseOneNode(MD_String8 string, MD_u64 offset)
for(MD_u64 tag_check_off = off; tag_check_off < string.size;) for(MD_u64 tag_check_off = off; tag_check_off < string.size;)
{ {
MD_Token token = MD_TokenFromString(MD_StringSkip(string, tag_check_off)); MD_Token token = MD_TokenFromString(MD_StringSkip(string, tag_check_off));
if(token.kind == MD_TokenKind_Symbol && MD_StringMatch(token.outer_string, MD_S8Lit("@"), 0)) if(token.kind == MD_TokenKind_Reserved &&
token.outer_string.str[0] == '@')
{ {
MD_Error *error = MD_MakeTokenError(string, token, MD_MessageKind_Error, MD_Error *error = MD_MakeTokenError(string, token, MD_MessageKind_Error,
MD_S8Lit("Tags are not allowed between a `:` and node children. Place tags before the name of the node list.")); MD_S8Lit("Tags are not allowed between a `:` and node children. Place tags before the name of the node list."));