ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-08-06 15:48:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

better lexer testing in sanity_tests

Browse Source
This commit is contained in:
Allen Webster
2021-09-02 19:49:09 -07:00
parent 213c20fd93
commit eef0ff2577
2 changed files with 19 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/md.c
+1 -1
View File
@@ -1917,7 +1917,7 @@ MD_TokenFromString(MD_String8 string)
at += 1; at += 1;
} }
} }
else if (MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '.'){ else if (MD_CharIsAlpha(*at) || MD_CharIsDigit(*at) || *at == '.' || *at == '_'){
good = 1; good = 1;
at += 1; at += 1;
} }
tests/sanity_tests.c
+18 -20
View File
@@ -88,24 +88,17 @@ int main(void)
Test("Lexer") Test("Lexer")
{ {
MD_String8 string = MD_S8Lit("abc def 123 456 123_456 abc123 123abc"); MD_String8 string = MD_S8Lit("abc def 123 456 123_456 abc123 123abc +-*");
MD_Token tokens[] = MD_Token tokens[100];
{
MD_TokenFromString(MD_S8Skip(string, 0)), MD_Token *token = tokens;
MD_TokenFromString(MD_S8Skip(string, 3)), MD_Token *token_opl = tokens + MD_ArrayCount(tokens);
MD_TokenFromString(MD_S8Skip(string, 4)), MD_u64 pos = 0;
MD_TokenFromString(MD_S8Skip(string, 7)), for (; pos < string.size && token < token_opl; ){
MD_TokenFromString(MD_S8Skip(string, 8)), *token = MD_TokenFromString(MD_S8Skip(string, pos));
MD_TokenFromString(MD_S8Skip(string, 11)), pos += token->raw_string.size;
MD_TokenFromString(MD_S8Skip(string, 12)), token += 1;
MD_TokenFromString(MD_S8Skip(string, 15)), }
MD_TokenFromString(MD_S8Skip(string, 16)),
MD_TokenFromString(MD_S8Skip(string, 19)),
MD_TokenFromString(MD_S8Skip(string, 20)),
MD_TokenFromString(MD_S8Skip(string, 23)),
MD_TokenFromString(MD_S8Skip(string, 24)),
MD_TokenFromString(MD_S8Skip(string, 27)),
};
TestResult(TokenMatch(tokens[0], MD_S8Lit("abc"), MD_TokenKind_Identifier)); TestResult(TokenMatch(tokens[0], MD_S8Lit("abc"), MD_TokenKind_Identifier));
TestResult(TokenMatch(tokens[1], MD_S8Lit(" "), MD_TokenKind_Whitespace)); TestResult(TokenMatch(tokens[1], MD_S8Lit(" "), MD_TokenKind_Whitespace));
@@ -115,8 +108,13 @@ int main(void)
TestResult(TokenMatch(tokens[5], MD_S8Lit(" "), MD_TokenKind_Whitespace)); TestResult(TokenMatch(tokens[5], MD_S8Lit(" "), MD_TokenKind_Whitespace));
TestResult(TokenMatch(tokens[6], MD_S8Lit("456"), MD_TokenKind_NumericLiteral)); TestResult(TokenMatch(tokens[6], MD_S8Lit("456"), MD_TokenKind_NumericLiteral));
TestResult(TokenMatch(tokens[7], MD_S8Lit(" "), MD_TokenKind_Whitespace)); TestResult(TokenMatch(tokens[7], MD_S8Lit(" "), MD_TokenKind_Whitespace));
// TODO(rjf): Enable once numeric literal lexing is fixed TestResult(TokenMatch(tokens[8], MD_S8Lit("123_456"), MD_TokenKind_NumericLiteral));
//TestResult(TokenMatch(MD_Parse_LexNext(&ctx), MD_S8Lit("123_456"), MD_TokenKind_NumericLiteral)); TestResult(TokenMatch(tokens[9], MD_S8Lit(" "), MD_TokenKind_Whitespace));
TestResult(TokenMatch(tokens[10], MD_S8Lit("abc123"), MD_TokenKind_Identifier));
TestResult(TokenMatch(tokens[11], MD_S8Lit(" "), MD_TokenKind_Whitespace));
TestResult(TokenMatch(tokens[12], MD_S8Lit("123abc"), MD_TokenKind_NumericLiteral));
TestResult(TokenMatch(tokens[13], MD_S8Lit(" "), MD_TokenKind_Whitespace));
TestResult(TokenMatch(tokens[14], MD_S8Lit("+-*"), MD_TokenKind_Symbol));
} }
Test("Empty Sets") Test("Empty Sets")