Report and skip non-ASCII characters.

This commit is contained in:
Miguel Lechon
2021-03-14 12:12:30 +01:00
parent e3357f0759
commit 084d55d55d
4 changed files with 24 additions and 3 deletions
+4
View File
@@ -451,6 +451,10 @@ typedef enum MD_TokenKind
MD_TokenKind_Whitespace,
MD_TokenKind_Newline,
MD_TokenKind_WhitespaceMax,
MD_TokenKind_NonASCII,
// Provisional category to deal with single characters > 127
// while we don't support encodings other than ASCII
MD_TokenKind_MAX,
}
+12
View File
@@ -1292,6 +1292,12 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
token.kind = MD_TokenKind_Symbol;
at += 1;
}
else
{
token.kind = MD_TokenKind_NonASCII;
at += 1;
}
}break;
}
@@ -1689,6 +1695,12 @@ _MD_ParseOneNode(MD_ParseCtx *ctx)
}
goto end_parse;
}
else if(MD_Parse_RequireKind(ctx, MD_TokenKind_NonASCII, &token))
{
result.node = MD_MakeNodeFromToken(MD_NodeKind_Label, ctx->filename, ctx->file_contents.str, ctx->at, token);
_MD_Error(ctx, result.node, ctx->at-1, 0, "Non-ASCII character %d", token.string.str[0]);
}
end_parse:;