Report and skip non-ASCII characters.

This commit is contained in:
Miguel Lechon
2021-03-14 12:12:30 +01:00
parent e3357f0759
commit 084d55d55d
4 changed files with 24 additions and 3 deletions
+4
View File
@@ -451,6 +451,10 @@ typedef enum MD_TokenKind
MD_TokenKind_Whitespace,
MD_TokenKind_Newline,
MD_TokenKind_WhitespaceMax,
MD_TokenKind_NonASCII,
// Provisional category to deal with single characters > 127
// while we don't support encodings other than ASCII
MD_TokenKind_MAX,
}