Report and skip non-ASCII characters.

source/md.h

@@ -451,6 +451,10 @@ typedef enum MD_TokenKind
     MD_TokenKind_Whitespace,
     MD_TokenKind_Newline,
     MD_TokenKind_WhitespaceMax,
+
+    MD_TokenKind_NonASCII,
+    // Provisional category to deal with single characters > 127
+    // while we don't support encodings other than ASCII
     
     MD_TokenKind_MAX,
 }

source/md_impl.c

@@ -1292,6 +1292,12 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
                     token.kind = MD_TokenKind_Symbol;
                     at += 1;
                 }
+
+                else
+                {
+                    token.kind = MD_TokenKind_NonASCII;
+                    at += 1;
+                }
             }break;
         }
         
@@ -1689,6 +1695,12 @@ _MD_ParseOneNode(MD_ParseCtx *ctx)
         }
         goto end_parse;
     }
+
+    else if(MD_Parse_RequireKind(ctx, MD_TokenKind_NonASCII,     &token))
+    {
+        result.node = MD_MakeNodeFromToken(MD_NodeKind_Label, ctx->filename, ctx->file_contents.str, ctx->at, token);
+        _MD_Error(ctx, result.node, ctx->at-1, 0, "Non-ASCII character %d", token.string.str[0]);
+    }
     
     end_parse:;
     

tests/grammar.md

@@ -34,7 +34,7 @@ symbol_no_backslash_no_quotes_2 : ';'|'<'|'='|'>'|'?'|'@'|'['|']'|'^'|'_'|'{'|'|
 string_literal                  : @markup '"' [string_literal_items] @markup '"' | @markup '`' [string_literal_items] @markup '`'
 string_literal_items            : string_literal_item [string_literal_items]
 string_literal_item             : ascii_no_backslash_no_quotes | '\'' | '\\' ascii
-symbol_label                    : '~'|'!'|'%'|'^'|'&'|'*'|'+'|'-'|'/'|'|'|'<'|'>'|'$'|'='|'.'|'?'|'$'
+symbol_label                    : '~'|'!'|'%'|'^'|'&'|'*'|'+'|'-'|'/'|'|'|'<'|'>'|'$'|'='|'.'|'?'|'_'
 
 /* What follows is a range of annotated grammars that can be used to generate
  * tests of increasing complexity and completeness to check against MetaDesk.

tests/sanity_tests.c

@@ -455,9 +455,9 @@ int main(void)
             {"a:'''\nmulti-line text literal", {3}},
             {"/* foo", {1}},
             {"label:@tag {1, 2, 3} /* /* unterminated comment */", {8, 22}},
-            {"#include <stdio.h>", {2}},
             {"@\"tag\" node", {2}},
-            {"{a,,#b}", {4, 5}},
+            {"{a,,#b,}", {4, 5}},
+            {"foo""\x80""bar", {4}},
         };
 
         int max_error_count = MD_ArrayCount(tests[0].columns);
@@ -478,6 +478,11 @@ int main(void)
                     }
                     e = e->next;
                 }
+
+                if(e && e->next)
+                {
+                    columns_match = 0;
+                }
             }
             TestResult(columns_match);
         }