diff --git a/src/base/base_strings.c b/src/base/base_strings.c index 63faf8e2..f98d6e44 100644 --- a/src/base/base_strings.c +++ b/src/base/base_strings.c @@ -1517,7 +1517,7 @@ utf8_decode(U8 *str, U64 max){ }break; case 2: { - if (2 < max) + if (1 < max) { U8 cont_byte = str[1]; if (utf8_class[cont_byte >> 3] == 0) diff --git a/src/text_cache/text_cache.c b/src/text_cache/text_cache.c index e8706c1d..29200950 100644 --- a/src/text_cache/text_cache.c +++ b/src/text_cache/text_cache.c @@ -262,7 +262,7 @@ txt_token_array_from_string__c_cpp(Arena *arena, U64 *bytes_processed_counter, S }break; case TXT_TokenKind_Identifier: { - ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$'); + ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128); }break; case TXT_TokenKind_Numeric: { @@ -606,7 +606,7 @@ txt_token_array_from_string__odin(Arena *arena, U64 *bytes_processed_counter, St }break; case TXT_TokenKind_Identifier: { - ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$'); + ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128); }break; case TXT_TokenKind_Numeric: { @@ -892,7 +892,7 @@ txt_token_array_from_string__jai(Arena *arena, U64 *bytes_processed_counter, Str }break; case TXT_TokenKind_Identifier: { - ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$'); + ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128); }break; case TXT_TokenKind_Numeric: { @@ -1177,7 +1177,7 @@ txt_token_array_from_string__zig(Arena *arena, U64 *bytes_processed_counter, Str }break; case TXT_TokenKind_Identifier: { - ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$'); + ender_found = (!char_is_alpha(byte) && !char_is_digit(byte, 10) && byte != '_' && byte != '$' && byte < 128); }break; case TXT_TokenKind_Numeric: {