ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-08-04 22:58:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make core:odin use a string for the source rather than []byte

Browse Source
This commit is contained in:
gingerBill
2021-05-16 12:38:27 +01:00
parent ce08e832f7
commit df3512b112
3 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/odin/tokenizer/tokenizer.odin
+3 -3
View File
@@ -14,7 +14,7 @@ Flags :: distinct bit_set[Flag; u32];
Tokenizer :: struct {
// Immutable data
path: string,
src: []byte,
src: string,
err: Error_Handler,
flags: Flags,
@@ -31,7 +31,7 @@ Tokenizer :: struct {
error_count: int,
}
init :: proc(t: ^Tokenizer, src: []byte, path: string, err: Error_Handler = default_error_handler) {
init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = default_error_handler) {
t.src = src;
t.err = err;
t.ch = ' ';
@@ -87,7 +87,7 @@ advance_rune :: proc(using t: ^Tokenizer) {
case r == 0:
error(t, t.offset, "illegal character NUL");
case r >= utf8.RUNE_SELF:
r, w = utf8.decode_rune(src[read_offset:]);
r, w = utf8.decode_rune_in_string(src[read_offset:]);
if r == utf8.RUNE_ERROR && w == 1 {
error(t, t.offset, "illegal UTF-8 encoding");
} else if r == utf8.RUNE_BOM && offset > 0 {