ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-07-29 10:50:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into prototype-fmt

Browse Source
This commit is contained in:
Daniel Gavin
2021-05-20 12:15:14 +02:00
parent 50035f257e 92abddddc5
commit 44ee0f2cdc
79 changed files with 8070 additions and 3455 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/odin/tokenizer/token.odin
+2
View File
@@ -107,6 +107,7 @@ Token_Kind :: enum u32 {
Comma, // ,
Ellipsis, // ..
Range_Half, // ..<
Range_Full, // ..=
Back_Slash, // \
B_Operator_End,
@@ -233,6 +234,7 @@ tokens := [Token_Kind.COUNT]string {
",",
"..",
"..<",
"..=",
"\\",
"",
core/odin/tokenizer/tokenizer.odin
+6 -3
View File
@@ -14,7 +14,7 @@ Flags :: distinct bit_set[Flag; u32];
Tokenizer :: struct {
// Immutable data
path: string,
src: []byte,
src: string,
err: Error_Handler,
flags: Flags,
@@ -31,7 +31,7 @@ Tokenizer :: struct {
error_count: int,
}
init :: proc(t: ^Tokenizer, src: []byte, path: string, err: Error_Handler = default_error_handler) {
init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = default_error_handler) {
t.src = src;
t.err = err;
t.ch = ' ';
@@ -87,7 +87,7 @@ advance_rune :: proc(using t: ^Tokenizer) {
case r == 0:
error(t, t.offset, "illegal character NUL");
case r >= utf8.RUNE_SELF:
r, w = utf8.decode_rune(src[read_offset:]);
r, w = utf8.decode_rune_in_string(src[read_offset:]);
if r == utf8.RUNE_ERROR && w == 1 {
error(t, t.offset, "illegal UTF-8 encoding");
} else if r == utf8.RUNE_BOM && offset > 0 {
@@ -623,6 +623,9 @@ scan :: proc(t: ^Tokenizer) -> Token {
if t.ch == '<' {
advance_rune(t);
kind = .Range_Half;
} else if t.ch == '=' {
advance_rune(t);
kind = .Range_Full;
}
}
}