Remove unneeded semicolons from the core library

This commit is contained in:
gingerBill
2021-08-31 22:21:13 +01:00
parent b176af2742
commit 251da264ed
187 changed files with 27227 additions and 27227 deletions
+24 -24
View File
@@ -2,34 +2,34 @@ package c
import b "core:builtin"
CHAR_BIT :: 8;
CHAR_BIT :: 8
bool :: b.bool;
char :: b.u8;
byte :: b.byte;
schar :: b.i8;
uchar :: b.u8;
short :: b.i16;
ushort :: b.u16;
int :: b.i32;
uint :: b.u32;
bool :: b.bool
char :: b.u8
byte :: b.byte
schar :: b.i8
uchar :: b.u8
short :: b.i16
ushort :: b.u16
int :: b.i32
uint :: b.u32
long :: b.i32 when (ODIN_OS == "windows" || size_of(b.rawptr) == 4) else b.i64;
ulong :: b.u32 when (ODIN_OS == "windows" || size_of(b.rawptr) == 4) else b.u64;
long :: b.i32 when (ODIN_OS == "windows" || size_of(b.rawptr) == 4) else b.i64
ulong :: b.u32 when (ODIN_OS == "windows" || size_of(b.rawptr) == 4) else b.u64
longlong :: b.i64;
ulonglong :: b.u64;
float :: b.f32;
double :: b.f64;
complex_float :: b.complex64;
complex_double :: b.complex128;
longlong :: b.i64
ulonglong :: b.u64
float :: b.f32
double :: b.f64
complex_float :: b.complex64
complex_double :: b.complex128
#assert(size_of(b.uintptr) == size_of(b.int));
size_t :: b.uint;
ssize_t :: b.int;
ptrdiff_t :: b.int;
uintptr_t :: b.uintptr;
intptr_t :: b.int;
size_t :: b.uint
ssize_t :: b.int
ptrdiff_t :: b.int
uintptr_t :: b.uintptr
intptr_t :: b.int
wchar_t :: b.u16 when (ODIN_OS == "windows") else b.u32;
wchar_t :: b.u16 when (ODIN_OS == "windows") else b.u32
+7 -7
View File
@@ -6,20 +6,20 @@ const_expr :: proc(rest: ^^Token, tok: ^Token) -> i64 {
// TODO(bill): Handle const_expr correctly
// This is effectively a mini-parser
assert(rest != nil);
assert(tok != nil);
rest^ = tokenizer.new_eof(tok);
assert(rest != nil)
assert(tok != nil)
rest^ = tokenizer.new_eof(tok)
switch v in tok.val {
case i64:
return v;
return v
case f64:
return i64(v);
return i64(v)
case string:
return 0;
return 0
case []u16:
// TODO
case []u32:
// TODO
}
return 0;
return 0
}
File diff suppressed because it is too large Load Diff
+71 -71
View File
@@ -5,150 +5,150 @@ import "core:unicode/utf8"
unquote_char :: proc(str: string, quote: byte) -> (r: rune, multiple_bytes: bool, tail_string: string, success: bool) {
hex_to_int :: proc(c: byte) -> int {
switch c {
case '0'..='9': return int(c-'0');
case 'a'..='f': return int(c-'a')+10;
case 'A'..='F': return int(c-'A')+10;
case '0'..='9': return int(c-'0')
case 'a'..='f': return int(c-'a')+10
case 'A'..='F': return int(c-'A')+10
}
return -1;
return -1
}
w: int;
w: int
if str[0] == quote && quote == '"' {
return;
return
} else if str[0] >= 0x80 {
r, w = utf8.decode_rune_in_string(str);
return r, true, str[w:], true;
r, w = utf8.decode_rune_in_string(str)
return r, true, str[w:], true
} else if str[0] != '\\' {
return rune(str[0]), false, str[1:], true;
return rune(str[0]), false, str[1:], true
}
if len(str) <= 1 {
return;
return
}
s := str;
c := s[1];
s = s[2:];
s := str
c := s[1]
s = s[2:]
switch c {
case: r = rune(c);
case: r = rune(c)
case 'a': r = '\a';
case 'b': r = '\b';
case 'e': r = '\e';
case 'f': r = '\f';
case 'n': r = '\n';
case 'r': r = '\r';
case 't': r = '\t';
case 'v': r = '\v';
case '\\': r = '\\';
case 'a': r = '\a'
case 'b': r = '\b'
case 'e': r = '\e'
case 'f': r = '\f'
case 'n': r = '\n'
case 'r': r = '\r'
case 't': r = '\t'
case 'v': r = '\v'
case '\\': r = '\\'
case '"': r = '"';
case '\'': r = '\'';
case '"': r = '"'
case '\'': r = '\''
case '0'..='7':
v := int(c-'0');
v := int(c-'0')
if len(s) < 2 {
return;
return
}
for i in 0..<len(s) {
d := int(s[i]-'0');
d := int(s[i]-'0')
if d < 0 || d > 7 {
return;
return
}
v = (v<<3) | d;
v = (v<<3) | d
}
s = s[2:];
s = s[2:]
if v > 0xff {
return;
return
}
r = rune(v);
r = rune(v)
case 'x', 'u', 'U':
count: int;
count: int
switch c {
case 'x': count = 2;
case 'u': count = 4;
case 'U': count = 8;
case 'x': count = 2
case 'u': count = 4
case 'U': count = 8
}
if len(s) < count {
return;
return
}
for i in 0..<count {
d := hex_to_int(s[i]);
d := hex_to_int(s[i])
if d < 0 {
return;
return
}
r = (r<<4) | rune(d);
r = (r<<4) | rune(d)
}
s = s[count:];
s = s[count:]
if c == 'x' {
break;
break
}
if r > utf8.MAX_RUNE {
return;
return
}
multiple_bytes = true;
multiple_bytes = true
}
success = true;
tail_string = s;
return;
success = true
tail_string = s
return
}
unquote_string :: proc(lit: string, allocator := context.allocator) -> (res: string, allocated, success: bool) {
contains_rune :: proc(s: string, r: rune) -> int {
for c, offset in s {
if c == r {
return offset;
return offset
}
}
return -1;
return -1
}
assert(len(lit) >= 2);
assert(len(lit) >= 2)
s := lit;
quote := '"';
s := lit
quote := '"'
if s == `""` {
return "", false, true;
return "", false, true
}
if contains_rune(s, '\n') >= 0 {
return s, false, false;
return s, false, false
}
if contains_rune(s, '\\') < 0 && contains_rune(s, quote) < 0 {
if quote == '"' {
return s, false, true;
return s, false, true
}
}
s = s[1:len(s)-1];
s = s[1:len(s)-1]
buf_len := 3*len(s) / 2;
buf := make([]byte, buf_len, allocator);
offset := 0;
buf_len := 3*len(s) / 2
buf := make([]byte, buf_len, allocator)
offset := 0
for len(s) > 0 {
r, multiple_bytes, tail_string, ok := unquote_char(s, byte(quote));
r, multiple_bytes, tail_string, ok := unquote_char(s, byte(quote))
if !ok {
delete(buf);
return s, false, false;
delete(buf)
return s, false, false
}
s = tail_string;
s = tail_string
if r < 0x80 || !multiple_bytes {
buf[offset] = byte(r);
offset += 1;
buf[offset] = byte(r)
offset += 1
} else {
b, w := utf8.encode_rune(r);
copy(buf[offset:], b[:w]);
offset += w;
b, w := utf8.encode_rune(r)
copy(buf[offset:], b[:w])
offset += w
}
}
new_string := string(buf[:offset]);
new_string := string(buf[:offset])
return new_string, true, true;
return new_string, true, true
}
+24 -24
View File
@@ -11,58 +11,58 @@ Hide_Set :: struct {
new_hide_set :: proc(name: string) -> ^Hide_Set {
hs := new(Hide_Set);
hs.name = name;
return hs;
hs := new(Hide_Set)
hs.name = name
return hs
}
hide_set_contains :: proc(hs: ^Hide_Set, name: string) -> bool {
for h := hs; h != nil; h = h.next {
if h.name == name {
return true;
return true
}
}
return false;
return false
}
hide_set_union :: proc(a, b: ^Hide_Set) -> ^Hide_Set {
head: Hide_Set;
curr := &head;
head: Hide_Set
curr := &head
for h := a; h != nil; h = h.next {
curr.next = new_hide_set(h.name);
curr = curr.next;
curr.next = new_hide_set(h.name)
curr = curr.next
}
curr.next = b;
return head.next;
curr.next = b
return head.next
}
hide_set_intersection :: proc(a, b: ^Hide_Set) -> ^Hide_Set {
head: Hide_Set;
curr := &head;
head: Hide_Set
curr := &head
for h := a; h != nil; h = h.next {
if hide_set_contains(b, h.name) {
curr.next = new_hide_set(h.name);
curr = curr.next;
curr.next = new_hide_set(h.name)
curr = curr.next
}
}
return head.next;
return head.next
}
add_hide_set :: proc(tok: ^Token, hs: ^Hide_Set) -> ^Token {
head: Token;
curr := &head;
head: Token
curr := &head
tok := tok;
tok := tok
for ; tok != nil; tok = tok.next {
t := copy_token(tok);
t.hide_set = hide_set_union(t.hide_set, hs);
curr.next = t;
curr = curr.next;
t := copy_token(tok)
t.hide_set = hide_set_union(t.hide_set, hs)
curr.next = t
curr = curr.next
}
return head.next;
return head.next
}
+13 -13
View File
@@ -80,29 +80,29 @@ Token :: struct {
origin: ^Token,
}
Is_Keyword_Proc :: #type proc(tok: ^Token) -> bool;
Is_Keyword_Proc :: #type proc(tok: ^Token) -> bool
copy_token :: proc(tok: ^Token) -> ^Token {
t := new_clone(tok^);
t.next = nil;
return t;
t := new_clone(tok^)
t.next = nil
return t
}
new_eof :: proc(tok: ^Token) -> ^Token {
t := new_clone(tok^);
t.kind = .EOF;
t.lit = "";
return t;
t := new_clone(tok^)
t.kind = .EOF
t.lit = ""
return t
}
default_is_keyword :: proc(tok: ^Token) -> bool {
if tok.kind == .Keyword {
return true;
return true
}
if len(tok.lit) > 0 {
return default_keyword_set[tok.lit];
return default_keyword_set[tok.lit]
}
return false;
return false
}
@@ -117,7 +117,7 @@ token_name := [Token_Kind]string {
.PP_Number = "preprocessor number",
.Comment = "comment",
.EOF = "eof",
};
}
default_keyword_set := map[string]bool{
"auto" = true,
@@ -166,4 +166,4 @@ default_keyword_set := map[string]bool{
"__restrict__" = true,
"__thread" = true,
"__attribute__" = true,
};
}
+267 -267
View File
@@ -6,7 +6,7 @@ import "core:strings"
import "core:unicode/utf8"
Error_Handler :: #type proc(pos: Pos, fmt: string, args: ..any);
Error_Handler :: #type proc(pos: Pos, fmt: string, args: ..any)
Tokenizer :: struct {
@@ -34,415 +34,415 @@ Tokenizer :: struct {
}
init_defaults :: proc(t: ^Tokenizer, err: Error_Handler = default_error_handler, warn: Error_Handler = default_warn_handler) {
t.err = err;
t.warn = warn;
t.err = err
t.warn = warn
}
@(private)
offset_to_pos :: proc(t: ^Tokenizer, offset: int) -> (pos: Pos) {
pos.file = t.path;
pos.offset = offset;
pos.line = t.line_count;
pos.column = offset - t.line_offset + 1;
return;
pos.file = t.path
pos.offset = offset
pos.line = t.line_count
pos.column = offset - t.line_offset + 1
return
}
default_error_handler :: proc(pos: Pos, msg: string, args: ..any) {
fmt.eprintf("%s(%d:%d) ", pos.file, pos.line, pos.column);
fmt.eprintf(msg, ..args);
fmt.eprintf("\n");
fmt.eprintf("%s(%d:%d) ", pos.file, pos.line, pos.column)
fmt.eprintf(msg, ..args)
fmt.eprintf("\n")
}
default_warn_handler :: proc(pos: Pos, msg: string, args: ..any) {
fmt.eprintf("%s(%d:%d) warning: ", pos.file, pos.line, pos.column);
fmt.eprintf(msg, ..args);
fmt.eprintf("\n");
fmt.eprintf("%s(%d:%d) warning: ", pos.file, pos.line, pos.column)
fmt.eprintf(msg, ..args)
fmt.eprintf("\n")
}
error_offset :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {
pos := offset_to_pos(t, offset);
pos := offset_to_pos(t, offset)
if t.err != nil {
t.err(pos, msg, ..args);
t.err(pos, msg, ..args)
}
t.error_count += 1;
t.error_count += 1
}
warn_offset :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {
pos := offset_to_pos(t, offset);
pos := offset_to_pos(t, offset)
if t.warn != nil {
t.warn(pos, msg, ..args);
t.warn(pos, msg, ..args)
}
t.warning_count += 1;
t.warning_count += 1
}
error :: proc(t: ^Tokenizer, tok: ^Token, msg: string, args: ..any) {
pos := tok.pos;
pos := tok.pos
if t.err != nil {
t.err(pos, msg, ..args);
t.err(pos, msg, ..args)
}
t.error_count += 1;
t.error_count += 1
}
warn :: proc(t: ^Tokenizer, tok: ^Token, msg: string, args: ..any) {
pos := tok.pos;
pos := tok.pos
if t.warn != nil {
t.warn(pos, msg, ..args);
t.warn(pos, msg, ..args)
}
t.warning_count += 1;
t.warning_count += 1
}
advance_rune :: proc(t: ^Tokenizer) {
if t.read_offset < len(t.src) {
t.offset = t.read_offset;
t.offset = t.read_offset
if t.ch == '\n' {
t.at_bol = true;
t.line_offset = t.offset;
t.line_count += 1;
t.at_bol = true
t.line_offset = t.offset
t.line_count += 1
}
r, w := rune(t.src[t.read_offset]), 1;
r, w := rune(t.src[t.read_offset]), 1
switch {
case r == 0:
error_offset(t, t.offset, "illegal character NUL");
error_offset(t, t.offset, "illegal character NUL")
case r >= utf8.RUNE_SELF:
r, w = utf8.decode_rune(t.src[t.read_offset:]);
r, w = utf8.decode_rune(t.src[t.read_offset:])
if r == utf8.RUNE_ERROR && w == 1 {
error_offset(t, t.offset, "illegal UTF-8 encoding");
error_offset(t, t.offset, "illegal UTF-8 encoding")
} else if r == utf8.RUNE_BOM && t.offset > 0 {
error_offset(t, t.offset, "illegal byte order mark");
error_offset(t, t.offset, "illegal byte order mark")
}
}
t.read_offset += w;
t.ch = r;
t.read_offset += w
t.ch = r
} else {
t.offset = len(t.src);
t.offset = len(t.src)
if t.ch == '\n' {
t.at_bol = true;
t.line_offset = t.offset;
t.line_count += 1;
t.at_bol = true
t.line_offset = t.offset
t.line_count += 1
}
t.ch = -1;
t.ch = -1
}
}
advance_rune_n :: proc(t: ^Tokenizer, n: int) {
for in 0..<n {
advance_rune(t);
advance_rune(t)
}
}
is_digit :: proc(r: rune) -> bool {
return '0' <= r && r <= '9';
return '0' <= r && r <= '9'
}
skip_whitespace :: proc(t: ^Tokenizer) {
for {
switch t.ch {
case ' ', '\t', '\r', '\v', '\f', '\n':
t.has_space = true;
advance_rune(t);
t.has_space = true
advance_rune(t)
case:
return;
return
}
}
}
scan_comment :: proc(t: ^Tokenizer) -> string {
offset := t.offset-1;
next := -1;
offset := t.offset-1
next := -1
general: {
if t.ch == '/'{ // line comments
advance_rune(t);
advance_rune(t)
for t.ch != '\n' && t.ch >= 0 {
advance_rune(t);
advance_rune(t)
}
next = t.offset;
next = t.offset
if t.ch == '\n' {
next += 1;
next += 1
}
break general;
break general
}
/* style comment */
advance_rune(t);
advance_rune(t)
for t.ch >= 0 {
ch := t.ch;
advance_rune(t);
ch := t.ch
advance_rune(t)
if ch == '*' && t.ch == '/' {
advance_rune(t);
next = t.offset;
break general;
advance_rune(t)
next = t.offset
break general
}
}
error_offset(t, offset, "comment not terminated");
error_offset(t, offset, "comment not terminated")
}
lit := t.src[offset : t.offset];
lit := t.src[offset : t.offset]
// NOTE(bill): Strip CR for line comments
for len(lit) > 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' {
lit = lit[:len(lit)-1];
lit = lit[:len(lit)-1]
}
return string(lit);
return string(lit)
}
scan_identifier :: proc(t: ^Tokenizer) -> string {
offset := t.offset;
offset := t.offset
for is_ident1(t.ch) {
advance_rune(t);
advance_rune(t)
}
return string(t.src[offset : t.offset]);
return string(t.src[offset : t.offset])
}
scan_string :: proc(t: ^Tokenizer) -> string {
offset := t.offset-1;
offset := t.offset-1
for {
ch := t.ch;
ch := t.ch
if ch == '\n' || ch < 0 {
error_offset(t, offset, "string literal was not terminated");
break;
error_offset(t, offset, "string literal was not terminated")
break
}
advance_rune(t);
advance_rune(t)
if ch == '"' {
break;
break
}
if ch == '\\' {
scan_escape(t);
scan_escape(t)
}
}
return string(t.src[offset : t.offset]);
return string(t.src[offset : t.offset])
}
digit_val :: proc(r: rune) -> int {
switch r {
case '0'..='9':
return int(r-'0');
return int(r-'0')
case 'A'..='F':
return int(r-'A' + 10);
return int(r-'A' + 10)
case 'a'..='f':
return int(r-'a' + 10);
return int(r-'a' + 10)
}
return 16;
return 16
}
scan_escape :: proc(t: ^Tokenizer) -> bool {
offset := t.offset;
offset := t.offset
esc := t.ch;
n: int;
base, max: u32;
esc := t.ch
n: int
base, max: u32
switch esc {
case 'a', 'b', 'e', 'f', 'n', 't', 'v', 'r', '\\', '\'', '"':
advance_rune(t);
return true;
advance_rune(t)
return true
case '0'..='7':
for digit_val(t.ch) < 8 {
advance_rune(t);
advance_rune(t)
}
return true;
return true
case 'x':
advance_rune(t);
advance_rune(t)
for digit_val(t.ch) < 16 {
advance_rune(t);
advance_rune(t)
}
return true;
return true
case 'u':
advance_rune(t);
n, base, max = 4, 16, utf8.MAX_RUNE;
advance_rune(t)
n, base, max = 4, 16, utf8.MAX_RUNE
case 'U':
advance_rune(t);
n, base, max = 8, 16, utf8.MAX_RUNE;
advance_rune(t)
n, base, max = 8, 16, utf8.MAX_RUNE
case:
if t.ch < 0 {
error_offset(t, offset, "escape sequence was not terminated");
error_offset(t, offset, "escape sequence was not terminated")
} else {
break;
break
}
return false;
return false
}
x: u32;
x: u32
main_loop: for n > 0 {
d := u32(digit_val(t.ch));
d := u32(digit_val(t.ch))
if d >= base {
if t.ch == '"' || t.ch == '\'' {
break main_loop;
break main_loop
}
if t.ch < 0 {
error_offset(t, t.offset, "escape sequence was not terminated");
error_offset(t, t.offset, "escape sequence was not terminated")
} else {
error_offset(t, t.offset, "illegal character '%r' : %d in escape sequence", t.ch, t.ch);
error_offset(t, t.offset, "illegal character '%r' : %d in escape sequence", t.ch, t.ch)
}
return false;
return false
}
x = x*base + d;
advance_rune(t);
n -= 1;
x = x*base + d
advance_rune(t)
n -= 1
}
if x > max || 0xd800 <= x && x <= 0xe000 {
error_offset(t, offset, "escape sequence is an invalid Unicode code point");
return false;
error_offset(t, offset, "escape sequence is an invalid Unicode code point")
return false
}
return true;
return true
}
scan_rune :: proc(t: ^Tokenizer) -> string {
offset := t.offset-1;
valid := true;
n := 0;
offset := t.offset-1
valid := true
n := 0
for {
ch := t.ch;
ch := t.ch
if ch == '\n' || ch < 0 {
if valid {
error_offset(t, offset, "rune literal not terminated");
valid = false;
error_offset(t, offset, "rune literal not terminated")
valid = false
}
break;
break
}
advance_rune(t);
advance_rune(t)
if ch == '\'' {
break;
break
}
n += 1;
n += 1
if ch == '\\' {
if !scan_escape(t) {
valid = false;
valid = false
}
}
}
if valid && n != 1 {
error_offset(t, offset, "illegal rune literal");
error_offset(t, offset, "illegal rune literal")
}
return string(t.src[offset : t.offset]);
return string(t.src[offset : t.offset])
}
scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (Token_Kind, string) {
scan_mantissa :: proc(t: ^Tokenizer, base: int) {
for digit_val(t.ch) < base {
advance_rune(t);
advance_rune(t)
}
}
scan_exponent :: proc(t: ^Tokenizer) {
if t.ch == 'e' || t.ch == 'E' || t.ch == 'p' || t.ch == 'P' {
advance_rune(t);
advance_rune(t)
if t.ch == '-' || t.ch == '+' {
advance_rune(t);
advance_rune(t)
}
if digit_val(t.ch) < 10 {
scan_mantissa(t, 10);
scan_mantissa(t, 10)
} else {
error_offset(t, t.offset, "illegal floating-point exponent");
error_offset(t, t.offset, "illegal floating-point exponent")
}
}
}
scan_fraction :: proc(t: ^Tokenizer) -> (early_exit: bool) {
if t.ch == '.' && peek(t) == '.' {
return true;
return true
}
if t.ch == '.' {
advance_rune(t);
scan_mantissa(t, 10);
advance_rune(t)
scan_mantissa(t, 10)
}
return false;
return false
}
check_end := true;
check_end := true
offset := t.offset;
seen_point := seen_decimal_point;
offset := t.offset
seen_point := seen_decimal_point
if seen_point {
offset -= 1;
scan_mantissa(t, 10);
scan_exponent(t);
offset -= 1
scan_mantissa(t, 10)
scan_exponent(t)
} else {
if t.ch == '0' {
int_base :: proc(t: ^Tokenizer, base: int, msg: string) {
prev := t.offset;
advance_rune(t);
scan_mantissa(t, base);
prev := t.offset
advance_rune(t)
scan_mantissa(t, base)
if t.offset - prev <= 1 {
error_offset(t, t.offset, msg);
error_offset(t, t.offset, msg)
}
}
advance_rune(t);
advance_rune(t)
switch t.ch {
case 'b', 'B':
int_base(t, 2, "illegal binary integer");
int_base(t, 2, "illegal binary integer")
case 'x', 'X':
int_base(t, 16, "illegal hexadecimal integer");
int_base(t, 16, "illegal hexadecimal integer")
case:
seen_point = false;
scan_mantissa(t, 10);
seen_point = false
scan_mantissa(t, 10)
if t.ch == '.' {
seen_point = true;
seen_point = true
if scan_fraction(t) {
check_end = false;
check_end = false
}
}
if check_end {
scan_exponent(t);
check_end = false;
scan_exponent(t)
check_end = false
}
}
}
}
if check_end {
scan_mantissa(t, 10);
scan_mantissa(t, 10)
if !scan_fraction(t) {
scan_exponent(t);
scan_exponent(t)
}
}
return .Number, string(t.src[offset : t.offset]);
return .Number, string(t.src[offset : t.offset])
}
scan_punct :: proc(t: ^Tokenizer, ch: rune) -> (kind: Token_Kind) {
kind = .Punct;
kind = .Punct
switch ch {
case:
kind = .Invalid;
kind = .Invalid
case '<', '>':
if t.ch == ch {
advance_rune(t);
advance_rune(t)
}
if t.ch == '=' {
advance_rune(t);
advance_rune(t)
}
case '!', '+', '-', '*', '/', '%', '^', '=':
if t.ch == '=' {
advance_rune(t);
advance_rune(t)
}
case '#':
if t.ch == '#' {
advance_rune(t);
advance_rune(t)
}
case '&':
if t.ch == '=' || t.ch == '&' {
advance_rune(t);
advance_rune(t)
}
case '|':
if t.ch == '=' || t.ch == '|' {
advance_rune(t);
advance_rune(t)
}
case '(', ')', '[', ']', '{', '}':
// okay
@@ -452,216 +452,216 @@ scan_punct :: proc(t: ^Tokenizer, ch: rune) -> (kind: Token_Kind) {
// okay
case '.':
if t.ch == '.' && peek(t) == '.' {
advance_rune(t);
advance_rune(t); // consume last '.'
advance_rune(t)
advance_rune(t) // consume last '.'
}
}
return;
return
}
peek :: proc(t: ^Tokenizer) -> byte {
if t.read_offset < len(t.src) {
return t.src[t.read_offset];
return t.src[t.read_offset]
}
return 0;
return 0
}
peek_str :: proc(t: ^Tokenizer, str: string) -> bool {
if t.read_offset < len(t.src) {
return strings.has_prefix(string(t.src[t.offset:]), str);
return strings.has_prefix(string(t.src[t.offset:]), str)
}
return false;
return false
}
scan_literal_prefix :: proc(t: ^Tokenizer, str: string, prefix: ^string) -> bool {
if peek_str(t, str) {
offset := t.offset;
offset := t.offset
for _ in str {
advance_rune(t);
advance_rune(t)
}
prefix^ = string(t.src[offset:][:len(str)-1]);
return true;
prefix^ = string(t.src[offset:][:len(str)-1])
return true
}
return false;
return false
}
allow_next_to_be_newline :: proc(t: ^Tokenizer) -> bool {
if t.ch == '\n' {
advance_rune(t);
return true;
advance_rune(t)
return true
} else if t.ch == '\r' && peek(t) == '\n' { // allow for MS-DOS style line endings
advance_rune(t); // \r
advance_rune(t); // \n
return true;
advance_rune(t) // \r
advance_rune(t) // \n
return true
}
return false;
return false
}
scan :: proc(t: ^Tokenizer, f: ^File) -> ^Token {
skip_whitespace(t);
skip_whitespace(t)
offset := t.offset;
offset := t.offset
kind: Token_Kind;
lit: string;
prefix: string;
kind: Token_Kind
lit: string
prefix: string
switch ch := t.ch; {
case scan_literal_prefix(t, `u8"`, &prefix):
kind = .String;
lit = scan_string(t);
kind = .String
lit = scan_string(t)
case scan_literal_prefix(t, `u"`, &prefix):
kind = .String;
lit = scan_string(t);
kind = .String
lit = scan_string(t)
case scan_literal_prefix(t, `L"`, &prefix):
kind = .String;
lit = scan_string(t);
kind = .String
lit = scan_string(t)
case scan_literal_prefix(t, `U"`, &prefix):
kind = .String;
lit = scan_string(t);
kind = .String
lit = scan_string(t)
case scan_literal_prefix(t, `u'`, &prefix):
kind = .Char;
lit = scan_rune(t);
kind = .Char
lit = scan_rune(t)
case scan_literal_prefix(t, `L'`, &prefix):
kind = .Char;
lit = scan_rune(t);
kind = .Char
lit = scan_rune(t)
case scan_literal_prefix(t, `U'`, &prefix):
kind = .Char;
lit = scan_rune(t);
kind = .Char
lit = scan_rune(t)
case is_ident0(ch):
lit = scan_identifier(t);
kind = .Ident;
lit = scan_identifier(t)
kind = .Ident
case '0' <= ch && ch <= '9':
kind, lit = scan_number(t, false);
kind, lit = scan_number(t, false)
case:
advance_rune(t);
advance_rune(t)
switch ch {
case -1:
kind = .EOF;
kind = .EOF
case '\\':
kind = .Punct;
kind = .Punct
if allow_next_to_be_newline(t) {
t.at_bol = true;
t.has_space = false;
return scan(t, f);
t.at_bol = true
t.has_space = false
return scan(t, f)
}
case '.':
if is_digit(t.ch) {
kind, lit = scan_number(t, true);
kind, lit = scan_number(t, true)
} else {
kind = scan_punct(t, ch);
kind = scan_punct(t, ch)
}
case '"':
kind = .String;
lit = scan_string(t);
kind = .String
lit = scan_string(t)
case '\'':
kind = .Char;
lit = scan_rune(t);
kind = .Char
lit = scan_rune(t)
case '/':
if t.ch == '/' || t.ch == '*' {
kind = .Comment;
lit = scan_comment(t);
t.has_space = true;
break;
kind = .Comment
lit = scan_comment(t)
t.has_space = true
break
}
fallthrough;
fallthrough
case:
kind = scan_punct(t, ch);
kind = scan_punct(t, ch)
if kind == .Invalid && ch != utf8.RUNE_BOM {
error_offset(t, t.offset, "illegal character '%r': %d", ch, ch);
error_offset(t, t.offset, "illegal character '%r': %d", ch, ch)
}
}
}
if lit == "" {
lit = string(t.src[offset : t.offset]);
lit = string(t.src[offset : t.offset])
}
if kind == .Comment {
return scan(t, f);
return scan(t, f)
}
tok := new(Token);
tok.kind = kind;
tok.lit = lit;
tok.pos = offset_to_pos(t, offset);
tok.file = f;
tok.prefix = prefix;
tok.at_bol = t.at_bol;
tok.has_space = t.has_space;
tok := new(Token)
tok.kind = kind
tok.lit = lit
tok.pos = offset_to_pos(t, offset)
tok.file = f
tok.prefix = prefix
tok.at_bol = t.at_bol
tok.has_space = t.has_space
t.at_bol, t.has_space = false, false;
t.at_bol, t.has_space = false, false
return tok;
return tok
}
tokenize :: proc(t: ^Tokenizer, f: ^File) -> ^Token {
setup_tokenizer: {
t.src = f.src;
t.ch = ' ';
t.offset = 0;
t.read_offset = 0;
t.line_offset = 0;
t.line_count = len(t.src) > 0 ? 1 : 0;
t.error_count = 0;
t.path = f.name;
t.src = f.src
t.ch = ' '
t.offset = 0
t.read_offset = 0
t.line_offset = 0
t.line_count = len(t.src) > 0 ? 1 : 0
t.error_count = 0
t.path = f.name
advance_rune(t);
advance_rune(t)
if t.ch == utf8.RUNE_BOM {
advance_rune(t);
advance_rune(t)
}
}
t.at_bol = true;
t.has_space = false;
t.at_bol = true
t.has_space = false
head: Token;
curr := &head;
head: Token
curr := &head
for {
tok := scan(t, f);
tok := scan(t, f)
if tok == nil {
break;
break
}
curr.next = tok;
curr = curr.next;
curr.next = tok
curr = curr.next
if tok.kind == .EOF {
break;
break
}
}
return head.next;
return head.next
}
add_new_file :: proc(t: ^Tokenizer, name: string, src: []byte, id: int) -> ^File {
file := new(File);
file.id = id;
file.src = src;
file.name = name;
file.display_name = name;
return file;
file := new(File)
file.id = id
file.src = src
file.name = name
file.display_name = name
return file
}
tokenize_file :: proc(t: ^Tokenizer, path: string, id: int, loc := #caller_location) -> ^Token {
src, ok := os.read_entire_file(path);
src, ok := os.read_entire_file(path)
if !ok {
return nil;
return nil
}
return tokenize(t, add_new_file(t, path, src, id));
return tokenize(t, add_new_file(t, path, src, id))
}
inline_tokenize :: proc(t: ^Tokenizer, tok: ^Token, src: []byte) -> ^Token {
file := new(File);
file.src = src;
file := new(File)
file.src = src
if tok.file != nil {
file.id = tok.file.id;
file.name = tok.file.name;
file.display_name = tok.file.name;
file.id = tok.file.id
file.name = tok.file.name
file.display_name = tok.file.name
}
return tokenize(t, file);
return tokenize(t, file)
}
+13 -13
View File
@@ -4,10 +4,10 @@ package c_frontend_tokenizer
in_range :: proc(range: []rune, c: rune) -> bool #no_bounds_check {
for i := 0; range[i] != -1; i += 2 {
if range[i] <= c && c <= range[i+1] {
return true;
return true
}
}
return false;
return false
}
@@ -15,11 +15,11 @@ in_range :: proc(range: []rune, c: rune) -> bool #no_bounds_check {
//
// is_ident0 returns true if a given character is acceptable as the first character of an identifier.
is_ident0 :: proc(c: rune) -> bool {
return in_range(_range_ident0, c);
return in_range(_range_ident0, c)
}
// is_ident0 returns true if a given character is acceptable as a non-first character of an identifier.
is_ident1 :: proc(c: rune) -> bool {
return is_ident0(c) || in_range(_range_ident1, c);
return is_ident0(c) || in_range(_range_ident1, c)
}
// Returns the number of columns needed to display a given character in a fixed-width font.
@@ -27,18 +27,18 @@ is_ident1 :: proc(c: rune) -> bool {
char_width :: proc(c: rune) -> int {
switch {
case in_range(_range_width0, c):
return 0;
return 0
case in_range(_range_width2, c):
return 2;
return 2
}
return 1;
return 1
}
display_width :: proc(str: string) -> (w: int) {
for c in str {
w += char_width(c);
w += char_width(c)
}
return;
return
}
@@ -59,12 +59,12 @@ _range_ident0 := []rune{
0x90000, 0x9FFFD, 0xA0000, 0xAFFFD, 0xB0000, 0xBFFFD, 0xC0000, 0xCFFFD,
0xD0000, 0xDFFFD, 0xE0000, 0xEFFFD,
-1,
};
}
_range_ident1 := []rune{
'0', '9', '$', '$', 0x0300, 0x036F, 0x1DC0, 0x1DFF, 0x20D0, 0x20FF, 0xFE20, 0xFE2F,
-1,
};
}
_range_width0 := []rune{
@@ -105,7 +105,7 @@ _range_width0 := []rune{
0x1D167, 0x1D169, 0x1D173, 0x1D182, 0x1D185, 0x1D18B, 0x1D1AA, 0x1D1AD,
0x1D242, 0x1D244, 0xE0001, 0xE0001, 0xE0020, 0xE007F, 0xE0100, 0xE01EF,
-1,
};
}
_range_width2 := []rune{
0x1100, 0x115F, 0x2329, 0x2329, 0x232A, 0x232A, 0x2E80, 0x303E,
@@ -113,4 +113,4 @@ _range_width2 := []rune{
0xFE30, 0xFE6F, 0xFF00, 0xFF60, 0xFFE0, 0xFFE6, 0x1F000, 0x1F644,
0x20000, 0x2FFFD, 0x30000, 0x3FFFD,
-1,
};
}