Remove unneeded semicolons from the core library

This commit is contained in:
gingerBill
2021-08-31 22:21:13 +01:00
parent b176af2742
commit 251da264ed
187 changed files with 27227 additions and 27227 deletions
+23 -23
View File
@@ -14,14 +14,14 @@ Loadahead_Reader :: struct {
}
lookahead_reader_init :: proc(lr: ^Loadahead_Reader, r: io.Reader, buf: []byte) -> ^Loadahead_Reader {
lr.r = r;
lr.buf = buf;
lr.n = 0;
return lr;
lr.r = r
lr.buf = buf
lr.n = 0
return lr
}
lookahead_reader_buffer :: proc(lr: ^Loadahead_Reader) -> []byte {
return lr.buf[:lr.n];
return lr.buf[:lr.n]
}
@@ -31,35 +31,35 @@ lookahead_reader_buffer :: proc(lr: ^Loadahead_Reader) -> []byte {
lookahead_reader_peek :: proc(lr: ^Loadahead_Reader, n: int) -> ([]byte, io.Error) {
switch {
case n < 0:
return nil, .Negative_Read;
return nil, .Negative_Read
case n > len(lr.buf):
return nil, .Buffer_Full;
return nil, .Buffer_Full
}
n := n;
err: io.Error;
read_count: int;
n := n
err: io.Error
read_count: int
if lr.n < n {
read_count, err = io.read_at_least(lr.r, lr.buf[lr.n:], n-lr.n);
read_count, err = io.read_at_least(lr.r, lr.buf[lr.n:], n-lr.n)
if err == .Unexpected_EOF {
err = .EOF;
err = .EOF
}
}
lr.n += read_count;
lr.n += read_count
if n > lr.n {
n = lr.n;
n = lr.n
}
return lr.buf[:n], err;
return lr.buf[:n], err
}
// lookahead_reader_peek_all returns a slice of the Lookahead_Reader populating the full buffer
// If the Lookahead_Reader cannot hold enough bytes, it will read from the underlying reader to populate the rest.
// NOTE: The returned buffer is not a copy of the underlying buffer
lookahead_reader_peek_all :: proc(lr: ^Loadahead_Reader) -> ([]byte, io.Error) {
return lookahead_reader_peek(lr, len(lr.buf));
return lookahead_reader_peek(lr, len(lr.buf))
}
@@ -67,17 +67,17 @@ lookahead_reader_peek_all :: proc(lr: ^Loadahead_Reader) -> ([]byte, io.Error) {
lookahead_reader_consume :: proc(lr: ^Loadahead_Reader, n: int) -> io.Error {
switch {
case n == 0:
return nil;
return nil
case n < 0:
return .Negative_Read;
return .Negative_Read
case lr.n < n:
return .Short_Buffer;
return .Short_Buffer
}
copy(lr.buf, lr.buf[n:lr.n]);
lr.n -= n;
return nil;
copy(lr.buf, lr.buf[n:lr.n])
lr.n -= n
return nil
}
lookahead_reader_consume_all :: proc(lr: ^Loadahead_Reader) -> io.Error {
return lookahead_reader_consume(lr, lr.n);
return lookahead_reader_consume(lr, lr.n)
}
+27 -27
View File
@@ -10,59 +10,59 @@ Read_Writer :: struct {
read_writer_init :: proc(rw: ^Read_Writer, r: ^Reader, w: ^Writer) {
rw.r, rw.w = r, w;
rw.r, rw.w = r, w
}
read_writer_to_stream :: proc(rw: ^Read_Writer) -> (s: io.Stream) {
s.stream_data = rw;
s.stream_vtable = _read_writer_vtable;
return;
s.stream_data = rw
s.stream_vtable = _read_writer_vtable
return
}
@(private)
_read_writer_vtable := &io.Stream_VTable{
impl_read = proc(s: io.Stream, p: []byte) -> (n: int, err: io.Error) {
b := (^Read_Writer)(s.stream_data).r;
return reader_read(b, p);
b := (^Read_Writer)(s.stream_data).r
return reader_read(b, p)
},
impl_read_byte = proc(s: io.Stream) -> (c: byte, err: io.Error) {
b := (^Read_Writer)(s.stream_data).r;
return reader_read_byte(b);
b := (^Read_Writer)(s.stream_data).r
return reader_read_byte(b)
},
impl_unread_byte = proc(s: io.Stream) -> io.Error {
b := (^Read_Writer)(s.stream_data).r;
return reader_unread_byte(b);
b := (^Read_Writer)(s.stream_data).r
return reader_unread_byte(b)
},
impl_read_rune = proc(s: io.Stream) -> (r: rune, size: int, err: io.Error) {
b := (^Read_Writer)(s.stream_data).r;
return reader_read_rune(b);
b := (^Read_Writer)(s.stream_data).r
return reader_read_rune(b)
},
impl_unread_rune = proc(s: io.Stream) -> io.Error {
b := (^Read_Writer)(s.stream_data).r;
return reader_unread_rune(b);
b := (^Read_Writer)(s.stream_data).r
return reader_unread_rune(b)
},
impl_write_to = proc(s: io.Stream, w: io.Writer) -> (n: i64, err: io.Error) {
b := (^Read_Writer)(s.stream_data).r;
return reader_write_to(b, w);
b := (^Read_Writer)(s.stream_data).r
return reader_write_to(b, w)
},
impl_flush = proc(s: io.Stream) -> io.Error {
b := (^Read_Writer)(s.stream_data).w;
return writer_flush(b);
b := (^Read_Writer)(s.stream_data).w
return writer_flush(b)
},
impl_write = proc(s: io.Stream, p: []byte) -> (n: int, err: io.Error) {
b := (^Read_Writer)(s.stream_data).w;
return writer_write(b, p);
b := (^Read_Writer)(s.stream_data).w
return writer_write(b, p)
},
impl_write_byte = proc(s: io.Stream, c: byte) -> io.Error {
b := (^Read_Writer)(s.stream_data).w;
return writer_write_byte(b, c);
b := (^Read_Writer)(s.stream_data).w
return writer_write_byte(b, c)
},
impl_write_rune = proc(s: io.Stream, r: rune) -> (int, io.Error) {
b := (^Read_Writer)(s.stream_data).w;
return writer_write_rune(b, r);
b := (^Read_Writer)(s.stream_data).w
return writer_write_rune(b, r)
},
impl_read_from = proc(s: io.Stream, r: io.Reader) -> (n: i64, err: io.Error) {
b := (^Read_Writer)(s.stream_data).w;
return writer_read_from(b, r);
b := (^Read_Writer)(s.stream_data).w
return writer_read_from(b, r)
},
};
}
+177 -177
View File
@@ -22,85 +22,85 @@ Reader :: struct {
}
DEFAULT_BUF_SIZE :: 4096;
DEFAULT_BUF_SIZE :: 4096
@(private)
MIN_READ_BUFFER_SIZE :: 16;
MIN_READ_BUFFER_SIZE :: 16
@(private)
DEFAULT_MAX_CONSECUTIVE_EMPTY_READS :: 128;
DEFAULT_MAX_CONSECUTIVE_EMPTY_READS :: 128
reader_init :: proc(b: ^Reader, rd: io.Reader, size: int = DEFAULT_BUF_SIZE, allocator := context.allocator) {
size := size;
size = max(size, MIN_READ_BUFFER_SIZE);
reader_reset(b, rd);
b.buf_allocator = allocator;
b.buf = make([]byte, size, allocator);
size := size
size = max(size, MIN_READ_BUFFER_SIZE)
reader_reset(b, rd)
b.buf_allocator = allocator
b.buf = make([]byte, size, allocator)
}
reader_init_with_buf :: proc(b: ^Reader, rd: io.Reader, buf: []byte) {
reader_reset(b, rd);
b.buf_allocator = {};
b.buf = buf;
reader_reset(b, rd)
b.buf_allocator = {}
b.buf = buf
}
// reader_destroy destroys the underlying buffer with its associated allocator IFF that allocator has been set
reader_destroy :: proc(b: ^Reader) {
delete(b.buf, b.buf_allocator);
b^ = {};
delete(b.buf, b.buf_allocator)
b^ = {}
}
reader_size :: proc(b: ^Reader) -> int {
return len(b.buf);
return len(b.buf)
}
reader_reset :: proc(b: ^Reader, r: io.Reader) {
b.rd = r;
b.r, b.w = 0, 0;
b.err = nil;
b.last_byte = -1;
b.last_rune_size = -1;
b.rd = r
b.r, b.w = 0, 0
b.err = nil
b.last_byte = -1
b.last_rune_size = -1
}
@(private)
_reader_read_new_chunk :: proc(b: ^Reader) -> io.Error {
if b.r > 0 {
copy(b.buf, b.buf[b.r:b.w]);
b.w -= b.r;
b.r = 0;
copy(b.buf, b.buf[b.r:b.w])
b.w -= b.r
b.r = 0
}
if b.w >= len(b.buf) {
return .Buffer_Full;
return .Buffer_Full
}
if b.max_consecutive_empty_reads <= 0 {
b.max_consecutive_empty_reads = DEFAULT_MAX_CONSECUTIVE_EMPTY_READS;
b.max_consecutive_empty_reads = DEFAULT_MAX_CONSECUTIVE_EMPTY_READS
}
// read new data, and try a limited number of times
for i := b.max_consecutive_empty_reads; i > 0; i -= 1 {
n, err := io.read(b.rd, b.buf[b.w:]);
n, err := io.read(b.rd, b.buf[b.w:])
if n < 0 {
return .Negative_Read;
return .Negative_Read
}
b.w += n;
b.w += n
if err != nil {
b.err = err;
return nil;
b.err = err
return nil
}
if n > 0 {
return nil;
return nil
}
}
b.err = .No_Progress;
return nil;
b.err = .No_Progress
return nil
}
@(private)
_reader_consume_err :: proc(b: ^Reader) -> io.Error {
err := b.err;
b.err = nil;
return err;
err := b.err
b.err = nil
return err
}
// reader_peek returns the next n bytes without advancing the reader
@@ -109,151 +109,151 @@ _reader_consume_err :: proc(b: ^Reader) -> io.Error {
// explaining why the read is short
// The error will be .Buffer_Full if n is larger than the internal buffer size
reader_peek :: proc(b: ^Reader, n: int) -> (data: []byte, err: io.Error) {
n := n;
n := n
if n < 0 {
return nil, .Negative_Count;
return nil, .Negative_Count
}
b.last_byte = -1;
b.last_rune_size = -1;
b.last_byte = -1
b.last_rune_size = -1
for b.w-b.r < n && b.w-b.r < len(b.buf) && b.err == nil {
if fill_err := _reader_read_new_chunk(b); fill_err != nil {
return nil, fill_err;
return nil, fill_err
}
}
if n > len(b.buf) {
return b.buf[b.r : b.w], .Buffer_Full;
return b.buf[b.r : b.w], .Buffer_Full
}
if available := b.w - b.r; available < n {
n = available;
err = _reader_consume_err(b);
n = available
err = _reader_consume_err(b)
if err == nil {
err = .Buffer_Full;
err = .Buffer_Full
}
}
return b.buf[b.r : b.r+n], err;
return b.buf[b.r : b.r+n], err
}
// reader_buffered returns the number of bytes that can be read from the current buffer
reader_buffered :: proc(b: ^Reader) -> int {
return b.w - b.r;
return b.w - b.r
}
// reader_discard skips the next n bytes, and returns the number of bytes that were discarded
reader_discard :: proc(b: ^Reader, n: int) -> (discarded: int, err: io.Error) {
if n < 0 {
return 0, .Negative_Count;
return 0, .Negative_Count
}
if n == 0 {
return;
return
}
remaining := n;
remaining := n
for {
skip := reader_buffered(b);
skip := reader_buffered(b)
if skip == 0 {
if fill_err := _reader_read_new_chunk(b); fill_err != nil {
return 0, fill_err;
return 0, fill_err
}
skip = reader_buffered(b);
skip = reader_buffered(b)
}
skip = min(skip, remaining);
b.r += skip;
remaining -= skip;
skip = min(skip, remaining)
b.r += skip
remaining -= skip
if remaining == 0 {
return n, nil;
return n, nil
}
if b.err != nil {
return n - remaining, _reader_consume_err(b);
return n - remaining, _reader_consume_err(b)
}
}
return;
return
}
// reader_read reads data into p
// The bytes are taken from at most one read on the underlying Reader, which means n may be less than len(p)
reader_read :: proc(b: ^Reader, p: []byte) -> (n: int, err: io.Error) {
n = len(p);
n = len(p)
if n == 0 {
if reader_buffered(b) > 0 {
return 0, nil;
return 0, nil
}
return 0, _reader_consume_err(b);
return 0, _reader_consume_err(b)
}
if b.r == b.w {
if b.err != nil {
return 0, _reader_consume_err(b);
return 0, _reader_consume_err(b)
}
if len(p) >= len(b.buf) {
n, b.err = io.read(b.rd, p);
n, b.err = io.read(b.rd, p)
if n < 0 {
return 0, .Negative_Read;
return 0, .Negative_Read
}
if n > 0 {
b.last_byte = int(p[n-1]);
b.last_rune_size = -1;
b.last_byte = int(p[n-1])
b.last_rune_size = -1
}
return n, _reader_consume_err(b);
return n, _reader_consume_err(b)
}
b.r, b.w = 0, 0;
n, b.err = io.read(b.rd, b.buf);
b.r, b.w = 0, 0
n, b.err = io.read(b.rd, b.buf)
if n < 0 {
return 0, .Negative_Read;
return 0, .Negative_Read
}
if n == 0 {
return 0, _reader_consume_err(b);
return 0, _reader_consume_err(b)
}
b.w += n;
b.w += n
}
n = copy(p, b.buf[b.r:b.w]);
b.r += n;
b.last_byte = int(b.buf[b.r-1]);
b.last_rune_size = -1;
return n, nil;
n = copy(p, b.buf[b.r:b.w])
b.r += n
b.last_byte = int(b.buf[b.r-1])
b.last_rune_size = -1
return n, nil
}
// reader_read_byte reads and returns a single byte
// If no byte is available, it return an error
reader_read_byte :: proc(b: ^Reader) -> (byte, io.Error) {
b.last_rune_size = -1;
b.last_rune_size = -1
for b.r == b.w {
if b.err != nil {
return 0, _reader_consume_err(b);
return 0, _reader_consume_err(b)
}
if err := _reader_read_new_chunk(b); err != nil {
return 0, err;
return 0, err
}
}
c := b.buf[b.r];
b.r += 1;
b.last_byte = int(c);
return c, nil;
c := b.buf[b.r]
b.r += 1
b.last_byte = int(c)
return c, nil
}
// reader_unread_byte unreads the last byte. Only the most recently read byte can be unread
reader_unread_byte :: proc(b: ^Reader) -> io.Error {
if b.last_byte < 0 || b.r == 0 && b.w > 0 {
return .Invalid_Unread;
return .Invalid_Unread
}
if b.r > 0 {
b.r -= 1;
b.r -= 1
} else {
// b.r == 0 && b.w == 0
b.w = 1;
b.w = 1
}
b.buf[b.r] = byte(b.last_byte);
b.last_byte = -1;
b.last_rune_size = -1;
return nil;
b.buf[b.r] = byte(b.last_byte)
b.last_byte = -1
b.last_rune_size = -1
return nil
}
// reader_read_rune reads a single UTF-8 encoded unicode character
@@ -265,96 +265,96 @@ reader_read_rune :: proc(b: ^Reader) -> (r: rune, size: int, err: io.Error) {
b.err == nil &&
b.w-b.w < len(b.buf) {
if err = _reader_read_new_chunk(b); err != nil {
return;
return
}
}
b.last_rune_size = -1;
b.last_rune_size = -1
if b.r == b.w {
err = _reader_consume_err(b);
return;
err = _reader_consume_err(b)
return
}
r, size = rune(b.buf[b.r]), 1;
r, size = rune(b.buf[b.r]), 1
if r >= utf8.RUNE_SELF {
r, size = utf8.decode_rune(b.buf[b.r : b.w]);
r, size = utf8.decode_rune(b.buf[b.r : b.w])
}
b.r += size;
b.last_byte = int(b.buf[b.r-1]);
b.last_rune_size = size;
return;
b.r += size
b.last_byte = int(b.buf[b.r-1])
b.last_rune_size = size
return
}
// reader_unread_rune unreads the last rune. Only the most recently read rune can be unread
reader_unread_rune :: proc(b: ^Reader) -> io.Error {
if b.last_rune_size < 0 || b.r < b.last_rune_size {
return .Invalid_Unread;
return .Invalid_Unread
}
b.r -= b.last_rune_size;
b.last_byte = -1;
b.last_rune_size = -1;
return nil;
b.r -= b.last_rune_size
b.last_byte = -1
b.last_rune_size = -1
return nil
}
reader_write_to :: proc(b: ^Reader, w: io.Writer) -> (n: i64, err: io.Error) {
write_buf :: proc(b: ^Reader, w: io.Writer) -> (i64, io.Error) {
n, err := io.write(w, b.buf[b.r:b.w]);
n, err := io.write(w, b.buf[b.r:b.w])
if n < 0 {
return 0, .Negative_Write;
return 0, .Negative_Write
}
b.r += n;
return i64(n), err;
b.r += n
return i64(n), err
}
n, err = write_buf(b, w);
n, err = write_buf(b, w)
if err != nil {
return;
return
}
m: i64;
m: i64
if nr, ok := io.to_writer_to(b.rd); ok {
m, err = io.write_to(nr, w);
n += m;
return n, err;
m, err = io.write_to(nr, w)
n += m
return n, err
}
if nw, ok := io.to_reader_from(w); ok {
m, err = io.read_from(nw, b.rd);
n += m;
return n, err;
m, err = io.read_from(nw, b.rd)
n += m
return n, err
}
if b.w-b.r < len(b.buf) {
if err = _reader_read_new_chunk(b); err != nil {
return;
return
}
}
for b.r < b.w {
m, err = write_buf(b, w);
n += m;
m, err = write_buf(b, w)
n += m
if err != nil {
return;
return
}
if err = _reader_read_new_chunk(b); err != nil {
return;
return
}
}
if b.err == .EOF {
b.err = nil;
b.err = nil
}
err = _reader_consume_err(b);
return;
err = _reader_consume_err(b)
return
}
// reader_to_stream converts a Reader into an io.Stream
reader_to_stream :: proc(b: ^Reader) -> (s: io.Stream) {
s.stream_data = b;
s.stream_vtable = _reader_vtable;
return;
s.stream_data = b
s.stream_vtable = _reader_vtable
return
}
@@ -362,35 +362,35 @@ reader_to_stream :: proc(b: ^Reader) -> (s: io.Stream) {
@(private)
_reader_vtable := &io.Stream_VTable{
impl_destroy = proc(s: io.Stream) -> io.Error {
b := (^Reader)(s.stream_data);
reader_destroy(b);
return nil;
b := (^Reader)(s.stream_data)
reader_destroy(b)
return nil
},
impl_read = proc(s: io.Stream, p: []byte) -> (n: int, err: io.Error) {
b := (^Reader)(s.stream_data);
return reader_read(b, p);
b := (^Reader)(s.stream_data)
return reader_read(b, p)
},
impl_read_byte = proc(s: io.Stream) -> (c: byte, err: io.Error) {
b := (^Reader)(s.stream_data);
return reader_read_byte(b);
b := (^Reader)(s.stream_data)
return reader_read_byte(b)
},
impl_unread_byte = proc(s: io.Stream) -> io.Error {
b := (^Reader)(s.stream_data);
return reader_unread_byte(b);
b := (^Reader)(s.stream_data)
return reader_unread_byte(b)
},
impl_read_rune = proc(s: io.Stream) -> (r: rune, size: int, err: io.Error) {
b := (^Reader)(s.stream_data);
return reader_read_rune(b);
b := (^Reader)(s.stream_data)
return reader_read_rune(b)
},
impl_unread_rune = proc(s: io.Stream) -> io.Error {
b := (^Reader)(s.stream_data);
return reader_unread_rune(b);
b := (^Reader)(s.stream_data)
return reader_unread_rune(b)
},
impl_write_to = proc(s: io.Stream, w: io.Writer) -> (n: i64, err: io.Error) {
b := (^Reader)(s.stream_data);
return reader_write_to(b, w);
b := (^Reader)(s.stream_data)
return reader_write_to(b, w)
},
};
}
@@ -410,71 +410,71 @@ _reader_vtable := &io.Stream_VTable{
// reader_read_slice returns err != nil if and only if line does not end in delim
//
reader_read_slice :: proc(b: ^Reader, delim: byte) -> (line: []byte, err: io.Error) {
s := 0;
s := 0
for {
if i := bytes.index_byte(b.buf[b.r+s : b.w], delim); i >= 0 {
i += s;
line = b.buf[b.r:][:i+1];
b.r += i + 1;
break;
i += s
line = b.buf[b.r:][:i+1]
b.r += i + 1
break
}
if b.err != nil {
line = b.buf[b.r : b.w];
b.r = b.w;
err = _reader_consume_err(b);
break;
line = b.buf[b.r : b.w]
b.r = b.w
err = _reader_consume_err(b)
break
}
if reader_buffered(b) >= len(b.buf) {
b.r = b.w;
line = b.buf;
err = .Buffer_Full;
break;
b.r = b.w
line = b.buf
err = .Buffer_Full
break
}
s = b.w - b.r;
s = b.w - b.r
if err = _reader_read_new_chunk(b); err != nil {
break;
break
}
}
if i := len(line)-1; i >= 0 {
b.last_byte = int(line[i]);
b.last_rune_size = -1;
b.last_byte = int(line[i])
b.last_rune_size = -1
}
return;
return
}
// reader_read_bytes reads until the first occurrence of delim from the Reader
// It returns an allocated slice containing the data up to and including the delimiter
reader_read_bytes :: proc(b: ^Reader, delim: byte, allocator := context.allocator) -> (buf: []byte, err: io.Error) {
full: [dynamic]byte;
full.allocator = allocator;
full: [dynamic]byte
full.allocator = allocator
frag: []byte;
frag: []byte
for {
e: io.Error;
frag, e = reader_read_slice(b, delim);
e: io.Error
frag, e = reader_read_slice(b, delim)
if e == nil {
break;
break
}
if e != .Buffer_Full {
err = e;
break;
err = e
break
}
append(&full, ..frag);
append(&full, ..frag)
}
append(&full, ..frag);
return full[:], err;
append(&full, ..frag)
return full[:], err
}
// reader_read_string reads until the first occurrence of delim from the Reader
// It returns an allocated string containing the data up to and including the delimiter
reader_read_string :: proc(b: ^Reader, delim: byte, allocator := context.allocator) -> (string, io.Error) {
buf, err := reader_read_bytes(b, delim, allocator);
return string(buf), err;
buf, err := reader_read_bytes(b, delim, allocator)
return string(buf), err
}
+119 -119
View File
@@ -21,7 +21,7 @@ Scanner_Error :: union {
}
// Split_Proc is the signature of the split procedure used to tokenize the input.
Split_Proc :: proc(data: []byte, at_eof: bool) -> (advance: int, token: []byte, err: Scanner_Error, final_token: bool);
Split_Proc :: proc(data: []byte, at_eof: bool) -> (advance: int, token: []byte, err: Scanner_Error, final_token: bool)
Scanner :: struct {
r: io.Reader,
@@ -40,28 +40,28 @@ Scanner :: struct {
done: bool,
}
DEFAULT_MAX_SCAN_TOKEN_SIZE :: 1<<16;
DEFAULT_MAX_SCAN_TOKEN_SIZE :: 1<<16
@(private)
_INIT_BUF_SIZE :: 4096;
_INIT_BUF_SIZE :: 4096
scanner_init :: proc(s: ^Scanner, r: io.Reader, buf_allocator := context.allocator) -> ^Scanner {
s.r = r;
s.split = scan_lines;
s.max_token_size = DEFAULT_MAX_SCAN_TOKEN_SIZE;
s.buf.allocator = buf_allocator;
return s;
s.r = r
s.split = scan_lines
s.max_token_size = DEFAULT_MAX_SCAN_TOKEN_SIZE
s.buf.allocator = buf_allocator
return s
}
scanner_init_with_buffer :: proc(s: ^Scanner, r: io.Reader, buf: []byte) -> ^Scanner {
s.r = r;
s.split = scan_lines;
s.max_token_size = DEFAULT_MAX_SCAN_TOKEN_SIZE;
s.buf = mem.buffer_from_slice(buf);
resize(&s.buf, cap(s.buf));
return s;
s.r = r
s.split = scan_lines
s.max_token_size = DEFAULT_MAX_SCAN_TOKEN_SIZE
s.buf = mem.buffer_from_slice(buf)
resize(&s.buf, cap(s.buf))
return s
}
scanner_destroy :: proc(s: ^Scanner) {
delete(s.buf);
delete(s.buf)
}
@@ -69,9 +69,9 @@ scanner_destroy :: proc(s: ^Scanner) {
scanner_error :: proc(s: ^Scanner) -> Scanner_Error {
switch s._err {
case .EOF, .None:
return nil;
return nil
}
return s._err;
return s._err
}
// Returns the most recent token created by scanner_scan.
@@ -79,7 +79,7 @@ scanner_error :: proc(s: ^Scanner) -> Scanner_Error {
// by another call to scanner_scan.
// Treat the returned value as if it is immutable.
scanner_bytes :: proc(s: ^Scanner) -> []byte {
return s.token;
return s.token
}
// Returns the most recent token created by scanner_scan.
@@ -87,146 +87,146 @@ scanner_bytes :: proc(s: ^Scanner) -> []byte {
// by another call to scanner_scan.
// Treat the returned value as if it is immutable.
scanner_text :: proc(s: ^Scanner) -> string {
return string(s.token);
return string(s.token)
}
// scanner_scan advances the scanner
scanner_scan :: proc(s: ^Scanner) -> bool {
set_err :: proc(s: ^Scanner, err: Scanner_Error) {
err := err;
err := err
if err == .None {
err = nil;
err = nil
}
switch s._err {
case nil, .EOF:
s._err = err;
s._err = err
}
}
if s.done {
return false;
return false
}
s.scan_called = true;
s.scan_called = true
for {
// Check if a token is possible with what is available
// Allow the split procedure to recover if it fails
if s.start < s.end || s._err != nil {
advance, token, err, final_token := s.split(s.buf[s.start:s.end], s._err != nil);
advance, token, err, final_token := s.split(s.buf[s.start:s.end], s._err != nil)
if final_token {
s.token = token;
s.done = true;
return true;
s.token = token
s.done = true
return true
}
if err != nil {
set_err(s, err);
return false;
set_err(s, err)
return false
}
// Do advance
if advance < 0 {
set_err(s, .Negative_Advance);
return false;
set_err(s, .Negative_Advance)
return false
}
if advance > s.end-s.start {
set_err(s, .Advanced_Too_Far);
return false;
set_err(s, .Advanced_Too_Far)
return false
}
s.start += advance;
s.start += advance
s.token = token;
s.token = token
if s.token != nil {
if s._err == nil || advance > 0 {
s.successive_empty_token_count = 0;
s.successive_empty_token_count = 0
} else {
s.successive_empty_token_count += 1;
s.successive_empty_token_count += 1
if s.max_consecutive_empty_reads <= 0 {
s.max_consecutive_empty_reads = DEFAULT_MAX_CONSECUTIVE_EMPTY_READS;
s.max_consecutive_empty_reads = DEFAULT_MAX_CONSECUTIVE_EMPTY_READS
}
if s.successive_empty_token_count > s.max_consecutive_empty_reads {
set_err(s, .No_Progress);
return false;
set_err(s, .No_Progress)
return false
}
}
return true;
return true
}
}
// If an error is hit, no token can be created
if s._err != nil {
s.start = 0;
s.end = 0;
return false;
s.start = 0
s.end = 0
return false
}
// More data must be required to be read
if s.start > 0 && (s.end == len(s.buf) || s.start > len(s.buf)/2) {
copy(s.buf[:], s.buf[s.start:s.end]);
s.end -= s.start;
s.start = 0;
copy(s.buf[:], s.buf[s.start:s.end])
s.end -= s.start
s.start = 0
}
could_be_too_short := false;
could_be_too_short := false
// Resize the buffer if full
if s.end == len(s.buf) {
if s.max_token_size <= 0 {
s.max_token_size = DEFAULT_MAX_SCAN_TOKEN_SIZE;
s.max_token_size = DEFAULT_MAX_SCAN_TOKEN_SIZE
}
if len(s.buf) >= s.max_token_size {
set_err(s, .Too_Long);
return false;
set_err(s, .Too_Long)
return false
}
// overflow check
new_size := _INIT_BUF_SIZE;
new_size := _INIT_BUF_SIZE
if len(s.buf) > 0 {
overflowed: bool;
overflowed: bool
if new_size, overflowed = intrinsics.overflow_mul(len(s.buf), 2); overflowed {
set_err(s, .Too_Long);
return false;
set_err(s, .Too_Long)
return false
}
}
old_size := len(s.buf);
new_size = min(new_size, s.max_token_size);
resize(&s.buf, new_size);
s.end -= s.start;
s.start = 0;
old_size := len(s.buf)
new_size = min(new_size, s.max_token_size)
resize(&s.buf, new_size)
s.end -= s.start
s.start = 0
could_be_too_short = old_size >= len(s.buf);
could_be_too_short = old_size >= len(s.buf)
}
// Read data into the buffer
loop := 0;
loop := 0
for {
n, err := io.read(s.r, s.buf[s.end:len(s.buf)]);
n, err := io.read(s.r, s.buf[s.end:len(s.buf)])
if n < 0 || len(s.buf)-s.end < n {
set_err(s, .Bad_Read_Count);
break;
set_err(s, .Bad_Read_Count)
break
}
s.end += n;
s.end += n
if err != nil {
set_err(s, err);
break;
set_err(s, err)
break
}
if n > 0 {
s.successive_empty_token_count = 0;
break;
s.successive_empty_token_count = 0
break
}
loop += 1;
loop += 1
if s.max_consecutive_empty_reads <= 0 {
s.max_consecutive_empty_reads = DEFAULT_MAX_CONSECUTIVE_EMPTY_READS;
s.max_consecutive_empty_reads = DEFAULT_MAX_CONSECUTIVE_EMPTY_READS
}
if loop > s.max_consecutive_empty_reads {
if could_be_too_short {
set_err(s, .Too_Short);
set_err(s, .Too_Short)
} else {
set_err(s, .No_Progress);
set_err(s, .No_Progress)
}
break;
break
}
}
}
@@ -234,38 +234,38 @@ scanner_scan :: proc(s: ^Scanner) -> bool {
scan_bytes :: proc(data: []byte, at_eof: bool) -> (advance: int, token: []byte, err: Scanner_Error, final_token: bool) {
if at_eof && len(data) == 0 {
return;
return
}
return 1, data[0:1], nil, false;
return 1, data[0:1], nil, false
}
scan_runes :: proc(data: []byte, at_eof: bool) -> (advance: int, token: []byte, err: Scanner_Error, final_token: bool) {
if at_eof && len(data) == 0 {
return;
return
}
if data[0] < utf8.RUNE_SELF {
advance = 1;
token = data[0:1];
return;
advance = 1
token = data[0:1]
return
}
_, width := utf8.decode_rune(data);
_, width := utf8.decode_rune(data)
if width > 1 {
advance = width;
token = data[0:width];
return;
advance = width
token = data[0:width]
return
}
if !at_eof && !utf8.full_rune(data) {
return;
return
}
@thread_local ERROR_RUNE := []byte{0xef, 0xbf, 0xbd};
@thread_local ERROR_RUNE := []byte{0xef, 0xbf, 0xbd}
advance = 1;
token = ERROR_RUNE;
return;
advance = 1
token = ERROR_RUNE
return
}
scan_words :: proc(data: []byte, at_eof: bool) -> (advance: int, token: []byte, err: Scanner_Error, final_token: bool) {
@@ -273,68 +273,68 @@ scan_words :: proc(data: []byte, at_eof: bool) -> (advance: int, token: []byte,
switch r {
// lower ones
case ' ', '\t', '\n', '\v', '\f', '\r':
return true;
return true
case '\u0085', '\u00a0':
return true;
return true
// higher ones
case '\u2000' ..= '\u200a':
return true;
return true
case '\u1680', '\u2028', '\u2029', '\u202f', '\u205f', '\u3000':
return true;
return true
}
return false;
return false
}
// skip spaces at the beginning
start := 0;
start := 0
for width := 0; start < len(data); start += width {
r: rune;
r, width = utf8.decode_rune(data[start:]);
r: rune
r, width = utf8.decode_rune(data[start:])
if !is_space(r) {
break;
break
}
}
for width, i := 0, start; i < len(data); i += width {
r: rune;
r, width = utf8.decode_rune(data[i:]);
r: rune
r, width = utf8.decode_rune(data[i:])
if is_space(r) {
advance = i+width;
token = data[start:i];
return;
advance = i+width
token = data[start:i]
return
}
}
if at_eof && len(data) > start {
advance = len(data);
token = data[start:];
return;
advance = len(data)
token = data[start:]
return
}
advance = start;
return;
advance = start
return
}
scan_lines :: proc(data: []byte, at_eof: bool) -> (advance: int, token: []byte, err: Scanner_Error, final_token: bool) {
trim_carriage_return :: proc "contextless" (data: []byte) -> []byte {
if len(data) > 0 && data[len(data)-1] == '\r' {
return data[0:len(data)-1];
return data[0:len(data)-1]
}
return data;
return data
}
if at_eof && len(data) == 0 {
return;
return
}
if i := bytes.index_byte(data, '\n'); i >= 0 {
advance = i+1;
token = trim_carriage_return(data[0:i]);
return;
advance = i+1
token = trim_carriage_return(data[0:i])
return
}
if at_eof {
advance = len(data);
token = trim_carriage_return(data);
advance = len(data)
token = trim_carriage_return(data)
}
return;
return
}
+96 -96
View File
@@ -20,150 +20,150 @@ Writer :: struct {
}
writer_init :: proc(b: ^Writer, wr: io.Writer, size: int = DEFAULT_BUF_SIZE, allocator := context.allocator) {
size := size;
size = max(size, MIN_READ_BUFFER_SIZE);
writer_reset(b, wr);
b.buf_allocator = allocator;
b.buf = make([]byte, size, allocator);
size := size
size = max(size, MIN_READ_BUFFER_SIZE)
writer_reset(b, wr)
b.buf_allocator = allocator
b.buf = make([]byte, size, allocator)
}
writer_init_with_buf :: proc(b: ^Writer, wr: io.Writer, buf: []byte) {
writer_reset(b, wr);
b.buf_allocator = {};
b.buf = buf;
writer_reset(b, wr)
b.buf_allocator = {}
b.buf = buf
}
// writer_destroy destroys the underlying buffer with its associated allocator IFF that allocator has been set
writer_destroy :: proc(b: ^Writer) {
delete(b.buf, b.buf_allocator);
b^ = {};
delete(b.buf, b.buf_allocator)
b^ = {}
}
// writer_size returns the size of underlying buffer in bytes
writer_size :: proc(b: ^Writer) -> int {
return len(b.buf);
return len(b.buf)
}
writer_reset :: proc(b: ^Writer, w: io.Writer) {
b.wr = w;
b.n = 0;
b.err = nil;
b.wr = w
b.n = 0
b.err = nil
}
// writer_flush writes any buffered data into the underlying io.Writer
writer_flush :: proc(b: ^Writer) -> io.Error {
if b.err != nil {
return b.err;
return b.err
}
if b.n == 0 {
return nil;
return nil
}
n, err := io.write(b.wr, b.buf[0:b.n]);
n, err := io.write(b.wr, b.buf[0:b.n])
if n < b.n && err == nil {
err = .Short_Write;
err = .Short_Write
}
if err != nil {
if n > 0 && n < b.n {
copy(b.buf[:b.n-n], b.buf[n : b.n]);
copy(b.buf[:b.n-n], b.buf[n : b.n])
}
b.n -= n;
b.err = err;
return err;
b.n -= n
b.err = err
return err
}
b.n = 0;
return nil;
b.n = 0
return nil
}
// writer_available returns how many bytes are unused in the buffer
writer_available :: proc(b: ^Writer) -> int {
return len(b.buf) - b.n;
return len(b.buf) - b.n
}
// writer_buffered returns the number of bytes that have been writted into the current buffer
writer_buffered :: proc(b: ^Writer) -> int {
return b.n;
return b.n
}
// writer_write writes the contents of p into the buffer
// It returns the number of bytes written
// If n < len(p), it will return an error explaining why the write is short
writer_write :: proc(b: ^Writer, p: []byte) -> (n: int, err: io.Error) {
p := p;
p := p
for len(p) > writer_available(b) && b.err == nil {
m: int;
m: int
if writer_buffered(b) == 0 {
m, b.err = io.write(b.wr, p);
m, b.err = io.write(b.wr, p)
} else {
m = copy(b.buf[b.n:], p);
b.n += m;
writer_flush(b);
m = copy(b.buf[b.n:], p)
b.n += m
writer_flush(b)
}
n += m;
p = p[m:];
n += m
p = p[m:]
}
if b.err != nil {
return n, b.err;
return n, b.err
}
m := copy(b.buf[b.n:], p);
b.n += m;
m += n;
return m, nil;
m := copy(b.buf[b.n:], p)
b.n += m
m += n
return m, nil
}
// writer_write_byte writes a single byte
writer_write_byte :: proc(b: ^Writer, c: byte) -> io.Error {
if b.err != nil {
return b.err;
return b.err
}
if writer_available(b) <= 0 && writer_flush(b) != nil {
return b.err;
return b.err
}
b.buf[b.n] = c;
b.n += 1;
return nil;
b.buf[b.n] = c
b.n += 1
return nil
}
// writer_write_rune writes a single unicode code point, and returns the number of bytes written with any error
writer_write_rune :: proc(b: ^Writer, r: rune) -> (size: int, err: io.Error) {
if r < utf8.RUNE_SELF {
err = writer_write_byte(b, byte(r));
size = 0 if err != nil else 1;
return;
err = writer_write_byte(b, byte(r))
size = 0 if err != nil else 1
return
}
if b.err != nil {
return 0, b.err;
return 0, b.err
}
buf: [4]u8;
buf: [4]u8
n := writer_available(b);
n := writer_available(b)
if n < utf8.UTF_MAX {
writer_flush(b);
writer_flush(b)
if b.err != nil {
return 0, b.err;
return 0, b.err
}
n = writer_available(b);
n = writer_available(b)
if n < utf8.UTF_MAX {
// this only happens if the buffer is very small
w: int;
buf, w = utf8.encode_rune(r);
return writer_write(b, buf[:w]);
w: int
buf, w = utf8.encode_rune(r)
return writer_write(b, buf[:w])
}
}
buf, size = utf8.encode_rune(r);
copy(b.buf[b.n:], buf[:size]);
b.n += size;
return;
buf, size = utf8.encode_rune(r)
copy(b.buf[b.n:], buf[:size])
b.n += size
return
}
// writer_write writes a string into the buffer
// It returns the number of bytes written
// If n < len(p), it will return an error explaining why the write is short
writer_write_string :: proc(b: ^Writer, s: string) -> (int, io.Error) {
return writer_write(b, transmute([]byte)s);
return writer_write(b, transmute([]byte)s)
}
// writer_read_from is to support io.Reader_From types
@@ -171,60 +171,60 @@ writer_write_string :: proc(b: ^Writer, s: string) -> (int, io.Error) {
// this procedure calls the underlying read_from implementation without buffering
writer_read_from :: proc(b: ^Writer, r: io.Reader) -> (n: i64, err: io.Error) {
if b.err != nil {
return 0, b.err;
return 0, b.err
}
if writer_buffered(b) == 0 {
if w, ok := io.to_reader_from(b.wr); !ok {
n, err = io.read_from(w, r);
b.err = err;
return;
n, err = io.read_from(w, r)
b.err = err
return
}
}
for {
if writer_available(b) == 0 {
writer_flush(b) or_return;
writer_flush(b) or_return
}
if b.max_consecutive_empty_writes <= 0 {
b.max_consecutive_empty_writes = DEFAULT_MAX_CONSECUTIVE_EMPTY_READS;
b.max_consecutive_empty_writes = DEFAULT_MAX_CONSECUTIVE_EMPTY_READS
}
m: int;
nr := 0;
m: int
nr := 0
for nr < b.max_consecutive_empty_writes {
m, err = io.read(r, b.buf[b.n:]);
m, err = io.read(r, b.buf[b.n:])
if m != 0 || err != nil {
break;
break
}
nr += 1;
nr += 1
}
if nr == b.max_consecutive_empty_writes {
return n, .No_Progress;
return n, .No_Progress
}
b.n += m;
n += i64(m);
b.n += m
n += i64(m)
if err != nil {
break;
break
}
}
if err == .EOF {
if writer_available(b) == 0 {
err = writer_flush(b);
err = writer_flush(b)
} else {
err = nil;
err = nil
}
}
return;
return
}
// writer_to_stream converts a Writer into an io.Stream
writer_to_stream :: proc(b: ^Writer) -> (s: io.Stream) {
s.stream_data = b;
s.stream_vtable = _writer_vtable;
return;
s.stream_data = b
s.stream_vtable = _writer_vtable
return
}
@@ -232,28 +232,28 @@ writer_to_stream :: proc(b: ^Writer) -> (s: io.Stream) {
@(private)
_writer_vtable := &io.Stream_VTable{
impl_destroy = proc(s: io.Stream) -> io.Error {
b := (^Writer)(s.stream_data);
writer_destroy(b);
return nil;
b := (^Writer)(s.stream_data)
writer_destroy(b)
return nil
},
impl_flush = proc(s: io.Stream) -> io.Error {
b := (^Writer)(s.stream_data);
return writer_flush(b);
b := (^Writer)(s.stream_data)
return writer_flush(b)
},
impl_write = proc(s: io.Stream, p: []byte) -> (n: int, err: io.Error) {
b := (^Writer)(s.stream_data);
return writer_write(b, p);
b := (^Writer)(s.stream_data)
return writer_write(b, p)
},
impl_write_byte = proc(s: io.Stream, c: byte) -> io.Error {
b := (^Writer)(s.stream_data);
return writer_write_byte(b, c);
b := (^Writer)(s.stream_data)
return writer_write_byte(b, c)
},
impl_write_rune = proc(s: io.Stream, r: rune) -> (int, io.Error) {
b := (^Writer)(s.stream_data);
return writer_write_rune(b, r);
b := (^Writer)(s.stream_data)
return writer_write_rune(b, r)
},
impl_read_from = proc(s: io.Stream, r: io.Reader) -> (n: i64, err: io.Error) {
b := (^Writer)(s.stream_data);
return writer_read_from(b, r);
b := (^Writer)(s.stream_data)
return writer_read_from(b, r)
},
};
}