ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-08-03 22:28:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove core:c/frontend

Browse Source
This commit is contained in:
gingerBill
2024-12-01 11:54:54 +11:00
committed by flysand7
parent 4d9a9ec3f5
commit d85de2e54e
8 changed files with 0 additions and 2740 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/c/frontend/preprocessor/const_expr.odin
-25
View File
@@ -1,25 +0,0 @@
package c_frontend_preprocess
import "core:c/frontend/tokenizer"
const_expr :: proc(rest: ^^Token, tok: ^Token) -> i64 {
// TODO(bill): Handle const_expr correctly
// This is effectively a mini-parser
assert(rest != nil)
assert(tok != nil)
rest^ = tokenizer.new_eof(tok)
switch v in tok.val {
case i64:
return v
case f64:
return i64(v)
case string:
return 0
case []u16:
// TODO
case []u32:
// TODO
}
return 0
}
core/c/frontend/preprocessor/preprocess.odin
-1510
View File
File diff suppressed because it is too large Load Diff
core/c/frontend/preprocessor/unquote.odin
-154
View File
@@ -1,154 +0,0 @@
package c_frontend_preprocess
import "core:unicode/utf8"
unquote_char :: proc(str: string, quote: byte) -> (r: rune, multiple_bytes: bool, tail_string: string, success: bool) {
hex_to_int :: proc(c: byte) -> int {
switch c {
case '0'..='9': return int(c-'0')
case 'a'..='f': return int(c-'a')+10
case 'A'..='F': return int(c-'A')+10
}
return -1
}
w: int
if str[0] == quote && quote == '"' {
return
} else if str[0] >= 0x80 {
r, w = utf8.decode_rune_in_string(str)
return r, true, str[w:], true
} else if str[0] != '\\' {
return rune(str[0]), false, str[1:], true
}
if len(str) <= 1 {
return
}
s := str
c := s[1]
s = s[2:]
switch c {
case: r = rune(c)
case 'a': r = '\a'
case 'b': r = '\b'
case 'e': r = '\e'
case 'f': r = '\f'
case 'n': r = '\n'
case 'r': r = '\r'
case 't': r = '\t'
case 'v': r = '\v'
case '\\': r = '\\'
case '"': r = '"'
case '\'': r = '\''
case '0'..='7':
v := int(c-'0')
if len(s) < 2 {
return
}
for i in 0..<len(s) {
d := int(s[i]-'0')
if d < 0 || d > 7 {
return
}
v = (v<<3) | d
}
s = s[2:]
if v > 0xff {
return
}
r = rune(v)
case 'x', 'u', 'U':
count: int
switch c {
case 'x': count = 2
case 'u': count = 4
case 'U': count = 8
}
if len(s) < count {
return
}
for i in 0..<count {
d := hex_to_int(s[i])
if d < 0 {
return
}
r = (r<<4) | rune(d)
}
s = s[count:]
if c == 'x' {
break
}
if r > utf8.MAX_RUNE {
return
}
multiple_bytes = true
}
success = true
tail_string = s
return
}
unquote_string :: proc(lit: string, allocator := context.allocator) -> (res: string, allocated, success: bool) {
contains_rune :: proc(s: string, r: rune) -> int {
for c, offset in s {
if c == r {
return offset
}
}
return -1
}
assert(len(lit) >= 2)
s := lit
quote := '"'
if s == `""` {
return "", false, true
}
if contains_rune(s, '\n') >= 0 {
return s, false, false
}
if contains_rune(s, '\\') < 0 && contains_rune(s, quote) < 0 {
if quote == '"' {
return s, false, true
}
}
s = s[1:len(s)-1]
buf_len := 3*len(s) / 2
buf := make([]byte, buf_len, allocator)
offset := 0
for len(s) > 0 {
r, multiple_bytes, tail_string, ok := unquote_char(s, byte(quote))
if !ok {
delete(buf)
return s, false, false
}
s = tail_string
if r < 0x80 || !multiple_bytes {
buf[offset] = byte(r)
offset += 1
} else {
b, w := utf8.encode_rune(r)
copy(buf[offset:], b[:w])
offset += w
}
}
new_string := string(buf[:offset])
return new_string, true, true
}
core/c/frontend/tokenizer/doc.odin
-31
View File
@@ -1,31 +0,0 @@
/*
Example:
package demo
import tokenizer "core:c/frontend/tokenizer"
import preprocessor "core:c/frontend/preprocessor"
import "core:fmt"
main :: proc() {
t := &tokenizer.Tokenizer{};
tokenizer.init_defaults(t);
cpp := &preprocessor.Preprocessor{};
cpp.warn, cpp.err = t.warn, t.err;
preprocessor.init_lookup_tables(cpp);
preprocessor.init_default_macros(cpp);
cpp.include_paths = {"my/path/to/include"};
tok := tokenizer.tokenize_file(t, "the/source/file.c", 1);
tok = preprocessor.preprocess(cpp, tok);
if tok != nil {
for t := tok; t.kind != .EOF; t = t.next {
fmt.println(t.lit);
}
}
fmt.println("[Done]");
}
*/
package c_frontend_tokenizer
core/c/frontend/tokenizer/hide_set.odin
-68
View File
@@ -1,68 +0,0 @@
package c_frontend_tokenizer
// NOTE(bill): This is a really dumb approach for a hide set,
// but it's really simple and probably fast enough in practice
Hide_Set :: struct {
next: ^Hide_Set,
name: string,
}
new_hide_set :: proc(name: string) -> ^Hide_Set {
hs := new(Hide_Set)
hs.name = name
return hs
}
hide_set_contains :: proc(hs: ^Hide_Set, name: string) -> bool {
for h := hs; h != nil; h = h.next {
if h.name == name {
return true
}
}
return false
}
hide_set_union :: proc(a, b: ^Hide_Set) -> ^Hide_Set {
head: Hide_Set
curr := &head
for h := a; h != nil; h = h.next {
curr.next = new_hide_set(h.name)
curr = curr.next
}
curr.next = b
return head.next
}
hide_set_intersection :: proc(a, b: ^Hide_Set) -> ^Hide_Set {
head: Hide_Set
curr := &head
for h := a; h != nil; h = h.next {
if hide_set_contains(b, h.name) {
curr.next = new_hide_set(h.name)
curr = curr.next
}
}
return head.next
}
add_hide_set :: proc(tok: ^Token, hs: ^Hide_Set) -> ^Token {
head: Token
curr := &head
tok := tok
for ; tok != nil; tok = tok.next {
t := copy_token(tok)
t.hide_set = hide_set_union(t.hide_set, hs)
curr.next = t
curr = curr.next
}
return head.next
}
core/c/frontend/tokenizer/token.odin
-169
View File
@@ -1,169 +0,0 @@
package c_frontend_tokenizer
Pos :: struct {
file: string,
line: int,
column: int,
offset: int,
}
Token_Kind :: enum {
Invalid,
Ident,
Punct,
Keyword,
Char,
String,
Number,
PP_Number,
Comment,
EOF,
}
File :: struct {
name: string,
id: int,
src: []byte,
display_name: string,
line_delta: int,
}
Token_Type_Hint :: enum u8 {
None,
Int,
Long,
Long_Long,
Unsigned_Int,
Unsigned_Long,
Unsigned_Long_Long,
Float,
Double,
Long_Double,
UTF_8,
UTF_16,
UTF_32,
UTF_Wide,
}
Token_Value :: union {
i64,
f64,
string,
[]u16,
[]u32,
}
Token :: struct {
kind: Token_Kind,
next: ^Token,
lit: string,
pos: Pos,
file: ^File,
line_delta: int,
at_bol: bool,
has_space: bool,
type_hint: Token_Type_Hint,
val: Token_Value,
prefix: string,
// Preprocessor values
hide_set: ^Hide_Set,
origin: ^Token,
}
Is_Keyword_Proc :: #type proc(tok: ^Token) -> bool
copy_token :: proc(tok: ^Token) -> ^Token {
t, _ := new_clone(tok^)
t.next = nil
return t
}
new_eof :: proc(tok: ^Token) -> ^Token {
t, _ := new_clone(tok^)
t.kind = .EOF
t.lit = ""
return t
}
default_is_keyword :: proc(tok: ^Token) -> bool {
if tok.kind == .Keyword {
return true
}
if len(tok.lit) > 0 {
return default_keyword_set[tok.lit]
}
return false
}
token_name := [Token_Kind]string {
.Invalid = "invalid",
.Ident = "ident",
.Punct = "punct",
.Keyword = "keyword",
.Char = "char",
.String = "string",
.Number = "number",
.PP_Number = "preprocessor number",
.Comment = "comment",
.EOF = "eof",
}
default_keyword_set := map[string]bool{
"auto" = true,
"break" = true,
"case" = true,
"char" = true,
"const" = true,
"continue" = true,
"default" = true,
"do" = true,
"double" = true,
"else" = true,
"enum" = true,
"extern" = true,
"float" = true,
"for" = true,
"goto" = true,
"if" = true,
"int" = true,
"long" = true,
"register" = true,
"restrict" = true,
"return" = true,
"short" = true,
"signed" = true,
"sizeof" = true,
"static" = true,
"struct" = true,
"switch" = true,
"typedef" = true,
"union" = true,
"unsigned" = true,
"void" = true,
"volatile" = true,
"while" = true,
"_Alignas" = true,
"_Alignof" = true,
"_Atomic" = true,
"_Bool" = true,
"_Generic" = true,
"_Noreturn" = true,
"_Thread_local" = true,
"__restrict" = true,
"typeof" = true,
"asm" = true,
"__restrict__" = true,
"__thread" = true,
"__attribute__" = true,
}
core/c/frontend/tokenizer/tokenizer.odin
-667
View File
@@ -1,667 +0,0 @@
package c_frontend_tokenizer
import "core:fmt"
import "core:os"
import "core:strings"
import "core:unicode/utf8"
Error_Handler :: #type proc(pos: Pos, fmt: string, args: ..any)
Tokenizer :: struct {
// Immutable data
path: string,
src: []byte,
// Tokenizing state
ch: rune,
offset: int,
read_offset: int,
line_offset: int,
line_count: int,
// Extra information for tokens
at_bol: bool,
has_space: bool,
// Mutable data
err: Error_Handler,
warn: Error_Handler,
error_count: int,
warning_count: int,
}
init_defaults :: proc(t: ^Tokenizer, err: Error_Handler = default_error_handler, warn: Error_Handler = default_warn_handler) {
t.err = err
t.warn = warn
}
@(private)
offset_to_pos :: proc(t: ^Tokenizer, offset: int) -> (pos: Pos) {
pos.file = t.path
pos.offset = offset
pos.line = t.line_count
pos.column = offset - t.line_offset + 1
return
}
default_error_handler :: proc(pos: Pos, msg: string, args: ..any) {
fmt.eprintf("%s(%d:%d) ", pos.file, pos.line, pos.column)
fmt.eprintf(msg, ..args)
fmt.eprintf("\n")
}
default_warn_handler :: proc(pos: Pos, msg: string, args: ..any) {
fmt.eprintf("%s(%d:%d) warning: ", pos.file, pos.line, pos.column)
fmt.eprintf(msg, ..args)
fmt.eprintf("\n")
}
error_offset :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {
pos := offset_to_pos(t, offset)
if t.err != nil {
t.err(pos, msg, ..args)
}
t.error_count += 1
}
warn_offset :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) {
pos := offset_to_pos(t, offset)
if t.warn != nil {
t.warn(pos, msg, ..args)
}
t.warning_count += 1
}
error :: proc(t: ^Tokenizer, tok: ^Token, msg: string, args: ..any) {
pos := tok.pos
if t.err != nil {
t.err(pos, msg, ..args)
}
t.error_count += 1
}
warn :: proc(t: ^Tokenizer, tok: ^Token, msg: string, args: ..any) {
pos := tok.pos
if t.warn != nil {
t.warn(pos, msg, ..args)
}
t.warning_count += 1
}
advance_rune :: proc(t: ^Tokenizer) {
if t.read_offset < len(t.src) {
t.offset = t.read_offset
if t.ch == '\n' {
t.at_bol = true
t.line_offset = t.offset
t.line_count += 1
}
r, w := rune(t.src[t.read_offset]), 1
switch {
case r == 0:
error_offset(t, t.offset, "illegal character NUL")
case r >= utf8.RUNE_SELF:
r, w = utf8.decode_rune(t.src[t.read_offset:])
if r == utf8.RUNE_ERROR && w == 1 {
error_offset(t, t.offset, "illegal UTF-8 encoding")
} else if r == utf8.RUNE_BOM && t.offset > 0 {
error_offset(t, t.offset, "illegal byte order mark")
}
}
t.read_offset += w
t.ch = r
} else {
t.offset = len(t.src)
if t.ch == '\n' {
t.at_bol = true
t.line_offset = t.offset
t.line_count += 1
}
t.ch = -1
}
}
advance_rune_n :: proc(t: ^Tokenizer, n: int) {
for _ in 0..<n {
advance_rune(t)
}
}
is_digit :: proc(r: rune) -> bool {
return '0' <= r && r <= '9'
}
skip_whitespace :: proc(t: ^Tokenizer) {
for {
switch t.ch {
case ' ', '\t', '\r', '\v', '\f', '\n':
t.has_space = true
advance_rune(t)
case:
return
}
}
}
scan_comment :: proc(t: ^Tokenizer) -> string {
offset := t.offset-1
next := -1
general: {
if t.ch == '/'{ // line comments
advance_rune(t)
for t.ch != '\n' && t.ch >= 0 {
advance_rune(t)
}
next = t.offset
if t.ch == '\n' {
next += 1
}
break general
}
/* style comment */
advance_rune(t)
for t.ch >= 0 {
ch := t.ch
advance_rune(t)
if ch == '*' && t.ch == '/' {
advance_rune(t)
next = t.offset
break general
}
}
error_offset(t, offset, "comment not terminated")
}
lit := t.src[offset : t.offset]
// NOTE(bill): Strip CR for line comments
for len(lit) > 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' {
lit = lit[:len(lit)-1]
}
return string(lit)
}
scan_identifier :: proc(t: ^Tokenizer) -> string {
offset := t.offset
for is_ident1(t.ch) {
advance_rune(t)
}
return string(t.src[offset : t.offset])
}
scan_string :: proc(t: ^Tokenizer) -> string {
offset := t.offset-1
for {
ch := t.ch
if ch == '\n' || ch < 0 {
error_offset(t, offset, "string literal was not terminated")
break
}
advance_rune(t)
if ch == '"' {
break
}
if ch == '\\' {
scan_escape(t)
}
}
return string(t.src[offset : t.offset])
}
digit_val :: proc(r: rune) -> int {
switch r {
case '0'..='9':
return int(r-'0')
case 'A'..='F':
return int(r-'A' + 10)
case 'a'..='f':
return int(r-'a' + 10)
}
return 16
}
scan_escape :: proc(t: ^Tokenizer) -> bool {
offset := t.offset
esc := t.ch
n: int
base, max: u32
switch esc {
case 'a', 'b', 'e', 'f', 'n', 't', 'v', 'r', '\\', '\'', '"':
advance_rune(t)
return true
case '0'..='7':
for digit_val(t.ch) < 8 {
advance_rune(t)
}
return true
case 'x':
advance_rune(t)
for digit_val(t.ch) < 16 {
advance_rune(t)
}
return true
case 'u':
advance_rune(t)
n, base, max = 4, 16, utf8.MAX_RUNE
case 'U':
advance_rune(t)
n, base, max = 8, 16, utf8.MAX_RUNE
case:
if t.ch < 0 {
error_offset(t, offset, "escape sequence was not terminated")
} else {
break
}
return false
}
x: u32
main_loop: for n > 0 {
d := u32(digit_val(t.ch))
if d >= base {
if t.ch == '"' || t.ch == '\'' {
break main_loop
}
if t.ch < 0 {
error_offset(t, t.offset, "escape sequence was not terminated")
} else {
error_offset(t, t.offset, "illegal character '%r' : %d in escape sequence", t.ch, t.ch)
}
return false
}
x = x*base + d
advance_rune(t)
n -= 1
}
if x > max || 0xd800 <= x && x <= 0xdfff {
error_offset(t, offset, "escape sequence is an invalid Unicode code point")
return false
}
return true
}
scan_rune :: proc(t: ^Tokenizer) -> string {
offset := t.offset-1
valid := true
n := 0
for {
ch := t.ch
if ch == '\n' || ch < 0 {
if valid {
error_offset(t, offset, "rune literal not terminated")
valid = false
}
break
}
advance_rune(t)
if ch == '\'' {
break
}
n += 1
if ch == '\\' {
if !scan_escape(t) {
valid = false
}
}
}
if valid && n != 1 {
error_offset(t, offset, "illegal rune literal")
}
return string(t.src[offset : t.offset])
}
scan_number :: proc(t: ^Tokenizer, seen_decimal_point: bool) -> (Token_Kind, string) {
scan_mantissa :: proc(t: ^Tokenizer, base: int) {
for digit_val(t.ch) < base {
advance_rune(t)
}
}
scan_exponent :: proc(t: ^Tokenizer) {
if t.ch == 'e' || t.ch == 'E' || t.ch == 'p' || t.ch == 'P' {
advance_rune(t)
if t.ch == '-' || t.ch == '+' {
advance_rune(t)
}
if digit_val(t.ch) < 10 {
scan_mantissa(t, 10)
} else {
error_offset(t, t.offset, "illegal floating-point exponent")
}
}
}
scan_fraction :: proc(t: ^Tokenizer) -> (early_exit: bool) {
if t.ch == '.' && peek(t) == '.' {
return true
}
if t.ch == '.' {
advance_rune(t)
scan_mantissa(t, 10)
}
return false
}
check_end := true
offset := t.offset
seen_point := seen_decimal_point
if seen_point {
offset -= 1
scan_mantissa(t, 10)
scan_exponent(t)
} else {
if t.ch == '0' {
int_base :: proc(t: ^Tokenizer, base: int, msg: string) {
prev := t.offset
advance_rune(t)
scan_mantissa(t, base)
if t.offset - prev <= 1 {
error_offset(t, t.offset, msg)
}
}
advance_rune(t)
switch t.ch {
case 'b', 'B':
int_base(t, 2, "illegal binary integer")
case 'x', 'X':
int_base(t, 16, "illegal hexadecimal integer")
case:
seen_point = false
scan_mantissa(t, 10)
if t.ch == '.' {
seen_point = true
if scan_fraction(t) {
check_end = false
}
}
if check_end {
scan_exponent(t)
check_end = false
}
}
}
}
if check_end {
scan_mantissa(t, 10)
if !scan_fraction(t) {
scan_exponent(t)
}
}
return .Number, string(t.src[offset : t.offset])
}
scan_punct :: proc(t: ^Tokenizer, ch: rune) -> (kind: Token_Kind) {
kind = .Punct
switch ch {
case:
kind = .Invalid
case '<', '>':
if t.ch == ch {
advance_rune(t)
}
if t.ch == '=' {
advance_rune(t)
}
case '!', '+', '-', '*', '/', '%', '^', '=':
if t.ch == '=' {
advance_rune(t)
}
case '#':
if t.ch == '#' {
advance_rune(t)
}
case '&':
if t.ch == '=' || t.ch == '&' {
advance_rune(t)
}
case '|':
if t.ch == '=' || t.ch == '|' {
advance_rune(t)
}
case '(', ')', '[', ']', '{', '}':
// okay
case '~', ',', ':', ';', '?':
// okay
case '`':
// okay
case '.':
if t.ch == '.' && peek(t) == '.' {
advance_rune(t)
advance_rune(t) // consume last '.'
}
}
return
}
peek :: proc(t: ^Tokenizer) -> byte {
if t.read_offset < len(t.src) {
return t.src[t.read_offset]
}
return 0
}
peek_str :: proc(t: ^Tokenizer, str: string) -> bool {
if t.read_offset < len(t.src) {
return strings.has_prefix(string(t.src[t.offset:]), str)
}
return false
}
scan_literal_prefix :: proc(t: ^Tokenizer, str: string, prefix: ^string) -> bool {
if peek_str(t, str) {
offset := t.offset
for _ in str {
advance_rune(t)
}
prefix^ = string(t.src[offset:][:len(str)-1])
return true
}
return false
}
allow_next_to_be_newline :: proc(t: ^Tokenizer) -> bool {
if t.ch == '\n' {
advance_rune(t)
return true
} else if t.ch == '\r' && peek(t) == '\n' { // allow for MS-DOS style line endings
advance_rune(t) // \r
advance_rune(t) // \n
return true
}
return false
}
scan :: proc(t: ^Tokenizer, f: ^File) -> ^Token {
skip_whitespace(t)
offset := t.offset
kind: Token_Kind
lit: string
prefix: string
switch ch := t.ch; {
case scan_literal_prefix(t, `u8"`, &prefix):
kind = .String
lit = scan_string(t)
case scan_literal_prefix(t, `u"`, &prefix):
kind = .String
lit = scan_string(t)
case scan_literal_prefix(t, `L"`, &prefix):
kind = .String
lit = scan_string(t)
case scan_literal_prefix(t, `U"`, &prefix):
kind = .String
lit = scan_string(t)
case scan_literal_prefix(t, `u'`, &prefix):
kind = .Char
lit = scan_rune(t)
case scan_literal_prefix(t, `L'`, &prefix):
kind = .Char
lit = scan_rune(t)
case scan_literal_prefix(t, `U'`, &prefix):
kind = .Char
lit = scan_rune(t)
case is_ident0(ch):
lit = scan_identifier(t)
kind = .Ident
case '0' <= ch && ch <= '9':
kind, lit = scan_number(t, false)
case:
advance_rune(t)
switch ch {
case -1:
kind = .EOF
case '\\':
kind = .Punct
if allow_next_to_be_newline(t) {
t.at_bol = true
t.has_space = false
return scan(t, f)
}
case '.':
if is_digit(t.ch) {
kind, lit = scan_number(t, true)
} else {
kind = scan_punct(t, ch)
}
case '"':
kind = .String
lit = scan_string(t)
case '\'':
kind = .Char
lit = scan_rune(t)
case '/':
if t.ch == '/' || t.ch == '*' {
kind = .Comment
lit = scan_comment(t)
t.has_space = true
break
}
fallthrough
case:
kind = scan_punct(t, ch)
if kind == .Invalid && ch != utf8.RUNE_BOM {
error_offset(t, t.offset, "illegal character '%r': %d", ch, ch)
}
}
}
if lit == "" {
lit = string(t.src[offset : t.offset])
}
if kind == .Comment {
return scan(t, f)
}
tok := new(Token)
tok.kind = kind
tok.lit = lit
tok.pos = offset_to_pos(t, offset)
tok.file = f
tok.prefix = prefix
tok.at_bol = t.at_bol
tok.has_space = t.has_space
t.at_bol, t.has_space = false, false
return tok
}
tokenize :: proc(t: ^Tokenizer, f: ^File) -> ^Token {
setup_tokenizer: {
t.src = f.src
t.ch = ' '
t.offset = 0
t.read_offset = 0
t.line_offset = 0
t.line_count = len(t.src) > 0 ? 1 : 0
t.error_count = 0
t.path = f.name
advance_rune(t)
if t.ch == utf8.RUNE_BOM {
advance_rune(t)
}
}
t.at_bol = true
t.has_space = false
head: Token
curr := &head
for {
tok := scan(t, f)
if tok == nil {
break
}
curr.next = tok
curr = curr.next
if tok.kind == .EOF {
break
}
}
return head.next
}
add_new_file :: proc(t: ^Tokenizer, name: string, src: []byte, id: int) -> ^File {
file := new(File)
file.id = id
file.src = src
file.name = name
file.display_name = name
return file
}
tokenize_file :: proc(t: ^Tokenizer, path: string, id: int, loc := #caller_location) -> ^Token {
src, ok := os.read_entire_file(path)
if !ok {
return nil
}
return tokenize(t, add_new_file(t, path, src, id))
}
inline_tokenize :: proc(t: ^Tokenizer, tok: ^Token, src: []byte) -> ^Token {
file := new(File)
file.src = src
if tok.file != nil {
file.id = tok.file.id
file.name = tok.file.name
file.display_name = tok.file.name
}
return tokenize(t, file)
}
core/c/frontend/tokenizer/unicode.odin
-116
View File
@@ -1,116 +0,0 @@
package c_frontend_tokenizer
in_range :: proc(range: []rune, c: rune) -> bool #no_bounds_check {
for i := 0; range[i] != -1; i += 2 {
if range[i] <= c && c <= range[i+1] {
return true
}
}
return false
}
// [https://www.sigbus.info/n1570#D] C11 allows ASCII and some multibyte characters in certan Unicode ranges to be used in an identifier.
//
// is_ident0 returns true if a given character is acceptable as the first character of an identifier.
is_ident0 :: proc(c: rune) -> bool {
return in_range(_range_ident0, c)
}
// is_ident0 returns true if a given character is acceptable as a non-first character of an identifier.
is_ident1 :: proc(c: rune) -> bool {
return is_ident0(c) || in_range(_range_ident1, c)
}
// Returns the number of columns needed to display a given character in a fixed-width font.
// Based on https://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
char_width :: proc(c: rune) -> int {
switch {
case in_range(_range_width0, c):
return 0
case in_range(_range_width2, c):
return 2
}
return 1
}
display_width :: proc(str: string) -> (w: int) {
for c in str {
w += char_width(c)
}
return
}
_range_ident0 := []rune{
'_', '_', 'a', 'z', 'A', 'Z', '$', '$',
0x00A8, 0x00A8, 0x00AA, 0x00AA, 0x00AD, 0x00AD, 0x00AF, 0x00AF,
0x00B2, 0x00B5, 0x00B7, 0x00BA, 0x00BC, 0x00BE, 0x00C0, 0x00D6,
0x00D8, 0x00F6, 0x00F8, 0x00FF, 0x0100, 0x02FF, 0x0370, 0x167F,
0x1681, 0x180D, 0x180F, 0x1DBF, 0x1E00, 0x1FFF, 0x200B, 0x200D,
0x202A, 0x202E, 0x203F, 0x2040, 0x2054, 0x2054, 0x2060, 0x206F,
0x2070, 0x20CF, 0x2100, 0x218F, 0x2460, 0x24FF, 0x2776, 0x2793,
0x2C00, 0x2DFF, 0x2E80, 0x2FFF, 0x3004, 0x3007, 0x3021, 0x302F,
0x3031, 0x303F, 0x3040, 0xD7FF, 0xF900, 0xFD3D, 0xFD40, 0xFDCF,
0xFDF0, 0xFE1F, 0xFE30, 0xFE44, 0xFE47, 0xFFFD,
0x10000, 0x1FFFD, 0x20000, 0x2FFFD, 0x30000, 0x3FFFD, 0x40000, 0x4FFFD,
0x50000, 0x5FFFD, 0x60000, 0x6FFFD, 0x70000, 0x7FFFD, 0x80000, 0x8FFFD,
0x90000, 0x9FFFD, 0xA0000, 0xAFFFD, 0xB0000, 0xBFFFD, 0xC0000, 0xCFFFD,
0xD0000, 0xDFFFD, 0xE0000, 0xEFFFD,
-1,
}
_range_ident1 := []rune{
'0', '9', '$', '$', 0x0300, 0x036F, 0x1DC0, 0x1DFF, 0x20D0, 0x20FF, 0xFE20, 0xFE2F,
-1,
}
_range_width0 := []rune{
0x0000, 0x001F, 0x007f, 0x00a0, 0x0300, 0x036F, 0x0483, 0x0486,
0x0488, 0x0489, 0x0591, 0x05BD, 0x05BF, 0x05BF, 0x05C1, 0x05C2,
0x05C4, 0x05C5, 0x05C7, 0x05C7, 0x0600, 0x0603, 0x0610, 0x0615,
0x064B, 0x065E, 0x0670, 0x0670, 0x06D6, 0x06E4, 0x06E7, 0x06E8,
0x06EA, 0x06ED, 0x070F, 0x070F, 0x0711, 0x0711, 0x0730, 0x074A,
0x07A6, 0x07B0, 0x07EB, 0x07F3, 0x0901, 0x0902, 0x093C, 0x093C,
0x0941, 0x0948, 0x094D, 0x094D, 0x0951, 0x0954, 0x0962, 0x0963,
0x0981, 0x0981, 0x09BC, 0x09BC, 0x09C1, 0x09C4, 0x09CD, 0x09CD,
0x09E2, 0x09E3, 0x0A01, 0x0A02, 0x0A3C, 0x0A3C, 0x0A41, 0x0A42,
0x0A47, 0x0A48, 0x0A4B, 0x0A4D, 0x0A70, 0x0A71, 0x0A81, 0x0A82,
0x0ABC, 0x0ABC, 0x0AC1, 0x0AC5, 0x0AC7, 0x0AC8, 0x0ACD, 0x0ACD,
0x0AE2, 0x0AE3, 0x0B01, 0x0B01, 0x0B3C, 0x0B3C, 0x0B3F, 0x0B3F,
0x0B41, 0x0B43, 0x0B4D, 0x0B4D, 0x0B56, 0x0B56, 0x0B82, 0x0B82,
0x0BC0, 0x0BC0, 0x0BCD, 0x0BCD, 0x0C3E, 0x0C40, 0x0C46, 0x0C48,
0x0C4A, 0x0C4D, 0x0C55, 0x0C56, 0x0CBC, 0x0CBC, 0x0CBF, 0x0CBF,
0x0CC6, 0x0CC6, 0x0CCC, 0x0CCD, 0x0CE2, 0x0CE3, 0x0D41, 0x0D43,
0x0D4D, 0x0D4D, 0x0DCA, 0x0DCA, 0x0DD2, 0x0DD4, 0x0DD6, 0x0DD6,
0x0E31, 0x0E31, 0x0E34, 0x0E3A, 0x0E47, 0x0E4E, 0x0EB1, 0x0EB1,
0x0EB4, 0x0EB9, 0x0EBB, 0x0EBC, 0x0EC8, 0x0ECD, 0x0F18, 0x0F19,
0x0F35, 0x0F35, 0x0F37, 0x0F37, 0x0F39, 0x0F39, 0x0F71, 0x0F7E,
0x0F80, 0x0F84, 0x0F86, 0x0F87, 0x0F90, 0x0F97, 0x0F99, 0x0FBC,
0x0FC6, 0x0FC6, 0x102D, 0x1030, 0x1032, 0x1032, 0x1036, 0x1037,
0x1039, 0x1039, 0x1058, 0x1059, 0x1160, 0x11FF, 0x135F, 0x135F,
0x1712, 0x1714, 0x1732, 0x1734, 0x1752, 0x1753, 0x1772, 0x1773,
0x17B4, 0x17B5, 0x17B7, 0x17BD, 0x17C6, 0x17C6, 0x17C9, 0x17D3,
0x17DD, 0x17DD, 0x180B, 0x180D, 0x18A9, 0x18A9, 0x1920, 0x1922,
0x1927, 0x1928, 0x1932, 0x1932, 0x1939, 0x193B, 0x1A17, 0x1A18,
0x1B00, 0x1B03, 0x1B34, 0x1B34, 0x1B36, 0x1B3A, 0x1B3C, 0x1B3C,
0x1B42, 0x1B42, 0x1B6B, 0x1B73, 0x1DC0, 0x1DCA, 0x1DFE, 0x1DFF,
0x200B, 0x200F, 0x202A, 0x202E, 0x2060, 0x2063, 0x206A, 0x206F,
0x20D0, 0x20EF, 0x302A, 0x302F, 0x3099, 0x309A, 0xA806, 0xA806,
0xA80B, 0xA80B, 0xA825, 0xA826, 0xFB1E, 0xFB1E, 0xFE00, 0xFE0F,
0xFE20, 0xFE23, 0xFEFF, 0xFEFF, 0xFFF9, 0xFFFB, 0x10A01, 0x10A03,
0x10A05, 0x10A06, 0x10A0C, 0x10A0F, 0x10A38, 0x10A3A, 0x10A3F, 0x10A3F,
0x1D167, 0x1D169, 0x1D173, 0x1D182, 0x1D185, 0x1D18B, 0x1D1AA, 0x1D1AD,
0x1D242, 0x1D244, 0xE0001, 0xE0001, 0xE0020, 0xE007F, 0xE0100, 0xE01EF,
-1,
}
_range_width2 := []rune{
0x1100, 0x115F, 0x2329, 0x2329, 0x232A, 0x232A, 0x2E80, 0x303E,
0x3040, 0xA4CF, 0xAC00, 0xD7A3, 0xF900, 0xFAFF, 0xFE10, 0xFE19,
0xFE30, 0xFE6F, 0xFF00, 0xFF60, 0xFFE0, 0xFFE6, 0x1F000, 0x1F644,
0x20000, 0x2FFFD, 0x30000, 0x3FFFD,
-1,
}