ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-06-29 16:51:49 -07:00
Code Issues Packages Projects Releases Wiki Activity

Semicolons mandatory again (and probably forever now...)

Browse Source
This commit is contained in:
Ginger Bill
2016-12-16 11:31:08 +00:00
parent ad84314143
commit f567983260
19 changed files with 2601 additions and 3173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/tokenizer.c
+21 -84
View File
@@ -304,8 +304,6 @@ typedef struct Tokenizer {
u8 * line; // current line pos
isize line_count;
bool insert_semicolon; // Inserts a semicolon before the next newline
isize error_count;
Array(String) allocated_strings;
} Tokenizer;
@@ -418,7 +416,7 @@ gb_inline void destroy_tokenizer(Tokenizer *t) {
void tokenizer_skip_whitespace(Tokenizer *t) {
while (t->curr_rune == ' ' ||
t->curr_rune == '\t' ||
(t->curr_rune == '\n' && !t->insert_semicolon) ||
t->curr_rune == '\n' ||
t->curr_rune == '\r') {
advance_to_next_rune(t);
}
@@ -677,8 +675,6 @@ Token tokenizer_get_token(Tokenizer *t) {
token.pos.line = t->line_count;
token.pos.column = t->curr - t->line + 1;
bool insert_semicolon = false;
Rune curr_rune = t->curr_rune;
if (rune_is_letter(curr_rune)) {
token.kind = Token_Ident;
@@ -706,48 +702,19 @@ Token tokenizer_get_token(Tokenizer *t) {
}
}
}
switch (token.kind) {
case Token_Ident:
case Token_return:
case Token_break:
case Token_continue:
case Token_fallthrough:
insert_semicolon = true;
break;
}
} else {
insert_semicolon = true;
}
} else if (gb_is_between(curr_rune, '0', '9')) {
insert_semicolon = true;
token = scan_number_to_token(t, false);
} else {
advance_to_next_rune(t);
switch (curr_rune) {
case GB_RUNE_EOF:
if (t->insert_semicolon) {
t->insert_semicolon = false;
token.string = str_lit("\n");
token.kind = Token_Semicolon;
return token;
}
token.kind = Token_EOF;
break;
case '\n':
// NOTE(bill): This will be only be reached if t->insert_semicolom was set
// earlier and exited early from tokenizer_skip_whitespace()
t->insert_semicolon = false;
token.string = str_lit("\n");
token.kind = Token_Semicolon;
return token;
case '\'': // Rune Literal
{
insert_semicolon = true;
token.kind = Token_Rune;
Rune quote = curr_rune;
bool valid = true;
@@ -780,7 +747,6 @@ Token tokenizer_get_token(Tokenizer *t) {
if (success == 2) {
array_add(&t->allocated_strings, token.string);
}
t->insert_semicolon = true;
return token;
} else {
tokenizer_err(t, "Invalid rune literal");
@@ -790,8 +756,6 @@ Token tokenizer_get_token(Tokenizer *t) {
case '`': // Raw String Literal
case '"': // String Literal
{
insert_semicolon = true;
i32 success;
Rune quote = curr_rune;
token.kind = Token_String;
@@ -829,7 +793,6 @@ Token tokenizer_get_token(Tokenizer *t) {
if (success == 2) {
array_add(&t->allocated_strings, token.string);
}
t->insert_semicolon = true;
return token;
} else {
tokenizer_err(t, "Invalid string literal");
@@ -839,7 +802,6 @@ Token tokenizer_get_token(Tokenizer *t) {
case '.':
token.kind = Token_Period; // Default
if (gb_is_between(t->curr_rune, '0', '9')) { // Might be a number
insert_semicolon = true;
token = scan_number_to_token(t, true);
} else if (t->curr_rune == '.') { // Could be an ellipsis
advance_to_next_rune(t);
@@ -881,21 +843,18 @@ Token tokenizer_get_token(Tokenizer *t) {
token.kind = Token_OpenParen;
break;
case ')':
insert_semicolon = true;
token.kind = Token_CloseParen;
break;
case '[':
token.kind = Token_OpenBracket;
break;
case ']':
insert_semicolon = true;
token.kind = Token_CloseBracket;
break;
case '{':
token.kind = Token_OpenBrace;
break;
case '}':
insert_semicolon = true;
token.kind = Token_CloseBrace;
break;
@@ -906,56 +865,37 @@ Token tokenizer_get_token(Tokenizer *t) {
case '!': token.kind = token_kind_variant2(t, Token_Not, Token_NotEq); break;
case '+':
token.kind = token_kind_variant3(t, Token_Add, Token_AddEq, '+', Token_Increment);
if (token.kind == Token_Increment) {
insert_semicolon = true;
}
break;
case '-':
token.kind = token_kind_variant4(t, Token_Sub, Token_SubEq, '-', Token_Decrement, '>', Token_ArrowRight);
if (token.kind == Token_Decrement) {
insert_semicolon = true;
}
break;
case '/': {
if (t->curr_rune == '/' || t->curr_rune == '*') {
if (t->insert_semicolon && tokenizer_find_line_end(t)) {
t->curr_rune = '/';
t->curr = token.string.text;
t->read_curr = t->curr+1;
t->insert_semicolon = false;
token.kind = Token_Semicolon;
token.string = str_lit("\n");
return token;
}
if (t->curr_rune == '/') {
while (t->curr_rune != '\n') {
advance_to_next_rune(t);
}
token.kind = Token_Comment;
} else if (t->curr_rune == '*') {
isize comment_scope = 1;
if (t->curr_rune == '/') {
while (t->curr_rune != '\n') {
advance_to_next_rune(t);
while (comment_scope > 0) {
}
token.kind = Token_Comment;
} else if (t->curr_rune == '*') {
isize comment_scope = 1;
advance_to_next_rune(t);
while (comment_scope > 0) {
if (t->curr_rune == '/') {
advance_to_next_rune(t);
if (t->curr_rune == '*') {
advance_to_next_rune(t);
comment_scope++;
}
} else if (t->curr_rune == '*') {
advance_to_next_rune(t);
if (t->curr_rune == '/') {
advance_to_next_rune(t);
if (t->curr_rune == '*') {
advance_to_next_rune(t);
comment_scope++;
}
} else if (t->curr_rune == '*') {
advance_to_next_rune(t);
if (t->curr_rune == '/') {
advance_to_next_rune(t);
comment_scope--;
}
} else {
advance_to_next_rune(t);
comment_scope--;
}
} else {
advance_to_next_rune(t);
}
token.kind = Token_Comment;
}
token.kind = Token_Comment;
} else {
token.kind = token_kind_variant2(t, Token_Quo, Token_QuoEq);
}
@@ -994,14 +934,11 @@ Token tokenizer_get_token(Tokenizer *t) {
int len = cast(int)gb_utf8_encode_rune(str, curr_rune);
tokenizer_err(t, "Illegal character: %.*s (%d) ", len, str, curr_rune);
}
insert_semicolon = t->insert_semicolon;
token.kind = Token_Invalid;
break;
}
}
t->insert_semicolon = insert_semicolon;
token.string.len = t->curr - token.string.text;
return token;
}