ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-07-27 09:50:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1112 from odin-lang/optional-semicolons

Browse Source 
Optional Semicolons
This commit is contained in:
gingerBill
2021-09-06 19:29:08 +01:00
committed by GitHub
parent db8f0874bd 2800d4b8d0
commit b63d49aafa
242 changed files with 38583 additions and 40484 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/build_settings.cpp
+6 -2
View File
@@ -120,8 +120,10 @@ enum CommandKind : u32 {
Command_doc = 1<<5,
Command_version = 1<<6,
Command_test = 1<<7,
Command_strip_semicolon = 1<<8,
Command__does_check = Command_run|Command_build|Command_check|Command_query|Command_doc|Command_test,
Command__does_check = Command_run|Command_build|Command_check|Command_query|Command_doc|Command_test|Command_strip_semicolon,
Command__does_build = Command_run|Command_build|Command_test,
Command_all = ~(u32)0,
};
@@ -134,6 +136,7 @@ char const *odin_command_strings[32] = {
"doc",
"version",
"test",
"strip-semicolon",
};
@@ -201,7 +204,8 @@ struct BuildContext {
bool different_os;
bool keep_object_files;
bool disallow_do;
bool insert_semicolon;
bool strict_style;
bool ignore_warnings;
src/main.cpp
+196 -10
View File
@@ -733,7 +733,7 @@ bool parse_build_flags(Array<String> args) {
add_flag(&build_flags, BuildFlag_ShowUnusedWithLocation, str_lit("show-unused-with-location"), BuildFlagParam_None, Command_check);
add_flag(&build_flags, BuildFlag_ShowSystemCalls, str_lit("show-system-calls"), BuildFlagParam_None, Command_all);
add_flag(&build_flags, BuildFlag_ThreadCount, str_lit("thread-count"), BuildFlagParam_Integer, Command_all);
add_flag(&build_flags, BuildFlag_KeepTempFiles, str_lit("keep-temp-files"), BuildFlagParam_None, Command__does_build);
add_flag(&build_flags, BuildFlag_KeepTempFiles, str_lit("keep-temp-files"), BuildFlagParam_None, Command__does_build|Command_strip_semicolon);
add_flag(&build_flags, BuildFlag_Collection, str_lit("collection"), BuildFlagParam_String, Command__does_check);
add_flag(&build_flags, BuildFlag_Define, str_lit("define"), BuildFlagParam_String, Command__does_check, true);
add_flag(&build_flags, BuildFlag_BuildMode, str_lit("build-mode"), BuildFlagParam_String, Command__does_build); // Commands_build is not used to allow for a better error message
@@ -1307,12 +1307,12 @@ bool parse_build_flags(Array<String> args) {
break;
case BuildFlag_InsertSemicolon:
build_context.insert_semicolon = true;
gb_printf_err("-insert-semicolon flag is not required any more\n");
bad_flags = true;
break;
case BuildFlag_StrictStyle:
gb_printf_err("-strict-style flag is not required any more\n");
bad_flags = true;
build_context.strict_style = true;
break;
@@ -1644,7 +1644,7 @@ void print_show_help(String const arg0, String const &command) {
} else if (command == "run") {
print_usage_line(1, "run same as 'build', but also then runs the newly compiled executable.");
} else if (command == "check") {
print_usage_line(1, "check parse and type check .odin file");
print_usage_line(1, "check parse and type check .odin file(s)");
} else if (command == "test") {
print_usage_line(1, "test build ands runs procedures with the attribute @(test) in the initial package");
} else if (command == "query") {
@@ -1656,14 +1656,18 @@ void print_show_help(String const arg0, String const &command) {
print_usage_line(3, "odin doc core/path core/path/filepath");
} else if (command == "version") {
print_usage_line(1, "version print version");
}
} else if (command == "strip-semicolon") {
print_usage_line(1, "strip-semicolon");
print_usage_line(2, "parse and type check .odin file(s) and then remove unneeded semicolons from the entire project");
}
bool doc = command == "doc";
bool build = command == "build";
bool run_or_build = command == "run" || command == "build" || command == "test";
bool test_only = command == "test";
bool check_only = command == "check";
bool check = run_or_build || command == "check";
bool strip_semicolon = command == "strip-semicolon";
bool check_only = command == "check" || strip_semicolon;
bool check = run_or_build || check_only;
print_usage_line(0, "");
print_usage_line(1, "Flags");
@@ -1730,6 +1734,10 @@ void print_show_help(String const arg0, String const &command) {
print_usage_line(1, "-keep-temp-files");
print_usage_line(2, "Keeps the temporary files generated during compilation");
print_usage_line(0, "");
} else if (strip_semicolon) {
print_usage_line(1, "-keep-temp-files");
print_usage_line(2, "Keeps the temporary files generated during stripping the unneeded semicolons from files");
print_usage_line(0, "");
}
if (check) {
@@ -1862,8 +1870,8 @@ void print_show_help(String const arg0, String const &command) {
print_usage_line(2, "Sets the default allocator to be the nil_allocator, an allocator which does nothing");
print_usage_line(0, "");
print_usage_line(1, "-insert-semicolon");
print_usage_line(2, "Inserts semicolons on newlines during tokenization using a basic rule");
print_usage_line(1, "-strict-style");
print_usage_line(2, "Errs on unneeded tokens, such as unneeded semicolons");
print_usage_line(0, "");
print_usage_line(1, "-ignore-warnings");
@@ -2002,6 +2010,172 @@ bool check_env(void) {
return true;
}
struct StripSemicolonFile {
String old_fullpath;
String old_fullpath_backup;
String new_fullpath;
AstFile *file;
i64 written;
};
gbFileError write_file_with_stripped_tokens(gbFile *f, AstFile *file, i64 *written_) {
i64 written = 0;
gbFileError err = gbFileError_None;
u8 const *file_data = file->tokenizer.start;
i32 prev_offset = 0;
i32 const end_offset = cast(i32)(file->tokenizer.end - file->tokenizer.start);
for_array(i, file->tokens) {
Token *token = &file->tokens[i];
if (token->flags & TokenFlag_Remove) {
i32 offset = token->pos.offset;
i32 to_write = offset-prev_offset;
if (!gb_file_write(f, file_data+prev_offset, to_write)) {
return gbFileError_Invalid;
}
written += to_write;
prev_offset = token_pos_end(*token).offset;
}
}
if (end_offset > prev_offset) {
i32 to_write = end_offset-prev_offset;
if (!gb_file_write(f, file_data+prev_offset, end_offset-prev_offset)) {
return gbFileError_Invalid;
}
written += to_write;
}
if (written_) *written_ = written;
return err;
}
int strip_semicolons(Parser *parser) {
isize file_count = 0;
for_array(i, parser->packages) {
AstPackage *pkg = parser->packages[i];
file_count += pkg->files.count;
}
gb_printf_err("File count to be stripped of unneeded tokens: %td\n", file_count);
auto generated_files = array_make<StripSemicolonFile>(permanent_allocator(), 0, file_count);
for_array(i, parser->packages) {
AstPackage *pkg = parser->packages[i];
for_array(j, pkg->files) {
AstFile *file = pkg->files[j];
String old_fullpath = copy_string(permanent_allocator(), file->fullpath);
// assumes .odin extension
String fullpath_base = substring(old_fullpath, 0, old_fullpath.len-5);
String old_fullpath_backup = concatenate_strings(permanent_allocator(), fullpath_base, str_lit("~backup.odin-temp"));
String new_fullpath = concatenate_strings(permanent_allocator(), fullpath_base, str_lit("~temp.odin-temp"));
array_add(&generated_files, StripSemicolonFile{old_fullpath, old_fullpath_backup, new_fullpath, file});
}
}
isize generated_count = 0;
bool failed = false;
for_array(i, generated_files) {
auto *file = &generated_files[i];
char const *filename = cast(char const *)file->new_fullpath.text;
gbFileError err = gbFileError_None;
defer (if (err != gbFileError_None) {
failed = true;
});
gbFile f = {};
err = gb_file_create(&f, filename);
if (err) {
break;
}
defer (err = gb_file_close(&f));
generated_count += 1;
i64 written = 0;
defer (gb_file_truncate(&f, written));
debugf("Write file with stripped tokens: %s\n", filename);
err = write_file_with_stripped_tokens(&f, file->file, &written);
if (err) {
break;
}
file->written = written;
}
if (failed) {
for (isize i = 0; i < generated_count; i++) {
auto *file = &generated_files[i];
char const *filename = nullptr;
filename = cast(char const *)file->new_fullpath.text;
GB_ASSERT_MSG(gb_file_remove(filename), "unable to delete file %s", filename);
}
return 1;
}
isize overwritten_files = 0;
for_array(i, generated_files) {
auto *file = &generated_files[i];
char const *old_fullpath = cast(char const *)file->old_fullpath.text;
char const *old_fullpath_backup = cast(char const *)file->old_fullpath_backup.text;
char const *new_fullpath = cast(char const *)file->new_fullpath.text;
debugf("Copy '%s' to '%s'\n", old_fullpath, old_fullpath_backup);
if (!gb_file_copy(old_fullpath, old_fullpath_backup, false)) {
gb_printf_err("failed to copy '%s' to '%s'\n", old_fullpath, old_fullpath_backup);
failed = true;
break;
}
debugf("Copy '%s' to '%s'\n", new_fullpath, old_fullpath);
if (!gb_file_copy(new_fullpath, old_fullpath, false)) {
gb_printf_err("failed to copy '%s' to '%s'\n", old_fullpath, new_fullpath);
debugf("Copy '%s' to '%s'\n", old_fullpath_backup, old_fullpath);
if (!gb_file_copy(old_fullpath_backup, old_fullpath, false)) {
gb_printf_err("failed to restore '%s' from '%s'\n", old_fullpath, old_fullpath_backup);
}
failed = true;
break;
}
debugf("Remove '%s'\n", old_fullpath_backup);
if (!gb_file_remove(old_fullpath_backup)) {
gb_printf_err("failed to remove '%s'\n", old_fullpath_backup);
}
overwritten_files++;
}
if (!build_context.keep_temp_files) {
for_array(i, generated_files) {
auto *file = &generated_files[i];
char const *filename = nullptr;
filename = cast(char const *)file->new_fullpath.text;
debugf("Remove '%s'\n", filename);
GB_ASSERT_MSG(gb_file_remove(filename), "unable to delete file %s", filename);
filename = cast(char const *)file->old_fullpath_backup.text;
debugf("Remove '%s'\n", filename);
if (gb_file_exists(filename) && !gb_file_remove(filename)) {
if (i < overwritten_files) {
gb_printf_err("unable to delete file %s", filename);
failed = true;
}
}
}
}
gb_printf_err("Files stripped of unneeded token: %td\n", file_count);
return cast(int)failed;
}
int main(int arg_count, char const **arg_ptr) {
#define TIME_SECTION(str) do { debugf("[Section] %s\n", str); timings_start_section(&global_timings, str_lit(str)); } while (0)
@@ -2090,6 +2264,14 @@ int main(int arg_count, char const **arg_ptr) {
build_context.command_kind = Command_check;
build_context.no_output_files = true;
init_filename = args[2];
} else if (command == "strip-semicolon") {
if (args.count < 3) {
usage(args[0]);
return 1;
}
build_context.command_kind = Command_strip_semicolon;
build_context.no_output_files = true;
init_filename = args[2];
} else if (command == "query") {
if (args.count < 3) {
usage(args[0]);
@@ -2209,6 +2391,10 @@ int main(int arg_count, char const **arg_ptr) {
if (any_errors()) {
return 1;
}
if (build_context.command_kind == Command_strip_semicolon) {
return strip_semicolons(parser);
}
if (build_context.generate_docs) {
if (global_error_collector.count != 0) {
src/parser.cpp
+46 -134
View File
@@ -1240,6 +1240,7 @@ Token advance_token(AstFile *f) {
f->lead_comment = nullptr;
f->line_comment = nullptr;
f->prev_token_index = f->curr_token_index;
Token prev = f->prev_token = f->curr_token;
bool ok = next_token0(f);
@@ -1281,9 +1282,6 @@ Token peek_token(AstFile *f) {
}
bool skip_possible_newline(AstFile *f) {
if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) {
return false;
}
if (token_is_newline(f->curr_token)) {
advance_token(f);
return true;
@@ -1292,9 +1290,6 @@ bool skip_possible_newline(AstFile *f) {
}
bool skip_possible_newline_for_literal(AstFile *f) {
if ((f->tokenizer.flags & TokenizerFlag_InsertSemicolon) == 0) {
return false;
}
Token curr = f->curr_token;
if (token_is_newline(curr)) {
Token next = peek_token(f);
@@ -1496,80 +1491,30 @@ Token expect_closing(AstFile *f, TokenKind kind, String context) {
return expect_token(f, kind);
}
bool is_semicolon_optional_for_node(AstFile *f, Ast *s) {
if (s == nullptr) {
return false;
}
if (build_context.insert_semicolon) {
return true;
}
switch (s->kind) {
case Ast_EmptyStmt:
case Ast_BlockStmt:
return true;
case Ast_IfStmt:
case Ast_WhenStmt:
case Ast_ForStmt:
case Ast_RangeStmt:
case Ast_SwitchStmt:
case Ast_TypeSwitchStmt:
return true;
case Ast_HelperType:
return is_semicolon_optional_for_node(f, s->HelperType.type);
case Ast_DistinctType:
return is_semicolon_optional_for_node(f, s->DistinctType.type);
case Ast_PointerType:
return is_semicolon_optional_for_node(f, s->PointerType.type);
case Ast_StructType:
case Ast_UnionType:
case Ast_EnumType:
// Require semicolon within a procedure body
return f->curr_proc == nullptr;
case Ast_ProcLit:
return true;
case Ast_PackageDecl:
case Ast_ImportDecl:
case Ast_ForeignImportDecl:
return true;
case Ast_ValueDecl:
if (s->ValueDecl.is_mutable) {
return false;
void assign_removal_flag_to_semicolon(AstFile *f) {
// NOTE(bill): this is used for rewriting files to strip unneeded semicolons
Token *prev_token = &f->tokens[f->prev_token_index];
Token *curr_token = &f->tokens[f->curr_token_index];
GB_ASSERT(prev_token->kind == Token_Semicolon);
if (prev_token->string == ";") {
bool ok = false;
if (curr_token->pos.line > prev_token->pos.line) {
ok = true;
} else if (curr_token->pos.line == prev_token->pos.line) {
switch (curr_token->kind) {
case Token_CloseBrace:
case Token_CloseParen:
ok = true;
break;
}
}
if (s->ValueDecl.values.count > 0) {
return is_semicolon_optional_for_node(f, s->ValueDecl.values[s->ValueDecl.values.count-1]);
if (ok) {
if (build_context.strict_style) {
syntax_error(*prev_token, "Found unneeded semicolon");
}
prev_token->flags |= TokenFlag_Remove;
}
break;
case Ast_ForeignBlockDecl:
return is_semicolon_optional_for_node(f, s->ForeignBlockDecl.body);
}
return false;
}
void expect_semicolon_newline_error(AstFile *f, Token const &token, Ast *s) {
if (!build_context.insert_semicolon && token.string == "\n") {
switch (token.kind) {
case Token_CloseBrace:
case Token_CloseParen:
case Token_else:
return;
}
if (is_semicolon_optional_for_node(f, s)) {
return;
}
Token tok = token;
tok.pos.column -= 1;
syntax_error(tok, "Expected ';', got newline");
}
}
@@ -1577,7 +1522,7 @@ void expect_semicolon(AstFile *f, Ast *s) {
Token prev_token = {};
if (allow_token(f, Token_Semicolon)) {
expect_semicolon_newline_error(f, f->prev_token, s);
assign_removal_flag_to_semicolon(f);
return;
}
switch (f->curr_token.kind) {
@@ -1591,7 +1536,7 @@ void expect_semicolon(AstFile *f, Ast *s) {
prev_token = f->prev_token;
if (prev_token.kind == Token_Semicolon) {
expect_semicolon_newline_error(f, f->prev_token, s);
assign_removal_flag_to_semicolon(f);
return;
}
@@ -1599,53 +1544,15 @@ void expect_semicolon(AstFile *f, Ast *s) {
return;
}
if (s != nullptr) {
bool insert_semi = (f->tokenizer.flags & TokenizerFlag_InsertSemicolon) != 0;
if (insert_semi) {
switch (f->curr_token.kind) {
case Token_CloseBrace:
case Token_CloseParen:
case Token_else:
case Token_EOF:
return;
default:
if (is_semicolon_optional_for_node(f, s)) {
return;
}
break;
}
} else if (prev_token.pos.line != f->curr_token.pos.line) {
if (is_semicolon_optional_for_node(f, s)) {
return;
}
} else {
switch (f->curr_token.kind) {
case Token_CloseBrace:
case Token_CloseParen:
case Token_else:
return;
case Token_EOF:
if (is_semicolon_optional_for_node(f, s)) {
return;
}
break;
}
}
String node_string = ast_strings[s->kind];
String p = token_to_string(f->curr_token);
syntax_error(prev_token, "Expected ';' after %.*s, got %.*s",
LIT(node_string), LIT(p));
} else {
switch (f->curr_token.kind) {
case Token_EOF:
return;
}
String p = token_to_string(f->curr_token);
syntax_error(prev_token, "Expected ';', got %.*s", LIT(p));
return;
}
switch (f->curr_token.kind) {
case Token_EOF:
return;
}
String p = token_to_string(f->curr_token);
syntax_error(prev_token, "Expected ';', got %.*s", LIT(p));
fix_advance_to_next_stmt(f);
}
@@ -4144,7 +4051,7 @@ Ast *parse_for_stmt(AstFile *f) {
if (f->curr_token.string != ";") {
syntax_error(f->curr_token, "Expected ';', got %.*s", LIT(token_to_string(f->curr_token)));
} else {
expect_semicolon(f, nullptr);
expect_token(f, Token_Semicolon);
}
if (f->curr_token.kind != Token_OpenBrace &&
@@ -4624,14 +4531,20 @@ Ast *parse_stmt(AstFile *f) {
return s;
} else if (tag == "assert") {
Ast *t = ast_basic_directive(f, hash_token, name);
return ast_expr_stmt(f, parse_call_expr(f, t));
Ast *stmt = ast_expr_stmt(f, parse_call_expr(f, t));
expect_semicolon(f, stmt);
return stmt;
} else if (tag == "panic") {
Ast *t = ast_basic_directive(f, hash_token, name);
return ast_expr_stmt(f, parse_call_expr(f, t));
Ast *stmt = ast_expr_stmt(f, parse_call_expr(f, t));
expect_semicolon(f, stmt);
return stmt;
} else if (name.string == "force_inline" ||
name.string == "force_no_inline") {
Ast *expr = parse_force_inlining_operand(f, name);
return ast_expr_stmt(f, expr);
Ast *stmt = ast_expr_stmt(f, expr);
expect_semicolon(f, stmt);
return stmt;
} else if (tag == "unroll") {
return parse_unrolled_for_loop(f, name);
} else if (tag == "include") {
@@ -4652,7 +4565,7 @@ Ast *parse_stmt(AstFile *f) {
case Token_Semicolon:
s = ast_empty_stmt(f, token);
advance_token(f);
expect_semicolon(f, nullptr);
return s;
}
@@ -4717,12 +4630,10 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
if (!string_ends_with(f->fullpath, str_lit(".odin"))) {
return ParseFile_WrongExtension;
}
TokenizerFlags tokenizer_flags = TokenizerFlag_InsertSemicolon;
zero_item(&f->tokenizer);
f->tokenizer.curr_file_id = f->id;
TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath, tokenizer_flags);
TokenizerInitError err = init_tokenizer_from_fullpath(&f->tokenizer, f->fullpath);
if (err != TokenizerInit_None) {
switch (err) {
case TokenizerInit_Empty:
@@ -4777,8 +4688,9 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
u64 end = time_stamp_time_now();
f->time_to_tokenize = cast(f64)(end-start)/cast(f64)time_stamp__freq();
f->prev_token_index = 0;
f->curr_token_index = 0;
f->prev_token = f->tokens[f->curr_token_index];
f->prev_token = f->tokens[f->prev_token_index];
f->curr_token = f->tokens[f->curr_token_index];
isize const page_size = 4*1024;
src/parser.hpp
+1
View File
@@ -102,6 +102,7 @@ struct AstFile {
Tokenizer tokenizer;
Array<Token> tokens;
isize curr_token_index;
isize prev_token_index;
Token curr_token;
Token prev_token; // previous non-comment
Token package_token;
src/tokenizer.cpp
+48 -57
View File
@@ -120,7 +120,7 @@ TOKEN_KIND(Token__KeywordBegin, ""), \
TOKEN_KIND(Token__KeywordEnd, ""), \
TOKEN_KIND(Token_Count, "")
enum TokenKind {
enum TokenKind : u8 {
#define TOKEN_KIND(e, s) e
TOKEN_KINDS
#undef TOKEN_KIND
@@ -235,21 +235,26 @@ TokenPos token_pos_add_column(TokenPos pos) {
return pos;
}
enum TokenFlag : u8 {
TokenFlag_Remove = 1<<1,
};
struct Token {
TokenKind kind;
u8 flags;
String string;
TokenPos pos;
};
Token empty_token = {Token_Invalid};
Token blank_token = {Token_Ident, {cast(u8 *)"_", 1}};
Token blank_token = {Token_Ident, 0, {cast(u8 *)"_", 1}};
Token make_token_ident(String s) {
Token t = {Token_Ident, s};
Token t = {Token_Ident, 0, s};
return t;
}
Token make_token_ident(char const *s) {
Token t = {Token_Ident, make_string_c(s)};
Token t = {Token_Ident, 0, make_string_c(s)};
return t;
}
@@ -701,12 +706,6 @@ enum TokenizerInitError {
TokenizerInit_Count,
};
enum TokenizerFlags {
TokenizerFlag_None = 0,
TokenizerFlag_InsertSemicolon = 1<<0,
};
struct Tokenizer {
i32 curr_file_id;
String fullpath;
@@ -721,7 +720,6 @@ struct Tokenizer {
i32 error_count;
TokenizerFlags flags;
bool insert_semicolon;
};
@@ -789,8 +787,7 @@ void advance_to_next_rune(Tokenizer *t) {
}
}
void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, isize size, TokenizerFlags flags) {
t->flags = flags;
void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data, isize size) {
t->fullpath = fullpath;
t->line_count = 1;
@@ -804,7 +801,7 @@ void init_tokenizer_with_data(Tokenizer *t, String const &fullpath, void *data,
}
}
TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath, TokenizerFlags flags = TokenizerFlag_None) {
TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &fullpath) {
TokenizerInitError err = TokenizerInit_None;
char *c_str = alloc_cstring(temporary_allocator(), fullpath);
@@ -813,15 +810,13 @@ TokenizerInitError init_tokenizer_from_fullpath(Tokenizer *t, String const &full
gbFileContents fc = gb_file_read_contents(heap_allocator(), true, c_str);
if (fc.size > I32_MAX) {
t->flags = flags;
t->fullpath = fullpath;
t->line_count = 1;
err = TokenizerInit_FileTooLarge;
gb_file_free_contents(&fc);
} else if (fc.data != nullptr) {
init_tokenizer_with_data(t, fullpath, fc.data, fc.size, flags);
init_tokenizer_with_data(t, fullpath, fc.data, fc.size);
} else {
t->flags = flags;
t->fullpath = fullpath;
t->line_count = 1;
gbFile f = {};
@@ -1163,9 +1158,7 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
return;
case '\\':
if (t->flags & TokenizerFlag_InsertSemicolon) {
t->insert_semicolon = false;
}
t->insert_semicolon = false;
tokenizer_get_token(t, token);
if (token->pos.line == current_pos.line) {
tokenizer_err(t, token_pos_add_column(current_pos), "Expected a newline after \\");
@@ -1493,43 +1486,41 @@ void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) {
token->string.len = t->curr - token->string.text;
semicolon_check:;
if (t->flags & TokenizerFlag_InsertSemicolon) {
switch (token->kind) {
case Token_Invalid:
case Token_Comment:
// Preserve insert_semicolon info
break;
case Token_Ident:
case Token_context:
case Token_typeid:
case Token_break:
case Token_continue:
case Token_fallthrough:
case Token_return:
case Token_or_return:
/*fallthrough*/
case Token_Integer:
case Token_Float:
case Token_Imag:
case Token_Rune:
case Token_String:
case Token_Undef:
/*fallthrough*/
case Token_Question:
case Token_Pointer:
case Token_CloseParen:
case Token_CloseBracket:
case Token_CloseBrace:
/*fallthrough*/
case Token_Increment:
case Token_Decrement:
/*fallthrough*/
t->insert_semicolon = true;
break;
default:
t->insert_semicolon = false;
break;
}
switch (token->kind) {
case Token_Invalid:
case Token_Comment:
// Preserve insert_semicolon info
break;
case Token_Ident:
case Token_context:
case Token_typeid:
case Token_break:
case Token_continue:
case Token_fallthrough:
case Token_return:
case Token_or_return:
/*fallthrough*/
case Token_Integer:
case Token_Float:
case Token_Imag:
case Token_Rune:
case Token_String:
case Token_Undef:
/*fallthrough*/
case Token_Question:
case Token_Pointer:
case Token_CloseParen:
case Token_CloseBracket:
case Token_CloseBrace:
/*fallthrough*/
case Token_Increment:
case Token_Decrement:
/*fallthrough*/
t->insert_semicolon = true;
break;
default:
t->insert_semicolon = false;
break;
}
return;