ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-07-29 10:50:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Modify CommentGroup parsing

Browse Source
This commit is contained in:
gingerBill
2018-06-17 10:29:20 +01:00
parent 6d1c32eb77
commit 3eb8aa8268
4 changed files with 77 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/unicode/utf8/utf8.odin
+1 -1
View File
@@ -26,7 +26,7 @@ RUNE1_MAX :: 1<<7 - 1;
RUNE2_MAX :: 1<<11 - 1; RUNE2_MAX :: 1<<11 - 1;
RUNE3_MAX :: 1<<16 - 1; RUNE3_MAX :: 1<<16 - 1;
// The default lowest and highest continuation byte. // The default lowest and highest continuation byte.
LOCB :: 0b1000_0000; LOCB :: 0b1000_0000;
HICB :: 0b1011_1111; HICB :: 0b1011_1111;
src/common.cpp
-1
View File
@@ -302,7 +302,6 @@ typedef struct Arena {
gbMutex mutex; gbMutex mutex;
isize total_used; isize total_used;
isize possible_used;
} Arena; } Arena;
#define ARENA_MIN_ALIGNMENT 16 #define ARENA_MIN_ALIGNMENT 16
src/parser.cpp
+54 -59
View File
@@ -417,13 +417,8 @@ bool ast_node_expect(AstNode *node, AstNodeKind kind) {
// NOTE(bill): And this below is why is I/we need a new language! Discriminated unions are a pain in C/C++ // NOTE(bill): And this below is why is I/we need a new language! Discriminated unions are a pain in C/C++
AstNode *alloc_ast_node(AstFile *f, AstNodeKind kind) { AstNode *alloc_ast_node(AstFile *f, AstNodeKind kind) {
Arena *arena = &global_ast_arena; gbAllocator a = ast_allocator();
gbAllocator a = arena_allocator(arena);
AstNode *node = gb_alloc_item(a, AstNode); AstNode *node = gb_alloc_item(a, AstNode);
gb_mutex_lock(&arena->mutex);
defer (gb_mutex_unlock(&arena->mutex));
arena->possible_used += ALIGN_UP(24 + ast_node_sizes[kind], 8);
node->kind = kind; node->kind = kind;
node->file = f; node->file = f;
return node; return node;
@@ -811,7 +806,7 @@ AstNode *ast_bad_decl(AstFile *f, Token begin, Token end) {
} }
AstNode *ast_field(AstFile *f, Array<AstNode *> names, AstNode *type, AstNode *default_value, u32 flags, AstNode *ast_field(AstFile *f, Array<AstNode *> names, AstNode *type, AstNode *default_value, u32 flags,
CommentGroup docs, CommentGroup comment) { CommentGroup *docs, CommentGroup *comment) {
AstNode *result = alloc_ast_node(f, AstNode_Field); AstNode *result = alloc_ast_node(f, AstNode_Field);
result->Field.names = names; result->Field.names = names;
result->Field.type = type; result->Field.type = type;
@@ -952,7 +947,7 @@ AstNode *ast_map_type(AstFile *f, Token token, AstNode *key, AstNode *value) {
AstNode *ast_foreign_block_decl(AstFile *f, Token token, AstNode *foreign_library, Token open, Token close, Array<AstNode *> decls, AstNode *ast_foreign_block_decl(AstFile *f, Token token, AstNode *foreign_library, Token open, Token close, Array<AstNode *> decls,
CommentGroup docs) { CommentGroup *docs) {
AstNode *result = alloc_ast_node(f, AstNode_ForeignBlockDecl); AstNode *result = alloc_ast_node(f, AstNode_ForeignBlockDecl);
result->ForeignBlockDecl.token = token; result->ForeignBlockDecl.token = token;
result->ForeignBlockDecl.foreign_library = foreign_library; result->ForeignBlockDecl.foreign_library = foreign_library;
@@ -973,7 +968,7 @@ AstNode *ast_label_decl(AstFile *f, Token token, AstNode *name) {
} }
AstNode *ast_value_decl(AstFile *f, Array<AstNode *> names, AstNode *type, Array<AstNode *> values, bool is_mutable, AstNode *ast_value_decl(AstFile *f, Array<AstNode *> names, AstNode *type, Array<AstNode *> values, bool is_mutable,
CommentGroup docs, CommentGroup comment) { CommentGroup *docs, CommentGroup *comment) {
AstNode *result = alloc_ast_node(f, AstNode_ValueDecl); AstNode *result = alloc_ast_node(f, AstNode_ValueDecl);
result->ValueDecl.names = names; result->ValueDecl.names = names;
result->ValueDecl.type = type; result->ValueDecl.type = type;
@@ -986,7 +981,7 @@ AstNode *ast_value_decl(AstFile *f, Array<AstNode *> names, AstNode *type, Array
return result; return result;
} }
AstNode *ast_package_decl(AstFile *f, Token token, Token name, CommentGroup docs, CommentGroup comment) { AstNode *ast_package_decl(AstFile *f, Token token, Token name, CommentGroup *docs, CommentGroup *comment) {
AstNode *result = alloc_ast_node(f, AstNode_PackageDecl); AstNode *result = alloc_ast_node(f, AstNode_PackageDecl);
result->PackageDecl.token = token; result->PackageDecl.token = token;
result->PackageDecl.name = name; result->PackageDecl.name = name;
@@ -996,7 +991,7 @@ AstNode *ast_package_decl(AstFile *f, Token token, Token name, CommentGroup docs
} }
AstNode *ast_import_decl(AstFile *f, Token token, bool is_using, Token relpath, Token import_name, AstNode *ast_import_decl(AstFile *f, Token token, bool is_using, Token relpath, Token import_name,
CommentGroup docs, CommentGroup comment) { CommentGroup *docs, CommentGroup *comment) {
AstNode *result = alloc_ast_node(f, AstNode_ImportDecl); AstNode *result = alloc_ast_node(f, AstNode_ImportDecl);
result->ImportDecl.token = token; result->ImportDecl.token = token;
result->ImportDecl.is_using = is_using; result->ImportDecl.is_using = is_using;
@@ -1008,7 +1003,7 @@ AstNode *ast_import_decl(AstFile *f, Token token, bool is_using, Token relpath,
} }
AstNode *ast_foreign_import_decl(AstFile *f, Token token, Token filepath, Token library_name, AstNode *ast_foreign_import_decl(AstFile *f, Token token, Token filepath, Token library_name,
CommentGroup docs, CommentGroup comment) { CommentGroup *docs, CommentGroup *comment) {
AstNode *result = alloc_ast_node(f, AstNode_ForeignImportDecl); AstNode *result = alloc_ast_node(f, AstNode_ForeignImportDecl);
result->ForeignImportDecl.token = token; result->ForeignImportDecl.token = token;
result->ForeignImportDecl.filepath = filepath; result->ForeignImportDecl.filepath = filepath;
@@ -1049,59 +1044,60 @@ Token consume_comment(AstFile *f, isize *end_line_) {
end_line++; end_line++;
} }
} }
} else {
end_line++;
} }
if (end_line_) *end_line_ = end_line; if (end_line_) *end_line_ = end_line;
next_token0(f); next_token0(f);
if (f->curr_token.pos.line > tok.pos.line || tok.kind == Token_EOF) {
end_line++;
}
return tok; return tok;
} }
CommentGroup consume_comment_group(AstFile *f, isize n, isize *end_line_) { CommentGroup *consume_comment_group(AstFile *f, isize n, isize *end_line_) {
Array<Token> list = {}; Array<Token> list = {};
list.allocator = heap_allocator();
isize end_line = f->curr_token.pos.line; isize end_line = f->curr_token.pos.line;
if (f->curr_token.kind == Token_Comment) { while (f->curr_token.kind == Token_Comment &&
array_init(&list, heap_allocator()); f->curr_token.pos.line <= end_line+n) {
while (f->curr_token.kind == Token_Comment && array_add(&list, consume_comment(f, &end_line));
f->curr_token.pos.line <= end_line+n) {
array_add(&list, consume_comment(f, &end_line));
}
} }
if (end_line_) *end_line_ = end_line; if (end_line_) *end_line_ = end_line;
CommentGroup comments = {}; CommentGroup *comments = nullptr;
comments.list = list; if (list.count > 0) {
array_add(&f->comments, comments); comments = gb_alloc_item(ast_allocator(), CommentGroup);
comments->list = list;
array_add(&f->comments, comments);
}
return comments; return comments;
} }
void comsume_comment_groups(AstFile *f, Token prev) { void comsume_comment_groups(AstFile *f, Token prev) {
if (f->curr_token.kind != Token_Comment) return; if (f->curr_token.kind == Token_Comment) {
CommentGroup comment = {}; CommentGroup *comment = nullptr;
isize end_line = 0; isize end_line = 0;
if (f->curr_token.pos.line == prev.pos.line) { if (f->curr_token.pos.line == prev.pos.line) {
comment = consume_comment_group(f, 0, &end_line); comment = consume_comment_group(f, 0, &end_line);
if (f->curr_token.pos.line != end_line) { if (f->curr_token.pos.line != end_line || f->curr_token.kind == Token_EOF) {
f->line_comment = comment; f->line_comment = comment;
}
} }
}
end_line = -1; end_line = -1;
while (f->curr_token.kind == Token_Comment) { while (f->curr_token.kind == Token_Comment) {
comment = consume_comment_group(f, 1, &end_line); comment = consume_comment_group(f, 1, &end_line);
} }
if (end_line < 0) { if (end_line+1 == f->curr_token.pos.line || end_line < 0) {
f->lead_comment = comment; f->lead_comment = comment;
} else if (end_line+1 == f->curr_token.pos.line) { }
f->lead_comment = comment;
}
GB_ASSERT(f->curr_token.kind != Token_Comment); GB_ASSERT(f->curr_token.kind != Token_Comment);
}
} }
@@ -1881,7 +1877,7 @@ AstNode *parse_operand(AstFile *f, bool lhs) {
auto variants = array_make<AstNode *>(heap_allocator()); auto variants = array_make<AstNode *>(heap_allocator());
AstNode *align = nullptr; AstNode *align = nullptr;
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
Token start_token = f->curr_token; Token start_token = f->curr_token;
while (allow_token(f, Token_Hash)) { while (allow_token(f, Token_Hash)) {
@@ -2378,7 +2374,7 @@ void parse_foreign_block_decl(AstFile *f, Array<AstNode *> *decls) {
} }
AstNode *parse_foreign_block(AstFile *f, Token token) { AstNode *parse_foreign_block(AstFile *f, Token token) {
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
AstNode *foreign_library = nullptr; AstNode *foreign_library = nullptr;
if (f->curr_token.kind == Token_export) { if (f->curr_token.kind == Token_export) {
foreign_library = ast_implicit(f, expect_token(f, Token_export)); foreign_library = ast_implicit(f, expect_token(f, Token_export));
@@ -2410,7 +2406,7 @@ AstNode *parse_foreign_block(AstFile *f, Token token) {
return decl; return decl;
} }
AstNode *parse_value_decl(AstFile *f, Array<AstNode *> names, CommentGroup docs) { AstNode *parse_value_decl(AstFile *f, Array<AstNode *> names, CommentGroup *docs) {
bool is_mutable = true; bool is_mutable = true;
AstNode *type = nullptr; AstNode *type = nullptr;
@@ -2476,7 +2472,7 @@ AstNode *parse_value_decl(AstFile *f, Array<AstNode *> names, CommentGroup docs)
AstNode *parse_simple_stmt(AstFile *f, u32 flags) { AstNode *parse_simple_stmt(AstFile *f, u32 flags) {
Token token = f->curr_token; Token token = f->curr_token;
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
Array<AstNode *> lhs = parse_lhs_expr_list(f); Array<AstNode *> lhs = parse_lhs_expr_list(f);
token = f->curr_token; token = f->curr_token;
@@ -2612,12 +2608,11 @@ AstNode *parse_results(AstFile *f) {
// f->expr_level = -1; // f->expr_level = -1;
if (f->curr_token.kind != Token_OpenParen) { if (f->curr_token.kind != Token_OpenParen) {
CommentGroup empty_group = {};
Token begin_token = f->curr_token; Token begin_token = f->curr_token;
Array<AstNode *> empty_names = {}; Array<AstNode *> empty_names = {};
auto list = array_make<AstNode *>(heap_allocator(), 0, 1); auto list = array_make<AstNode *>(heap_allocator(), 0, 1);
AstNode *type = parse_type(f); AstNode *type = parse_type(f);
array_add(&list, ast_field(f, empty_names, type, nullptr, 0, empty_group, empty_group)); array_add(&list, ast_field(f, empty_names, type, nullptr, 0, nullptr, nullptr));
return ast_field_list(f, begin_token, list); return ast_field_list(f, begin_token, list);
} }
@@ -2882,7 +2877,7 @@ bool parse_expect_struct_separator(AstFile *f, AstNode *param) {
AstNode *parse_struct_field_list(AstFile *f, isize *name_count_) { AstNode *parse_struct_field_list(AstFile *f, isize *name_count_) {
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
Token start_token = f->curr_token; Token start_token = f->curr_token;
auto decls = array_make<AstNode *>(heap_allocator()); auto decls = array_make<AstNode *>(heap_allocator());
@@ -2897,7 +2892,7 @@ AstNode *parse_struct_field_list(AstFile *f, isize *name_count_) {
AstNode *parse_field_list(AstFile *f, isize *name_count_, u32 allowed_flags, TokenKind follow, bool allow_default_parameters, bool allow_type_token) { AstNode *parse_field_list(AstFile *f, isize *name_count_, u32 allowed_flags, TokenKind follow, bool allow_default_parameters, bool allow_type_token) {
Token start_token = f->curr_token; Token start_token = f->curr_token;
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
auto params = array_make<AstNode *>(heap_allocator()); auto params = array_make<AstNode *>(heap_allocator());
@@ -2981,7 +2976,7 @@ AstNode *parse_field_list(AstFile *f, isize *name_count_, u32 allowed_flags, Tok
while (f->curr_token.kind != follow && while (f->curr_token.kind != follow &&
f->curr_token.kind != Token_EOF) { f->curr_token.kind != Token_EOF) {
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
u32 set_flags = parse_field_prefixes(f); u32 set_flags = parse_field_prefixes(f);
Array<AstNode *> names = parse_ident_list(f); Array<AstNode *> names = parse_ident_list(f);
@@ -3442,7 +3437,7 @@ enum ImportDeclKind {
}; };
AstNode *parse_import_decl(AstFile *f, ImportDeclKind kind) { AstNode *parse_import_decl(AstFile *f, ImportDeclKind kind) {
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
Token token = expect_token(f, Token_import); Token token = expect_token(f, Token_import);
Token import_name = {}; Token import_name = {};
bool is_using = kind != ImportDecl_Standard; bool is_using = kind != ImportDecl_Standard;
@@ -3475,7 +3470,7 @@ AstNode *parse_import_decl(AstFile *f, ImportDeclKind kind) {
} }
// AstNode *parse_export_decl(AstFile *f) { // AstNode *parse_export_decl(AstFile *f) {
// CommentGroup docs = f->lead_comment; // CommentGroup *docs = f->lead_comment;
// Token token = expect_token(f, Token_export); // Token token = expect_token(f, Token_export);
// Token file_path = expect_token_after(f, Token_String, "export"); // Token file_path = expect_token_after(f, Token_String, "export");
// AstNode *s = nullptr; // AstNode *s = nullptr;
@@ -3491,7 +3486,7 @@ AstNode *parse_import_decl(AstFile *f, ImportDeclKind kind) {
// } // }
AstNode *parse_foreign_decl(AstFile *f) { AstNode *parse_foreign_decl(AstFile *f) {
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
Token token = expect_token(f, Token_foreign); Token token = expect_token(f, Token_foreign);
switch (f->curr_token.kind) { switch (f->curr_token.kind) {
@@ -3589,7 +3584,7 @@ AstNode *parse_stmt(AstFile *f) {
} }
case Token_using: { case Token_using: {
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
Token token = expect_token(f, Token_using); Token token = expect_token(f, Token_using);
if (f->curr_token.kind == Token_import) { if (f->curr_token.kind == Token_import) {
return parse_import_decl(f, ImportDecl_Using); return parse_import_decl(f, ImportDecl_Using);
@@ -4285,7 +4280,7 @@ bool parse_file(Parser *p, AstFile *f) {
comsume_comment_groups(f, f->prev_token); comsume_comment_groups(f, f->prev_token);
CommentGroup docs = f->lead_comment; CommentGroup *docs = f->lead_comment;
f->package_token = expect_token(f, Token_package); f->package_token = expect_token(f, Token_package);
Token package_name = expect_token_after(f, Token_Ident, "package"); Token package_name = expect_token_after(f, Token_Ident, "package");
@@ -4298,9 +4293,9 @@ bool parse_file(Parser *p, AstFile *f) {
} }
f->package_name = package_name.string; f->package_name = package_name.string;
if (docs.list.count > 0) { if (docs != nullptr && docs->list.count > 0) {
for_array(i, docs.list) { for_array(i, docs->list) {
Token tok = docs.list[i]; GB_ASSERT(tok.kind == Token_Comment); Token tok = docs->list[i]; GB_ASSERT(tok.kind == Token_Comment);
String str = tok.string; String str = tok.string;
if (string_starts_with(str, str_lit("//"))) { if (string_starts_with(str, str_lit("//"))) {
String lc = string_trim_whitespace(substring(str, 2, str.len)); String lc = string_trim_whitespace(substring(str, 2, str.len));
src/parser.hpp
+22 -17
View File
@@ -80,10 +80,10 @@ struct AstFile {
// DeclInfo * decl_info; // NOTE(bill): Created in checker // DeclInfo * decl_info; // NOTE(bill): Created in checker
isize error_count; isize error_count;
CommentGroup lead_comment; // Comment (block) before the decl CommentGroup * lead_comment; // Comment (block) before the decl
CommentGroup line_comment; // Comment after the semicolon CommentGroup * line_comment; // Comment after the semicolon
CommentGroup docs; // current docs CommentGroup * docs; // current docs
Array<CommentGroup> comments; // All the comments! Array<CommentGroup *> comments; // All the comments!
#define PARSER_MAX_FIX_COUNT 6 #define PARSER_MAX_FIX_COUNT 6
@@ -337,7 +337,7 @@ AST_NODE_KIND(_DeclBegin, "", struct {}) \
Token open, close; \ Token open, close; \
Array<AstNode *> decls; \ Array<AstNode *> decls; \
Array<AstNode *> attributes; \ Array<AstNode *> attributes; \
CommentGroup docs; \ CommentGroup * docs; \
bool been_handled; \ bool been_handled; \
}) \ }) \
AST_NODE_KIND(Label, "label", struct { \ AST_NODE_KIND(Label, "label", struct { \
@@ -349,17 +349,17 @@ AST_NODE_KIND(_DeclBegin, "", struct {}) \
AstNode * type; \ AstNode * type; \
Array<AstNode *> values; \ Array<AstNode *> values; \
Array<AstNode *> attributes; \ Array<AstNode *> attributes; \
CommentGroup docs; \ CommentGroup * docs; \
CommentGroup comment; \ CommentGroup * comment; \
bool is_using; \ bool is_using; \
bool is_mutable; \ bool is_mutable; \
bool been_handled; \ bool been_handled; \
}) \ }) \
AST_NODE_KIND(PackageDecl, "package declaration", struct { \ AST_NODE_KIND(PackageDecl, "package declaration", struct { \
Token token; \ Token token; \
Token name; \ Token name; \
CommentGroup docs; \ CommentGroup *docs; \
CommentGroup comment; \ CommentGroup *comment; \
}) \ }) \
AST_NODE_KIND(ImportDecl, "import declaration", struct { \ AST_NODE_KIND(ImportDecl, "import declaration", struct { \
AstPackage *package; \ AstPackage *package; \
@@ -367,8 +367,8 @@ AST_NODE_KIND(_DeclBegin, "", struct {}) \
Token relpath; \ Token relpath; \
String fullpath; \ String fullpath; \
Token import_name; \ Token import_name; \
CommentGroup docs; \ CommentGroup *docs; \
CommentGroup comment; \ CommentGroup *comment; \
bool is_using; \ bool is_using; \
bool been_handled; \ bool been_handled; \
}) \ }) \
@@ -378,8 +378,8 @@ AST_NODE_KIND(_DeclBegin, "", struct {}) \
Token library_name; \ Token library_name; \
String collection_name; \ String collection_name; \
String fullpath; \ String fullpath; \
CommentGroup docs; \ CommentGroup *docs; \
CommentGroup comment; \ CommentGroup *comment; \
bool been_handled; \ bool been_handled; \
}) \ }) \
AST_NODE_KIND(_DeclEnd, "", struct {}) \ AST_NODE_KIND(_DeclEnd, "", struct {}) \
@@ -394,8 +394,8 @@ AST_NODE_KIND(_DeclEnd, "", struct {}) \
AstNode * type; \ AstNode * type; \
AstNode * default_value; \ AstNode * default_value; \
u32 flags; \ u32 flags; \
CommentGroup docs; \ CommentGroup * docs; \
CommentGroup comment; \ CommentGroup * comment; \
}) \ }) \
AST_NODE_KIND(FieldList, "field list", struct { \ AST_NODE_KIND(FieldList, "field list", struct { \
Token token; \ Token token; \
@@ -549,4 +549,9 @@ gb_inline bool is_ast_node_when_stmt(AstNode *node) {
gb_global Arena global_ast_arena = {}; gb_global Arena global_ast_arena = {};
gbAllocator ast_allocator(void) {
Arena *arena = &global_ast_arena;
return arena_allocator(arena);
}
AstNode *alloc_ast_node(AstFile *f, AstNodeKind kind); AstNode *alloc_ast_node(AstFile *f, AstNodeKind kind);