ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-07-30 19:30:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Parse directories to be packages

Browse Source
This commit is contained in:
gingerBill
2018-05-21 20:47:52 +01:00
parent 718b80ba39
commit 5b6770f3d2
39 changed files with 1447 additions and 1209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common.cpp
+100
View File
@@ -734,3 +734,103 @@ String path_to_full_path(gbAllocator a, String path) {
char *fullpath = gb_path_get_full_name(a, path_c);
return make_string_c(fullpath);
}
struct FileInfo {
String name;
String fullpath;
i64 size;
bool is_dir;
};
enum ReadDirectoryError {
ReadDirectory_None,
ReadDirectory_InvalidPath,
ReadDirectory_NotDir,
ReadDirectory_EOF,
ReadDirectory_Unknown,
ReadDirectory_COUNT,
};
#if defined(GB_SYSTEM_WINDOWS)
ReadDirectoryError read_directory(String path, Array<FileInfo> *fi) {
GB_ASSERT(fi != nullptr);
while (path.len > 0) {
Rune end = path[path.len-1];
if (end == '/') {
path.len -= 1;
} else if (end == '\\') {
path.len -= 1;
} else {
break;
}
}
if (path.len == 0) {
return ReadDirectory_InvalidPath;
}
if (!path_is_directory(path)) {
return ReadDirectory_NotDir;
}
gbAllocator a = heap_allocator();
char *new_path = gb_alloc_array(a, char, path.len+3);
defer (gb_free(a, new_path));
gb_memmove(new_path, path.text, path.len);
gb_memmove(new_path+path.len, "/*", 2);
new_path[path.len+2] = 0;
String np = make_string(cast(u8 *)new_path, path.len+2);
String16 wstr = string_to_string16(a, np);
defer (gb_free(a, wstr.text));
WIN32_FIND_DATAW file_data = {};
HANDLE find_file = FindFirstFileW(wstr.text, &file_data);
if (find_file == INVALID_HANDLE_VALUE) {
return ReadDirectory_Unknown;
}
defer (FindClose(find_file));
array_init(fi, a, 0, 100);
do {
wchar_t *filename_w = file_data.cFileName;
i64 size = cast(i64)file_data.nFileSizeLow;
size |= (cast(i64)file_data.nFileSizeHigh) << 32;
String name = string16_to_string(a, make_string16_c(filename_w));
if (name == "." || name == "..") {
gb_free(a, name.text);
continue;
}
String filepath = {};
filepath.len = path.len+1+name.len;
filepath.text = gb_alloc_array(a, u8, filepath.len+1);
defer (gb_free(a, filepath.text));
gb_memmove(filepath.text, path.text, path.len);
gb_memmove(filepath.text+path.len, "/", 1);
gb_memmove(filepath.text+path.len+1, name.text, name.len);
FileInfo info = {};
info.name = name;
info.fullpath = path_to_full_path(a, filepath);
info.size = size;
info.is_dir = (file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
array_add(fi, info);
} while (FindNextFileW(find_file, &file_data));
return ReadDirectory_None;
}
#else
#error Implement read_directory
#endif
src/docs.cpp
+10 -10
View File
@@ -93,15 +93,15 @@ void print_declaration(AstNode *decl) {
}
void generate_documentation(Parser *parser) {
for_array(file_index, parser->files) {
AstFile *file = parser->files[file_index];
Tokenizer *tokenizer = &file->tokenizer;
String fullpath = tokenizer->fullpath;
gb_printf("%.*s\n", LIT(fullpath));
// for_array(file_index, parser->files) {
// AstFile *file = parser->files[file_index];
// Tokenizer *tokenizer = &file->tokenizer;
// String fullpath = tokenizer->fullpath;
// gb_printf("%.*s\n", LIT(fullpath));
for_array(decl_index, file->decls) {
AstNode *decl = file->decls[decl_index];
print_declaration(decl);
}
}
// for_array(decl_index, file->decls) {
// AstNode *decl = file->decls[decl_index];
// print_declaration(decl);
// }
// }
}
src/main.cpp
+7 -3
View File
@@ -12,11 +12,13 @@
#include "checker.hpp"
#include "parser.cpp"
#if 0
#include "docs.cpp"
#include "checker.cpp"
#include "ir.cpp"
#include "ir_opt.cpp"
#include "ir_print.cpp"
#endif
// NOTE(bill): 'name' is used in debugging and profiling modes
i32 system_exec_command_line_app(char *name, bool is_silent, char *fmt, ...) {
@@ -772,8 +774,9 @@ int main(int arg_count, char **arg_ptr) {
print_usage_line(0, "%s 32-bit is not yet supported", args[0]);
return 1;
}
#if 0
init_universal_scope();
#endif
// TODO(bill): prevent compiling without a linker
timings_start_section(&timings, str_lit("parse files"));
@@ -784,10 +787,11 @@ int main(int arg_count, char **arg_ptr) {
}
defer (destroy_parser(&parser));
if (parse_files(&parser, init_filename) != ParseFile_None) {
if (parse_packages(&parser, init_filename) != ParseFile_None) {
return 1;
}
#if 0
if (build_context.generate_docs) {
generate_documentation(&parser);
return 0;
@@ -1054,6 +1058,6 @@ int main(int arg_count, char **arg_ptr) {
system_exec_command_line_app("odin run", false, "%.*s", LIT(output_base));
}
#endif
#endif
return 0;
}
src/parser.cpp
+116 -42
View File
@@ -3712,17 +3712,18 @@ AstNode *parse_stmt(AstFile *f) {
Token name = expect_token(f, Token_Ident);
String tag = name.string;
if (tag == "shared_global_scope") {
if (f->curr_proc == nullptr) {
f->is_global_scope = true;
s = ast_empty_stmt(f, f->curr_token);
} else {
syntax_error(token, "You cannot use #shared_global_scope within a procedure. This must be done at the file scope");
s = ast_bad_decl(f, token, f->curr_token);
}
expect_semicolon(f, s);
return s;
} else if (tag == "bounds_check") {
// if (tag == "shared_global_scope") {
// if (f->curr_proc == nullptr) {
// f->is_global_scope = true;
// s = ast_empty_stmt(f, f->curr_token);
// } else {
// syntax_error(token, "You cannot use #shared_global_scope within a procedure. This must be done at the file scope");
// s = ast_bad_decl(f, token, f->curr_token);
// }
// expect_semicolon(f, s);
// return s;
// } else
if (tag == "bounds_check") {
s = parse_stmt(f);
s->stmt_state_flags |= StmtStateFlag_bounds_check;
if ((s->stmt_state_flags & StmtStateFlag_no_bounds_check) != 0) {
@@ -3830,6 +3831,7 @@ ParseFileError init_ast_file(AstFile *f, String fullpath, TokenPos *err_pos) {
isize init_token_cap = cast(isize)gb_max(next_pow2(cast(i64)(file_size/2ll)), 16);
array_init(&f->tokens, heap_allocator(), 0, gb_max(init_token_cap, 16));
if (err == TokenizerInit_Empty) {
Token token = {Token_EOF};
token.pos.file = fullpath;
@@ -3881,8 +3883,9 @@ void destroy_ast_file(AstFile *f) {
bool init_parser(Parser *p) {
GB_ASSERT(p != nullptr);
array_init(&p->files, heap_allocator());
map_init(&p->packages, heap_allocator());
array_init(&p->imports, heap_allocator());
array_init(&p->files, heap_allocator());
gb_mutex_init(&p->file_add_mutex);
gb_mutex_init(&p->file_decl_mutex);
return true;
@@ -3921,8 +3924,8 @@ bool try_add_import_path(Parser *p, String path, String rel_path, TokenPos pos)
}
}
ImportedFile item = {};
item.kind = ImportedFile_Normal;
ImportedPackage item = {};
item.kind = ImportedPackage_Normal;
item.path = path;
item.rel_path = rel_path;
item.pos = pos;
@@ -4164,28 +4167,33 @@ void parse_file(Parser *p, AstFile *f) {
comsume_comment_groups(f, f->prev_token);
f->package_token = expect_token(f, Token_package);
Token package_name = expect_token_after(f, Token_Ident, "package");
if (package_name.kind == Token_Ident) {
if (package_name.string == "_") {
error(package_name, "Invalid package name '_'");
} else if (f->package->kind != ImportedPackage_Runtime && package_name.string == "runtime") {
error(package_name, "Use of reserved package name '%.*s'", LIT(package_name.string));
}
}
f->package_name = package_name.string;
f->decls = parse_stmt_list(f);
parse_setup_file_decls(p, f, base_dir, f->decls);
}
ParseFileError parse_import(Parser *p, ImportedFile imported_file) {
String import_path = imported_file.path;
String import_rel_path = imported_file.rel_path;
TokenPos pos = imported_file.pos;
ParseFileError parse_imported_file(Parser *p, AstPackage *package, FileInfo *fi, TokenPos pos) {
AstFile *file = gb_alloc_item(heap_allocator(), AstFile);
file->file_kind = imported_file.kind;
if (file->file_kind == ImportedFile_Shared) {
file->is_global_scope = true;
}
file->package = package;
TokenPos err_pos = {0};
ParseFileError err = init_ast_file(file, import_path, &err_pos);
ParseFileError err = init_ast_file(file, fi->fullpath, &err_pos);
if (err != ParseFile_None) {
if (err == ParseFile_EmptyFile) {
if (import_path == p->init_fullpath) {
if (fi->fullpath == p->init_fullpath) {
gb_printf_err("Initial file is empty - %.*s\n", LIT(p->init_fullpath));
gb_exit(1);
}
@@ -4195,7 +4203,7 @@ ParseFileError parse_import(Parser *p, ImportedFile imported_file) {
if (pos.line != 0) {
gb_printf_err("%.*s(%td:%td) ", LIT(pos.file), pos.line, pos.column);
}
gb_printf_err("Failed to parse file: %.*s\n\t", LIT(import_rel_path));
gb_printf_err("Failed to parse file: %.*s\n\t", LIT(fi->name));
switch (err) {
case ParseFile_WrongExtension:
gb_printf_err("Invalid file extension: File must have the extension '.odin'");
@@ -4207,7 +4215,7 @@ ParseFileError parse_import(Parser *p, ImportedFile imported_file) {
gb_printf_err("File permissions problem");
break;
case ParseFile_NotFound:
gb_printf_err("File cannot be found ('%.*s')", LIT(import_path));
gb_printf_err("File cannot be found ('%.*s')", LIT(fi->fullpath));
break;
case ParseFile_InvalidToken:
gb_printf_err("Invalid token found in file at (%td:%td)", err_pos.line, err_pos.column);
@@ -4230,8 +4238,17 @@ skip:
parse_file(p, file);
gb_mutex_lock(&p->file_add_mutex);
file->id = imported_file.index;
// file->id = imported_package.index;
HashKey key = hash_string(fi->fullpath);
map_set(&package->files, key, file);
array_add(&p->files, file);
if (package->name.len == 0) {
package->name = file->package_name;
} else if (file->tokens.count > 0 && package->name != file->package_name) {
error(file->package_token, "Different package name, expected '%.*s', got '%.*s'", LIT(package->name), LIT(file->package_name));
}
p->total_line_count += file->tokenizer.line_count;
p->total_token_count += file->tokens.count;
gb_mutex_unlock(&p->file_add_mutex);
@@ -4239,12 +4256,74 @@ skip:
return ParseFile_None;
}
ParseFileError parse_import(Parser *p, ImportedPackage imported_package) {
String import_path = imported_package.path;
String import_rel_path = imported_package.rel_path;
TokenPos pos = imported_package.pos;
HashKey path_key = hash_string(import_path);
if (map_get(&p->packages, path_key) != nullptr) {
return ParseFile_None;
}
Array<FileInfo> list = {};
ReadDirectoryError rd_err = read_directory(import_path, &list);
defer (array_free(&list));
if (rd_err != ReadDirectory_EOF && rd_err != ReadDirectory_None && pos.line != 0) {
gb_printf_err("%.*s(%td:%td) ", LIT(pos.file), pos.line, pos.column);
}
switch (rd_err) {
case ReadDirectory_InvalidPath:
gb_printf_err("Invalid path: %.*s\n", LIT(import_rel_path));
gb_mutex_lock(&global_error_collector.mutex);
global_error_collector.count++;
gb_mutex_unlock(&global_error_collector.mutex);
return ParseFile_InvalidFile;
case ReadDirectory_NotDir:
gb_printf_err("Expected a directory for a package, got a file: %.*s\n", LIT(import_rel_path));
gb_mutex_lock(&global_error_collector.mutex);
global_error_collector.count++;
gb_mutex_unlock(&global_error_collector.mutex);
return ParseFile_InvalidFile;
case ReadDirectory_Unknown:
gb_printf_err("Unknown error whilst reading directory");
gb_mutex_lock(&global_error_collector.mutex);
global_error_collector.count++;
gb_mutex_unlock(&global_error_collector.mutex);
return ParseFile_InvalidFile;
case ReadDirectory_EOF:
break;
}
AstPackage *package = gb_alloc_item(heap_allocator(), AstPackage);
package->kind = imported_package.kind;
package->fullpath = import_path;
map_init(&package->files, heap_allocator());
// TODO(bill): Fix concurrency
for_array(i, list) {
FileInfo *fi = &list[i];
if (string_ends_with(fi->name, str_lit(".odin"))) {
ParseFileError err = parse_imported_file(p, package, fi, pos);
if (err != ParseFile_None) {
return err;
}
}
}
return ParseFile_None;
}
GB_THREAD_PROC(parse_worker_file_proc) {
if (thread == nullptr) return 0;
auto *p = cast(Parser *)thread->user_data;
isize index = thread->user_index;
ImportedFile imported_file = p->imports[index];
ParseFileError err = parse_import(p, imported_file);
ImportedPackage imported_package = p->imports[index];
ParseFileError err = parse_import(p, imported_package);
return cast(isize)err;
}
@@ -4254,29 +4333,24 @@ struct ParserThreadWork {
isize import_index;
};
ParseFileError parse_files(Parser *p, String init_filename) {
ParseFileError parse_packages(Parser *p, String init_filename) {
GB_ASSERT(init_filename.text[init_filename.len] == 0);
char *fullpath_str = gb_path_get_full_name(heap_allocator(), cast(char *)&init_filename[0]);
String init_fullpath = string_trim_whitespace(make_string_c(fullpath_str));
TokenPos init_pos = {};
ImportedFile init_imported_file = {ImportedFile_Init, init_fullpath, init_fullpath, init_pos};
ImportedPackage init_imported_package = {ImportedPackage_Init, init_fullpath, init_fullpath, init_pos};
isize shared_file_count = 0;
if (!build_context.generate_docs) {
String s = get_fullpath_core(heap_allocator(), str_lit("_preload.odin"));
ImportedFile runtime_file = {ImportedFile_Shared, s, s, init_pos};
array_add(&p->imports, runtime_file);
shared_file_count++;
}
if (!build_context.generate_docs) {
String s = get_fullpath_core(heap_allocator(), str_lit("_soft_numbers.odin"));
ImportedFile runtime_file = {ImportedFile_Shared, s, s, init_pos};
array_add(&p->imports, runtime_file);
String s = get_fullpath_core(heap_allocator(), str_lit("runtime"));
ImportedPackage runtime_package = {ImportedPackage_Runtime, s, s, init_pos};
array_add(&p->imports, runtime_package);
shared_file_count++;
}
array_add(&p->imports, init_imported_file);
array_add(&p->imports, init_imported_package);
p->init_fullpath = init_fullpath;
// IMPORTANT TODO(bill): Figure out why this doesn't work on *nix sometimes
src/parser.hpp
+33 -21
View File
@@ -3,6 +3,8 @@ struct Scope;
struct Type;
struct Entity;
struct DeclInfo;
struct AstFile;
struct AstPackage;
enum ParseFileError {
@@ -23,23 +25,23 @@ struct CommentGroup {
};
enum ImportedFileKind {
ImportedFile_Normal,
ImportedFile_Shared,
ImportedFile_Init,
enum ImportedPackageKind {
ImportedPackage_Normal,
ImportedPackage_Runtime,
ImportedPackage_Init,
};
struct ImportedFile {
ImportedFileKind kind;
String path;
String rel_path;
TokenPos pos; // import
isize index;
struct ImportedPackage {
ImportedPackageKind kind;
String path;
String rel_path;
TokenPos pos; // import
isize index;
};
struct AstFile {
isize id;
AstPackage * package;
// isize id;
String fullpath;
gbArena arena;
Tokenizer tokenizer;
@@ -47,6 +49,8 @@ struct AstFile {
isize curr_token_index;
Token curr_token;
Token prev_token; // previous non-comment
Token package_token;
String package_name;
// >= 0: In Expression
// < 0: In Control Clause
@@ -58,8 +62,6 @@ struct AstFile {
isize when_level;
Array<AstNode *> decls;
ImportedFileKind file_kind;
bool is_global_scope;
Array<AstNode *> imports_and_exports; // 'import' 'using import' 'export'
@@ -81,14 +83,23 @@ struct AstFile {
};
struct AstPackage {
ImportedPackageKind kind;
String name;
String fullpath;
Map<AstFile *> files; // Key: String (names)
};
struct Parser {
String init_fullpath;
Array<AstFile *> files;
Array<ImportedFile> imports;
isize total_token_count;
isize total_line_count;
gbMutex file_add_mutex;
gbMutex file_decl_mutex;
String init_fullpath;
Map<AstPackage *> packages; // Key: String (fullpath)
Array<AstFile *> files;
Array<ImportedPackage> imports;
isize total_token_count;
isize total_line_count;
gbMutex file_add_mutex;
gbMutex file_decl_mutex;
};
enum ProcInlining {
@@ -526,3 +537,4 @@ gb_inline bool is_ast_node_when_stmt(AstNode *node) {
return node->kind == AstNode_WhenStmt;
}
src/string.cpp
+4
View File
@@ -82,6 +82,10 @@ gb_inline String make_string_c(char *text) {
return make_string(cast(u8 *)cast(void *)text, gb_strlen(text));
}
gb_inline String16 make_string16_c(wchar_t *text) {
return make_string16(text, string16_len(text));
}
String substring(String const &s, isize lo, isize hi) {
isize max = s.len;
GB_ASSERT_MSG(lo <= hi && hi <= max, "%td..%td..%td", lo, hi, max);
src/tokenizer.cpp
+1
View File
@@ -85,6 +85,7 @@ TOKEN_KIND(Token__KeywordBegin, ""), \
TOKEN_KIND(Token_import, "import"), \
TOKEN_KIND(Token_export, "export"), \
TOKEN_KIND(Token_foreign, "foreign"), \
TOKEN_KIND(Token_package, "package"), \
TOKEN_KIND(Token_type, "type"), \
TOKEN_KIND(Token_when, "when"), \
TOKEN_KIND(Token_if, "if"), \