Merge pull request #3644 from odin-lang/foreign-import-improvements

Allow `foreign import` import paths to be evaluated in the semantic phase rather than parsing

core/odin/ast/ast.odin

@@ -538,7 +538,7 @@ Foreign_Import_Decl :: struct {
 	import_tok:      tokenizer.Token,
 	name:            ^Ident,
 	collection_name: string,
-	fullpaths:       []string,
+	fullpaths:       []^Expr,
 	comment:         ^Comment_Group,
 }
 

core/odin/parser/parser.odin

@@ -1190,12 +1190,12 @@ parse_foreign_decl :: proc(p: ^Parser) -> ^ast.Decl {
 			error(p, name.pos, "illegal foreign import name: '_'")
 		}
 
-		fullpaths: [dynamic]string
+		fullpaths: [dynamic]^ast.Expr
 		if allow_token(p, .Open_Brace) {
 			for p.curr_tok.kind != .Close_Brace &&
 				p.curr_tok.kind != .EOF {
-				path := expect_token(p, .String)
+				path := parse_expr(p, false)
-				append(&fullpaths, path.text)
+				append(&fullpaths, path)
 
 				allow_token(p, .Comma) or_break
 			}
@@ -1203,7 +1203,9 @@ parse_foreign_decl :: proc(p: ^Parser) -> ^ast.Decl {
 		} else {
 			path := expect_token(p, .String)
 			reserve(&fullpaths, 1)
-			append(&fullpaths, path.text)
+			bl := ast.new(ast.Basic_Lit, path.pos, end_pos(path))
+			bl.tok = tok
+			append(&fullpaths, bl)
 		}
 
 		if len(fullpaths) == 0 {

src/checker.cpp

@@ -1283,6 +1283,7 @@ gb_internal void init_checker_info(CheckerInfo *i) {
 	mpsc_init(&i->definition_queue, a); //); // 1<<20);
 	mpsc_init(&i->required_global_variable_queue, a); // 1<<10);
 	mpsc_init(&i->required_foreign_imports_through_force_queue, a); // 1<<10);
+	mpsc_init(&i->foreign_imports_to_check_fullpaths, a); // 1<<10);
 	mpsc_init(&i->intrinsics_entry_point_usage, a); // 1<<10); // just waste some memory here, even if it probably never used
 
 	string_map_init(&i->load_directory_cache);
@@ -1307,6 +1308,7 @@ gb_internal void destroy_checker_info(CheckerInfo *i) {
 	mpsc_destroy(&i->definition_queue);
 	mpsc_destroy(&i->required_global_variable_queue);
 	mpsc_destroy(&i->required_foreign_imports_through_force_queue);
+	mpsc_destroy(&i->foreign_imports_to_check_fullpaths);
 
 	map_destroy(&i->objc_msgSend_types);
 	string_map_destroy(&i->load_file_cache);
@@ -4874,6 +4876,83 @@ gb_internal DECL_ATTRIBUTE_PROC(foreign_import_decl_attribute) {
 	return false;
 }
 
+gb_internal void check_foreign_import_fullpaths(Checker *c) {
+	CheckerContext ctx = make_checker_context(c);
+
+	UntypedExprInfoMap untyped = {};
+	defer (map_destroy(&untyped));
+
+	for (Entity *e = nullptr; mpsc_dequeue(&c->info.foreign_imports_to_check_fullpaths, &e); /**/) {
+		GB_ASSERT(e != nullptr);
+		GB_ASSERT(e->kind == Entity_LibraryName);
+		Ast *decl = e->LibraryName.decl;
+		ast_node(fl, ForeignImportDecl, decl);
+
+		AstFile *f = decl->file();
+
+		reset_checker_context(&ctx, f, &untyped);
+		ctx.collect_delayed_decls = false;
+
+		GB_ASSERT(ctx.scope == e->scope);
+
+		if (fl->fullpaths.count == 0) {
+			String base_dir = dir_from_path(decl->file()->fullpath);
+
+			auto fullpaths = array_make<String>(permanent_allocator(), 0, fl->filepaths.count);
+
+			for (Ast *fp_node : fl->filepaths) {
+				Operand op = {};
+				check_expr(&ctx, &op, fp_node);
+				if (op.mode != Addressing_Constant && op.value.kind != ExactValue_String) {
+					gbString s = expr_to_string(op.expr);
+					error(fp_node, "Expected a constant string value, got '%s'", s);
+					gb_string_free(s);
+					continue;
+				}
+				if (!is_type_string(op.type)) {
+					gbString s = type_to_string(op.type);
+					error(fp_node, "Expected a constant string value, got value of type '%s'", s);
+					gb_string_free(s);
+					continue;
+				}
+
+				String file_str = op.value.value_string;
+				file_str = string_trim_whitespace(file_str);
+
+				String fullpath = file_str;
+				if (allow_check_foreign_filepath()) {
+					String foreign_path = {};
+					bool ok = determine_path_from_string(nullptr, decl, base_dir, file_str, &foreign_path, /*use error not syntax_error*/true);
+					if (ok) {
+						fullpath = foreign_path;
+					}
+				}
+				array_add(&fullpaths, fullpath);
+			}
+			fl->fullpaths = slice_from_array(fullpaths);
+		}
+
+		for (String const &path : fl->fullpaths) {
+			String ext = path_extension(path);
+			if (str_eq_ignore_case(ext, ".c") ||
+			    str_eq_ignore_case(ext, ".cpp") ||
+			    str_eq_ignore_case(ext, ".cxx") ||
+			    str_eq_ignore_case(ext, ".h") ||
+			    str_eq_ignore_case(ext, ".hpp") ||
+			    str_eq_ignore_case(ext, ".hxx") ||
+			    false
+			) {
+				error(fl->token, "With 'foreign import', you cannot import a %.*s file/directory, you must precompile the library and link against that", LIT(ext));
+				break;
+			}
+		}
+
+		add_untyped_expressions(ctx.info, &untyped);
+
+		e->LibraryName.paths = fl->fullpaths;
+	}
+}
+
 gb_internal void check_add_foreign_import_decl(CheckerContext *ctx, Ast *decl) {
 	if (decl->state_flags & StateFlag_BeenHandled) return;
 	decl->state_flags |= StateFlag_BeenHandled;
@@ -4883,59 +4962,26 @@ gb_internal void check_add_foreign_import_decl(CheckerContext *ctx, Ast *decl) {
 	Scope *parent_scope = ctx->scope;
 	GB_ASSERT(parent_scope->flags&ScopeFlag_File);
 
-	GB_ASSERT(fl->fullpaths.count > 0);
+	String library_name = fl->library_name.string;
-	String fullpath = fl->fullpaths[0];
+	if (library_name.len == 0 && fl->fullpaths.count != 0) {
-	String library_name = path_to_entity_name(fl->library_name.string, fullpath);
+		String fullpath = fl->fullpaths[0];
-	if (is_blank_ident(library_name)) {
+		library_name = path_to_entity_name(fl->library_name.string, fullpath);
-		error(fl->token, "File name, %.*s, cannot be as a library name as it is not a valid identifier", LIT(fl->library_name.string));
+	}
+	if (library_name.len == 0 || is_blank_ident(library_name)) {
+		error(fl->token, "File name, '%.*s', cannot be as a library name as it is not a valid identifier", LIT(library_name));
 		return;
 	}
 
-	for (String const &path : fl->fullpaths) {
-		String ext = path_extension(path);
-		if (str_eq_ignore_case(ext, ".c") ||
-		    str_eq_ignore_case(ext, ".cpp") ||
-		    str_eq_ignore_case(ext, ".cxx") ||
-		    str_eq_ignore_case(ext, ".h") ||
-		    str_eq_ignore_case(ext, ".hpp") ||
-		    str_eq_ignore_case(ext, ".hxx") ||
-		    false
-		) {
-			error(fl->token, "With 'foreign import', you cannot import a %.*s file directory, you must precompile the library and link against that", LIT(ext));
-			break;
-		}
-	}
-
-
-	// if (fl->collection_name != "system") {
-	// 	char *c_str = gb_alloc_array(heap_allocator(), char, fullpath.len+1);
-	// 	defer (gb_free(heap_allocator(), c_str));
-	// 	gb_memmove(c_str, fullpath.text, fullpath.len);
-	// 	c_str[fullpath.len] = '\0';
-
-	// 	gbFile f = {};
-	// 	gbFileError file_err = gb_file_open(&f, c_str);
-	// 	defer (gb_file_close(&f));
-
-	// 	switch (file_err) {
-	// 	case gbFileError_Invalid:
-	// 		error(decl, "Invalid file or cannot be found ('%.*s')", LIT(fullpath));
-	// 		return;
-	// 	case gbFileError_NotExists:
-	// 		error(decl, "File cannot be found ('%.*s')", LIT(fullpath));
-	// 		return;
-	// 	}
-	// }
 
 	GB_ASSERT(fl->library_name.pos.line != 0);
 	fl->library_name.string = library_name;
 
 	Entity *e = alloc_entity_library_name(parent_scope, fl->library_name, t_invalid,
 	                                      fl->fullpaths, library_name);
+	e->LibraryName.decl = decl;
 	add_entity_flags_from_file(ctx, e, parent_scope);
 	add_entity(ctx, parent_scope, nullptr, e);
 
-
 	AttributeContext ac = {};
 	check_decl_attributes(ctx, fl->attributes, foreign_import_decl_attribute, &ac);
 	if (ac.require_declaration) {
@@ -4950,12 +4996,8 @@ gb_internal void check_add_foreign_import_decl(CheckerContext *ctx, Ast *decl) {
 		e->LibraryName.extra_linker_flags = extra_linker_flags;
 	}
 
-	if (has_asm_extension(fullpath)) {
+	mpsc_enqueue(&ctx->info->foreign_imports_to_check_fullpaths, e);
-		if (build_context.metrics.arch != TargetArch_amd64 && build_context.metrics.os != TargetOs_darwin) {
+
-			error(decl, "Assembly files are not yet supported on this platform: %.*s_%.*s",
-			      LIT(target_os_names[build_context.metrics.os]), LIT(target_arch_names[build_context.metrics.arch]));
-		}
-	}
 }
 
 // Returns true if a new package is present
@@ -6317,6 +6359,9 @@ gb_internal void check_parsed_files(Checker *c) {
 	TIME_SECTION("check procedure bodies");
 	check_procedure_bodies(c);
 
+	TIME_SECTION("check foreign import fullpaths");
+	check_foreign_import_fullpaths(c);
+
 	TIME_SECTION("add entities from procedure bodies");
 	check_merge_queues_into_arrays(c);
 

src/checker.hpp

@@ -414,6 +414,7 @@ struct CheckerInfo {
 	MPSCQueue<Entity *> entity_queue;
 	MPSCQueue<Entity *> required_global_variable_queue;
 	MPSCQueue<Entity *> required_foreign_imports_through_force_queue;
+	MPSCQueue<Entity *> foreign_imports_to_check_fullpaths;
 
 	MPSCQueue<Ast *> intrinsics_entry_point_usage;
 
@@ -434,6 +435,8 @@ struct CheckerInfo {
 	BlockingMutex                       load_directory_mutex;
 	StringMap<LoadDirectoryCache *>     load_directory_cache;
 	PtrMap<Ast *, LoadDirectoryCache *> load_directory_map; // Key: Ast_CallExpr *
+
+
 };
 
 struct CheckerContext {

src/entity.cpp

@@ -266,6 +266,7 @@ struct Entity {
 			Scope *scope;
 		} ImportName;
 		struct {
+			Ast *decl;
 			Slice<String> paths;
 			String name;
 			i64 priority_index;

src/parser.cpp

@@ -1284,14 +1284,16 @@ gb_internal Ast *ast_import_decl(AstFile *f, Token token, Token relpath, Token i
 	return result;
 }
 
-gb_internal Ast *ast_foreign_import_decl(AstFile *f, Token token, Array<Token> filepaths, Token library_name,
+gb_internal Ast *ast_foreign_import_decl(AstFile *f, Token token, Array<Ast *> filepaths, Token library_name,
-                             CommentGroup *docs, CommentGroup *comment) {
+                                         bool multiple_filepaths,
+                                         CommentGroup *docs, CommentGroup *comment) {
 	Ast *result = alloc_ast_node(f, Ast_ForeignImportDecl);
 	result->ForeignImportDecl.token        = token;
 	result->ForeignImportDecl.filepaths    = slice_from_array(filepaths);
 	result->ForeignImportDecl.library_name = library_name;
 	result->ForeignImportDecl.docs         = docs;
 	result->ForeignImportDecl.comment      = comment;
+	result->ForeignImportDecl.multiple_filepaths = multiple_filepaths;
 	result->ForeignImportDecl.attributes.allocator = ast_allocator(f);
 
 	return result;
@@ -4882,14 +4884,17 @@ gb_internal Ast *parse_foreign_decl(AstFile *f) {
 		if (is_blank_ident(lib_name)) {
 			syntax_error(lib_name, "Illegal foreign import name: '_'");
 		}
-		Array<Token> filepaths = {};
+		bool multiple_filepaths = false;
+
+		Array<Ast *> filepaths = {};
 		if (allow_token(f, Token_OpenBrace)) {
+			multiple_filepaths = true;
 			array_init(&filepaths, ast_allocator(f));
 
 			while (f->curr_token.kind != Token_CloseBrace &&
 			       f->curr_token.kind != Token_EOF) {
 
-				Token path = expect_token(f, Token_String);
+				Ast *path = parse_expr(f, false);
 				array_add(&filepaths, path);
 
 				if (!allow_field_separator(f)) {
@@ -4898,9 +4903,10 @@ gb_internal Ast *parse_foreign_decl(AstFile *f) {
 			}
 			expect_closing_brace_of_field_list(f);
 		} else {
-			filepaths = array_make<Token>(ast_allocator(f), 0, 1);
+			filepaths = array_make<Ast *>(ast_allocator(f), 0, 1);
 			Token path = expect_token(f, Token_String);
-			array_add(&filepaths, path);
+			Ast *lit = ast_basic_lit(f, path);
+			array_add(&filepaths, lit);
 		}
 
 		Ast *s = nullptr;
@@ -4909,9 +4915,9 @@ gb_internal Ast *parse_foreign_decl(AstFile *f) {
 			s = ast_bad_decl(f, lib_name, f->curr_token);
 		} else if (f->curr_proc != nullptr) {
 			syntax_error(lib_name, "You cannot use foreign import within a procedure. This must be done at the file scope");
-			s = ast_bad_decl(f, lib_name, filepaths[0]);
+			s = ast_bad_decl(f, lib_name, ast_token(filepaths[0]));
 		} else {
-			s = ast_foreign_import_decl(f, token, filepaths, lib_name, docs, f->line_comment);
+			s = ast_foreign_import_decl(f, token, filepaths, lib_name, multiple_filepaths, docs, f->line_comment);
 		}
 		expect_semicolon(f);
 		return s;
@@ -5648,9 +5654,19 @@ gb_internal bool is_package_name_reserved(String const &name) {
 }
 
 
-gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node, String base_dir, String const &original_string, String *path) {
+gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node, String base_dir, String const &original_string, String *path, bool use_check_errors=false) {
 	GB_ASSERT(path != nullptr);
 
+	void (*do_error)(Ast *, char const *, ...);
+	void (*do_warning)(Token const &, char const *, ...);
+
+	do_error = &syntax_error;
+	do_warning = &syntax_warning;
+	if (use_check_errors) {
+		do_error = &error;
+		do_error = &warning;
+	}
+
 	// NOTE(bill): if file_mutex == nullptr, this means that the code is used within the semantics stage
 
 	String collection_name = {};
@@ -5677,7 +5693,7 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
 
 	String file_str = {};
 	if (colon_pos == 0) {
-		syntax_error(node, "Expected a collection name");
+		do_error(node, "Expected a collection name");
 		return false;
 	}
 
@@ -5692,11 +5708,11 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
 	if (has_windows_drive) {
 		String sub_file_path = substring(file_str, 3, file_str.len);
 		if (!is_import_path_valid(sub_file_path)) {
-			syntax_error(node, "Invalid import path: '%.*s'", LIT(file_str));
+			do_error(node, "Invalid import path: '%.*s'", LIT(file_str));
 			return false;
 		}
 	} else if (!is_import_path_valid(file_str)) {
-		syntax_error(node, "Invalid import path: '%.*s'", LIT(file_str));
+		do_error(node, "Invalid import path: '%.*s'", LIT(file_str));
 		return false;
 	}
 
@@ -5718,16 +5734,16 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
 			}
 			if (replace_with_base) {
 				if (ast_file_vet_deprecated(node->file())) {
-					syntax_error(node, "import \"core:%.*s\" has been deprecated in favour of \"base:%.*s\"", LIT(file_str), LIT(file_str));
+					do_error(node, "import \"core:%.*s\" has been deprecated in favour of \"base:%.*s\"", LIT(file_str), LIT(file_str));
 				} else {
-					syntax_warning(ast_token(node), "import \"core:%.*s\" has been deprecated in favour of \"base:%.*s\"", LIT(file_str), LIT(file_str));
+					do_warning(ast_token(node), "import \"core:%.*s\" has been deprecated in favour of \"base:%.*s\"", LIT(file_str), LIT(file_str));
 				}
 			}
 		}
 
 		if (collection_name == "system") {
 			if (node->kind != Ast_ForeignImportDecl) {
-				syntax_error(node, "The library collection 'system' is restrict for 'foreign_library'");
+				do_error(node, "The library collection 'system' is restrict for 'foreign import'");
 				return false;
 			} else {
 				*path = file_str;
@@ -5735,7 +5751,7 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
 			}
 		} else if (!find_library_collection_path(collection_name, &base_dir)) {
 			// NOTE(bill): It's a naughty name
-			syntax_error(node, "Unknown library collection: '%.*s'", LIT(collection_name));
+			do_error(node, "Unknown library collection: '%.*s'", LIT(collection_name));
 			return false;
 		}
 	} else {
@@ -5759,7 +5775,7 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
 		if (collection_name == "core" || collection_name == "base") {
 			return true;
 		} else {
-			syntax_error(node, "The package '%.*s' must be imported with the 'base' library collection: 'base:%.*s'", LIT(file_str), LIT(file_str));
+			do_error(node, "The package '%.*s' must be imported with the 'base' library collection: 'base:%.*s'", LIT(file_str), LIT(file_str));
 			return false;
 		}
 	}
@@ -5844,30 +5860,29 @@ gb_internal void parse_setup_file_decls(Parser *p, AstFile *f, String const &bas
 		} else if (node->kind == Ast_ForeignImportDecl) {
 			ast_node(fl, ForeignImportDecl, node);
 
-			auto fullpaths = array_make<String>(permanent_allocator(), 0, fl->filepaths.count);
+			if (fl->filepaths.count == 0) {
-
+				syntax_error(decls[i], "No foreign paths found");
-			for (Token const &fp : fl->filepaths) {
+				decls[i] = ast_bad_decl(f, ast_token(fl->filepaths[0]), ast_end_token(fl->filepaths[fl->filepaths.count-1]));
-				String file_str = string_trim_whitespace(string_value_from_token(f, fp));
+				goto end;
+			} else if (!fl->multiple_filepaths &&
+			           fl->filepaths.count == 1) {
+				Ast *fp = fl->filepaths[0];
+				GB_ASSERT(fp->kind == Ast_BasicLit);
+				Token fp_token = fp->BasicLit.token;
+				String file_str = string_trim_whitespace(string_value_from_token(f, fp_token));
 				String fullpath = file_str;
 				if (allow_check_foreign_filepath()) {
 					String foreign_path = {};
 					bool ok = determine_path_from_string(&p->file_decl_mutex, node, base_dir, file_str, &foreign_path);
 					if (!ok) {
-						decls[i] = ast_bad_decl(f, fp, fl->filepaths[fl->filepaths.count-1]);
+						decls[i] = ast_bad_decl(f, fp_token, fp_token);
 						goto end;
 					}
 					fullpath = foreign_path;
 				}
-				array_add(&fullpaths, fullpath);
+				fl->fullpaths = slice_make<String>(permanent_allocator(), 1);
+				fl->fullpaths[0] = fullpath;
 			}
-			if (fullpaths.count == 0) {
-				syntax_error(decls[i], "No foreign paths found");
-				decls[i] = ast_bad_decl(f, fl->filepaths[0], fl->filepaths[fl->filepaths.count-1]);
-				goto end;
-			}
-
-			fl->fullpaths = slice_from_array(fullpaths);
-
 
 		} else if (node->kind == Ast_WhenStmt) {
 			ast_node(ws, WhenStmt, node);

src/parser.hpp

@@ -631,7 +631,8 @@ AST_KIND(_DeclBegin,      "", bool) \
 	}) \
 	AST_KIND(ForeignImportDecl, "foreign import declaration", struct { \
 		Token    token;           \
-		Slice<Token> filepaths;   \
+		Slice<Ast *> filepaths;   \
+		bool multiple_filepaths;  \
 		Token    library_name;    \
 		String   collection_name; \
 		Slice<String> fullpaths;  \

src/parser_pos.cpp

@@ -278,7 +278,7 @@ Token ast_end_token(Ast *node) {
 	case Ast_ImportDecl:         return node->ImportDecl.relpath;
 	case Ast_ForeignImportDecl:
 		if (node->ForeignImportDecl.filepaths.count > 0) {
-			return node->ForeignImportDecl.filepaths[node->ForeignImportDecl.filepaths.count-1];
+			return ast_end_token(node->ForeignImportDecl.filepaths[node->ForeignImportDecl.filepaths.count-1]);
 		}
 		if (node->ForeignImportDecl.library_name.kind != Token_Invalid) {
 			return node->ForeignImportDecl.library_name;