Make the link order of foreign imports deterministic

src/llvm_backend.cpp

@@ -29,29 +29,46 @@ void lb_add_foreign_library_path(lbModule *m, Entity *e) {
 	GB_ASSERT(e->kind == Entity_LibraryName);
 	GB_ASSERT(e->flags & EntityFlag_Used);
 
-	for_array(i, e->LibraryName.paths) {
+	mutex_lock(&m->gen->foreign_mutex);
-		String library_path = e->LibraryName.paths[i];
+	if (!ptr_set_update(&m->gen->foreign_libraries_set, e)) {
-		if (library_path.len == 0) {
+		array_add(&m->gen->foreign_libraries, e);
-			continue;
+	}
+	mutex_unlock(&m->gen->foreign_mutex);
 }
 
-		bool ok = true;
+GB_COMPARE_PROC(foreign_library_cmp) {
-		for_array(path_index, m->foreign_library_paths) {
+	int cmp = 0;
-			String path = m->foreign_library_paths[path_index];
+	Entity *x = *(Entity **)a;
-	#if defined(GB_SYSTEM_WINDOWS)
+	Entity *y = *(Entity **)b;
-			if (str_eq_ignore_case(path, library_path)) {
+	if (x == y) {
-	#else
+		return 0;
-			if (str_eq(path, library_path)) {
+	}
-	#endif
+
-				ok = false;
+	if (x->pkg != y->pkg) {
-				break;
+		isize order_x = x->pkg ? x->pkg->order : 0;
+		isize order_y = y->pkg ? y->pkg->order : 0;
+		cmp = isize_cmp(order_x, order_y);
+		if (cmp) {
+			return cmp;
+		}
+	}
+	if (x->file != y->file) {
+		String fullpath_x = x->file ? x->file->fullpath : (String{});
+		String fullpath_y = y->file ? y->file->fullpath : (String{});
+		String file_x = filename_from_path(fullpath_x);
+		String file_y = filename_from_path(fullpath_y);
+
+		cmp = string_compare(file_x, file_y);
+		if (cmp) {
+			return cmp;
 		}
 	}
 
-		if (ok) {
+	cmp = u64_cmp(x->order_in_src, y->order_in_src);
-			array_add(&m->foreign_library_paths, library_path);
+	if (cmp) {
-		}
+		return cmp;
 	}
+	return i32_cmp(x->token.pos.offset, y->token.pos.offset);
 }
 
 void lb_set_entity_from_other_modules_linkage_correctly(lbModule *other_module, Entity *e, String const &name) {
@@ -1922,4 +1939,6 @@ void lb_generate_code(lbGenerator *gen) {
 			}
 		}
 	}
+
+	gb_sort_array(gen->foreign_libraries.data, gen->foreign_libraries.count, foreign_library_cmp);
 }

src/llvm_backend.hpp

@@ -135,7 +135,6 @@ struct lbModule {
 	u32 nested_type_name_guid;
 
 	Array<lbProcedure *> procedures_to_generate;
-	Array<String> foreign_library_paths;
 
 	lbProcedure *curr_procedure;
 
@@ -162,6 +161,10 @@ struct lbGenerator {
 
 	PtrMap<Ast *, lbProcedure *> anonymous_proc_lits; 
 
+	BlockingMutex foreign_mutex;
+	PtrSet<Entity *> foreign_libraries_set;
+	Array<Entity *>  foreign_libraries;
+
 	std::atomic<u32> global_array_index;
 	std::atomic<u32> global_generated_index;
 };

src/llvm_backend_general.cpp

@@ -67,9 +67,7 @@ void lb_init_module(lbModule *m, Checker *c) {
 	map_init(&m->equal_procs, a);
 	map_init(&m->hasher_procs, a);
 	array_init(&m->procedures_to_generate, a, 0, 1024);
-	array_init(&m->foreign_library_paths,  a, 0, 1024);
 	array_init(&m->missing_procedures_to_check, a, 0, 16);
-
 	map_init(&m->debug_values, a);
 	array_init(&m->debug_incomplete_types, a, 0, 1024);
 
@@ -126,6 +124,11 @@ bool lb_init_generator(lbGenerator *gen, Checker *c) {
 	map_init(&gen->modules_through_ctx, permanent_allocator(), gen->info->packages.entries.count*2);
 	map_init(&gen->anonymous_proc_lits, heap_allocator(), 1024);
 
+
+	mutex_init(&gen->foreign_mutex);
+	array_init(&gen->foreign_libraries,       heap_allocator(), 0, 1024);
+	ptr_set_init(&gen->foreign_libraries_set, heap_allocator(), 1024);
+
 	if (USE_SEPARATE_MODULES) {
 		for_array(i, gen->info->packages.entries) {
 			AstPackage *pkg = gen->info->packages.entries[i].value;

src/main.cpp

@@ -164,6 +164,18 @@ i32 linker_stage(lbGenerator *gen) {
 		build_context.keep_object_files = true;
 	} else {
 	#if defined(GB_SYSTEM_WINDOWS)
+		bool is_windows = true;
+	#else
+		bool is_windows = false;
+	#endif
+	#if defined(GB_SYSTEM_OSX)
+		bool is_osx = true;
+	#else
+		bool is_osx = false;
+	#endif
+
+
+		if (is_windows) {
 			String section_name = str_lit("msvc-link");
 			if (build_context.use_lld) {
 				section_name = str_lit("lld-link");
@@ -199,32 +211,45 @@ i32 linker_stage(lbGenerator *gen) {
 			string_set_init(&asm_files, heap_allocator(), 64);
 			defer (string_set_destroy(&asm_files));
 
-		for_array(j, gen->modules.entries) {
+			for_array(j, gen->foreign_libraries) {
-			lbModule *m = gen->modules.entries[j].value;
+				Entity *e = gen->foreign_libraries[j];
-			for_array(i, m->foreign_library_paths) {
+				GB_ASSERT(e->kind == Entity_LibraryName);
-				String lib = m->foreign_library_paths[i];
+				for_array(i, e->LibraryName.paths) {
+					String lib = string_trim_whitespace(e->LibraryName.paths[i]);
+					if (lib.len == 0) {
+						continue;
+					}
+					// IMPORTANT NOTE(bill): calling `string_to_lower` here is not an issue because
+					// we will never uses these strings afterwards
+					string_to_lower(&lib);
 					if (has_asm_extension(lib)) {
-					string_set_add(&asm_files, lib);
+						if (!string_set_update(&asm_files, lib)) {
-				} else {
+							String asm_file = asm_files.entries[i].value;
-					string_set_add(&libs, lib);
+							String obj_file = concatenate_strings(permanent_allocator(), asm_file, str_lit(".obj"));
-				}
-			}
-		}
 
-		for_array(i, gen->default_module.foreign_library_paths) {
+							result = system_exec_command_line_app("nasm",
-			String lib = gen->default_module.foreign_library_paths[i];
+								"\"%.*s\\bin\\nasm\\windows\\nasm.exe\" \"%.*s\" "
-			if (has_asm_extension(lib)) {
+								"-f win64 "
-				string_set_add(&asm_files, lib);
+								"-o \"%.*s\" "
-			} else {
+								"%.*s "
-				string_set_add(&libs, lib);
+								"",
-			}
+								LIT(build_context.ODIN_ROOT), LIT(asm_file),
-		}
+								LIT(obj_file),
+								LIT(build_context.extra_assembler_flags)
+							);
 
-		for_array(i, libs.entries) {
+							if (result) {
-			String lib = libs.entries[i].value;
+								return result;
+							}
+							array_add(&gen->output_object_paths, obj_file);
+						}
+					} else {
+						if (!string_set_update(&libs, lib)) {
 							lib_str = gb_string_append_fmt(lib_str, " \"%.*s\"", LIT(lib));
 						}
-
+					}
+				}
+			}
 
 			if (build_context.build_mode == BuildMode_DynamicLibrary) {
 				link_settings = gb_string_append_fmt(link_settings, " /DLL");
@@ -247,27 +272,6 @@ i32 linker_stage(lbGenerator *gen) {
 				link_settings = gb_string_append_fmt(link_settings, " /DEBUG");
 			}
 
-		for_array(i, asm_files.entries) {
-			String asm_file = asm_files.entries[i].value;
-			String obj_file = concatenate_strings(permanent_allocator(), asm_file, str_lit(".obj"));
-
-			result = system_exec_command_line_app("nasm",
-				"\"%.*s\\bin\\nasm\\windows\\nasm.exe\" \"%.*s\" "
-				"-f win64 "
-				"-o \"%.*s\" "
-				"%.*s "
-				"",
-				LIT(build_context.ODIN_ROOT), LIT(asm_file),
-				LIT(obj_file),
-				LIT(build_context.extra_assembler_flags)
-			);
-
-			if (result) {
-				return result;
-			}
-			array_add(&gen->output_object_paths, obj_file);
-		}
-
 			gbString object_files = gb_string_make(heap_allocator(), "");
 			defer (gb_string_free(object_files));
 			for_array(i, gen->output_object_paths) {
@@ -351,12 +355,14 @@ i32 linker_stage(lbGenerator *gen) {
 					return result;
 				}
 			}
-	#else
+		} else {
 			timings_start_section(timings, str_lit("ld-link"));
 
 			// NOTE(vassvik): get cwd, for used for local shared libs linking, since those have to be relative to the exe
 			char cwd[256];
+			#if !defined(GB_SYSTEM_WINDOWS)
 			getcwd(&cwd[0], 256);
+			#endif
 			//printf("%s\n", cwd);
 
 			// NOTE(vassvik): needs to add the root to the library search paths, so that the full filenames of the library
@@ -364,8 +370,19 @@ i32 linker_stage(lbGenerator *gen) {
 			gbString lib_str = gb_string_make(heap_allocator(), "-L/");
 			defer (gb_string_free(lib_str));
 
-		for_array(i, gen->default_module.foreign_library_paths) {
+			StringSet libs = {};
-			String lib = gen->default_module.foreign_library_paths[i];
+			string_set_init(&libs, heap_allocator(), 64);
+			defer (string_set_destroy(&libs));
+
+			for_array(j, gen->foreign_libraries) {
+				Entity *e = gen->foreign_libraries[j];
+				GB_ASSERT(e->kind == Entity_LibraryName);
+				for_array(i, e->LibraryName.paths) {
+					String lib = e->LibraryName.paths[i];
+					if (string_set_update(&libs, lib)) {
+						continue;
+					}
+					lib_str = gb_string_append_fmt(lib_str, " \"%.*s\"", LIT(lib));
 
 					// NOTE(zangent): Sometimes, you have to use -framework on MacOS.
 					//   This allows you to specify '-f' in a #foreign_system_library,
@@ -406,6 +423,8 @@ i32 linker_stage(lbGenerator *gen) {
 						}
 					}
 				}
+			}
+
 
 			gbString object_files = gb_string_make(heap_allocator(), "");
 			defer (gb_string_free(object_files));
@@ -486,8 +505,7 @@ i32 linker_stage(lbGenerator *gen) {
 				return result;
 			}
 
-	#if defined(GB_SYSTEM_OSX)
+			if (is_osx && build_context.ODIN_DEBUG) {
-		if (build_context.ODIN_DEBUG) {
 				// NOTE: macOS links DWARF symbols dynamically. Dsymutil will map the stubs in the exe
 				// to the symbols in the object file
 				result = system_exec_command_line_app("dsymutil", "dsymutil %.*s", LIT(output_filename));
@@ -496,9 +514,7 @@ i32 linker_stage(lbGenerator *gen) {
 					return result;
 				}
 			}
-	#endif
+		}
-
-	#endif
 	}
 
 	return result;

src/ptr_set.cpp

@@ -13,7 +13,7 @@ struct PtrSet {
 template <typename T> void ptr_set_init   (PtrSet<T> *s, gbAllocator a, isize capacity = 16);
 template <typename T> void ptr_set_destroy(PtrSet<T> *s);
 template <typename T> T    ptr_set_add    (PtrSet<T> *s, T ptr);
-template <typename T> bool ptr_set_update (PtrSet<T> *s, T ptr); // returns true if it previously existsed
+template <typename T> bool ptr_set_update (PtrSet<T> *s, T ptr); // returns true if it previously existed
 template <typename T> bool ptr_set_exists (PtrSet<T> *s, T ptr);
 template <typename T> void ptr_set_remove (PtrSet<T> *s, T ptr);
 template <typename T> void ptr_set_clear  (PtrSet<T> *s);

src/string_set.cpp

@@ -13,6 +13,7 @@ struct StringSet {
 void string_set_init   (StringSet *s, gbAllocator a, isize capacity = 16);
 void string_set_destroy(StringSet *s);
 void string_set_add    (StringSet *s, String const &str);
+bool string_set_update (StringSet *s, String const &str); // returns true if it previously existed
 bool string_set_exists (StringSet *s, String const &str);
 void string_set_remove (StringSet *s, String const &str);
 void string_set_clear  (StringSet *s);
@@ -149,6 +150,34 @@ void string_set_add(StringSet *s, String const &str) {
 	}
 }
 
+bool string_set_update(StringSet *s, String const &str) {
+	bool exists = false;
+	MapIndex index;
+	MapFindResult fr;
+	StringHashKey key = string_hash_string(str);
+	if (s->hashes.count == 0) {
+		string_set_grow(s);
+	}
+	fr = string_set__find(s, key);
+	if (fr.entry_index != MAP_SENTINEL) {
+		index = fr.entry_index;
+		exists = true;
+	} else {
+		index = string_set__add_entry(s, key);
+		if (fr.entry_prev != MAP_SENTINEL) {
+			s->entries[fr.entry_prev].next = index;
+		} else {
+			s->hashes[fr.hash_index] = index;
+		}
+	}
+	s->entries[index].value = str;
+
+	if (string_set__full(s)) {
+		string_set_grow(s);
+	}
+	return exists;
+}
+
 
 void string_set__erase(StringSet *s, MapFindResult fr) {
 	MapFindResult last;