From b967ae2739735128db983e965bfd13a8d3233d9f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 12:21:23 +0100 Subject: [PATCH 01/13] Change internal map indices to use a distinct `uint` rather than just `int` --- core/runtime/core.odin | 2 +- core/runtime/core_builtin.odin | 2 +- core/runtime/dynamic_map_internal.odin | 50 +++++++++++++------------- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/core/runtime/core.odin b/core/runtime/core.odin index 0310aff6d..81cce8caf 100644 --- a/core/runtime/core.odin +++ b/core/runtime/core.odin @@ -394,7 +394,7 @@ Raw_Dynamic_Array :: struct { } Raw_Map :: struct { - hashes: []int, + hashes: []Map_Index, entries: Raw_Dynamic_Array, } diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 4c736b6f6..92b3e9677 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -289,7 +289,7 @@ clear_map :: proc "contextless" (m: ^$T/map[$K]$V) { entries := (^Raw_Dynamic_Array)(&raw_map.entries) entries.len = 0 for _, i in raw_map.hashes { - raw_map.hashes[i] = -1 + raw_map.hashes[i] = MAP_SENTINEL } } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index d2aaa49f4..ebefe7813 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -24,17 +24,18 @@ __get_map_hash_from_entry :: proc "contextless" (h: Map_Header, entry: ^Map_Entr return } - +Map_Index :: distinct uint +MAP_SENTINEL :: ~Map_Index(0) Map_Find_Result :: struct { - hash_index: int, - entry_prev: int, - entry_index: int, + hash_index: Map_Index, + entry_prev: Map_Index, + entry_index: Map_Index, } Map_Entry_Header :: struct { hash: uintptr, - next: int, + next: Map_Index, /* key: Key_Value, value: Value_Type, @@ -207,16 +208,16 @@ __slice_resize :: proc(array_: ^$T/[]$E, new_count: int, allocator: Allocator, l __dynamic_map_reset_entries :: proc(using header: Map_Header, loc := #caller_location) { for i in 0.. rawptr { } __dynamic_map_set :: proc(h: Map_Header, hash: Map_Hash, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check { - index: int + index: Map_Index if len(h.m.hashes) == 0 { __dynamic_map_reserve(h, INITIAL_MAP_CAP, loc) @@ -345,11 +346,11 @@ __dynamic_map_hash_equal :: proc "contextless" (h: Map_Header, a, b: Map_Hash) - } __dynamic_map_find :: proc(using h: Map_Header, hash: Map_Hash) -> Map_Find_Result #no_bounds_check { - fr := Map_Find_Result{-1, -1, -1} + fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL} if n := uintptr(len(m.hashes)); n != 0 { - fr.hash_index = int(hash.hash & (n-1)) + fr.hash_index = Map_Index(hash.hash & (n-1)) fr.entry_index = m.hashes[fr.hash_index] - for fr.entry_index >= 0 { + for fr.entry_index != MAP_SENTINEL { entry := __dynamic_map_get_entry(h, fr.entry_index) entry_hash := __get_map_hash_from_entry(h, entry) if __dynamic_map_hash_equal(h, entry_hash, hash) { @@ -364,28 +365,28 @@ __dynamic_map_find :: proc(using h: Map_Header, hash: Map_Hash) -> Map_Find_Resu return fr } -__dynamic_map_add_entry :: proc(using h: Map_Header, hash: Map_Hash, loc := #caller_location) -> int { - prev := m.entries.len - c := __dynamic_array_append_nothing(&m.entries, entry_size, entry_align, loc) +__dynamic_map_add_entry :: proc(using h: Map_Header, hash: Map_Hash, loc := #caller_location) -> Map_Index { + prev := Map_Index(m.entries.len) + c := Map_Index(__dynamic_array_append_nothing(&m.entries, entry_size, entry_align, loc)) if c != prev { end := __dynamic_map_get_entry(h, c-1) end.hash = hash.hash mem_copy(rawptr(uintptr(end) + key_offset), hash.key_ptr, key_size) - end.next = -1 + end.next = MAP_SENTINEL } return prev } __dynamic_map_delete_key :: proc(using h: Map_Header, hash: Map_Hash) { fr := __dynamic_map_find(h, hash) - if fr.entry_index >= 0 { + if fr.entry_index != MAP_SENTINEL { __dynamic_map_erase(h, fr) } } -__dynamic_map_get_entry :: proc(using h: Map_Header, index: int) -> ^Map_Entry_Header { +__dynamic_map_get_entry :: proc(using h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { // assert(0 <= index && index < m.entries.len) - return (^Map_Entry_Header)(uintptr(m.entries.data) + uintptr(index*entry_size)) + return (^Map_Entry_Header)(uintptr(m.entries.data) + uintptr(index*Map_Index(entry_size))) } __dynamic_map_copy_entry :: proc(h: Map_Header, new, old: ^Map_Entry_Header) { @@ -393,23 +394,24 @@ __dynamic_map_copy_entry :: proc(h: Map_Header, new, old: ^Map_Entry_Header) { } __dynamic_map_erase :: proc(using h: Map_Header, fr: Map_Find_Result) #no_bounds_check { - if fr.entry_prev < 0 { + if fr.entry_prev == MAP_SENTINEL { m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next } else { prev := __dynamic_map_get_entry(h, fr.entry_prev) curr := __dynamic_map_get_entry(h, fr.entry_index) prev.next = curr.next } - if fr.entry_index == m.entries.len-1 { + last_index := Map_Index(m.entries.len-1) + if fr.entry_index == last_index { // NOTE(bill): No need to do anything else, just pop } else { old := __dynamic_map_get_entry(h, fr.entry_index) - end := __dynamic_map_get_entry(h, m.entries.len-1) + end := __dynamic_map_get_entry(h, last_index) __dynamic_map_copy_entry(h, old, end) old_hash := __get_map_hash_from_entry(h, old) - if last := __dynamic_map_find(h, old_hash); last.entry_prev >= 0 { + if last := __dynamic_map_find(h, old_hash); last.entry_prev != MAP_SENTINEL { last_entry := __dynamic_map_get_entry(h, last.entry_prev) last_entry.next = fr.entry_index } else { From 0428d5ae2e8fc3e814a968698a6550e86aea9d5c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 12:27:34 +0100 Subject: [PATCH 02/13] Catch missing areas of `Map_Index` usage --- core/runtime/dynamic_map_internal.odin | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index ebefe7813..4f1311dff 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -262,7 +262,7 @@ __dynamic_map_rehash :: proc(using header: Map_Header, new_count: int, loc := #c __dynamic_map_get :: proc(h: Map_Header, hash: Map_Hash) -> rawptr { index := __dynamic_map_find(h, hash).entry_index - if index >= 0 { + if index != MAP_SENTINEL { data := uintptr(__dynamic_map_get_entry(h, index)) return rawptr(data + h.value_offset) } @@ -270,7 +270,7 @@ __dynamic_map_get :: proc(h: Map_Header, hash: Map_Hash) -> rawptr { } __dynamic_map_set :: proc(h: Map_Header, hash: Map_Hash, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check { - index: Map_Index + index := MAP_SENTINEL if len(h.m.hashes) == 0 { __dynamic_map_reserve(h, INITIAL_MAP_CAP, loc) @@ -278,14 +278,14 @@ __dynamic_map_set :: proc(h: Map_Header, hash: Map_Hash, value: rawptr, loc := # } fr := __dynamic_map_find(h, hash) - if fr.entry_index >= 0 { + if fr.entry_index != MAP_SENTINEL { index = fr.entry_index } else { index = __dynamic_map_add_entry(h, hash, loc) - if fr.entry_prev >= 0 { + if fr.entry_prev != MAP_SENTINEL { entry := __dynamic_map_get_entry(h, fr.entry_prev) entry.next = index - } else if fr.hash_index >= 0 { + } else if fr.hash_index != MAP_SENTINEL { h.m.hashes[fr.hash_index] = index } else { return nil From 7840c1b89f3e6a7b3d962e6f5e96aab77a3ddc3b Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 12:48:12 +0100 Subject: [PATCH 03/13] Change `__dynamic_map_get` and `__dynamic_map_set` to use separate parameters rather than take a singular struct --- core/runtime/dynamic_map_internal.odin | 21 ++++++++----- src/llvm_backend.cpp | 25 +++++++--------- src/llvm_backend.hpp | 2 +- src/llvm_backend_expr.cpp | 12 ++------ src/llvm_backend_general.cpp | 41 +++++++++++++------------- 5 files changed, 48 insertions(+), 53 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 4f1311dff..3c4b60938 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -18,10 +18,9 @@ __get_map_hash :: proc "contextless" (k: ^$K) -> (map_hash: Map_Hash) { return } -__get_map_hash_from_entry :: proc "contextless" (h: Map_Header, entry: ^Map_Entry_Header) -> (hash: Map_Hash) { +__get_map_hash_from_entry :: proc "contextless" (h: Map_Header, entry: ^Map_Entry_Header, hash: ^Map_Hash) { hash.hash = entry.hash hash.key_ptr = rawptr(uintptr(entry) + h.key_offset) - return } Map_Index :: distinct uint @@ -213,7 +212,8 @@ __dynamic_map_reset_entries :: proc(using header: Map_Header, loc := #caller_loc for i in 0.. rawptr { - index := __dynamic_map_find(h, hash).entry_index +// USED INTERNALLY BY THE COMPILER +__dynamic_map_get :: proc(h: Map_Header, key_hash: uintptr, key_ptr: rawptr) -> rawptr { + index := __dynamic_map_find(h, {key_hash, key_ptr}).entry_index if index != MAP_SENTINEL { data := uintptr(__dynamic_map_get_entry(h, index)) return rawptr(data + h.value_offset) @@ -269,7 +270,9 @@ __dynamic_map_get :: proc(h: Map_Header, hash: Map_Hash) -> rawptr { return nil } -__dynamic_map_set :: proc(h: Map_Header, hash: Map_Hash, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check { +// USED INTERNALLY BY THE COMPILER +__dynamic_map_set :: proc(h: Map_Header, key_hash: uintptr, key_ptr: rawptr, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check { + hash := Map_Hash{key_hash, key_ptr} index := MAP_SENTINEL if len(h.m.hashes) == 0 { @@ -352,7 +355,8 @@ __dynamic_map_find :: proc(using h: Map_Header, hash: Map_Hash) -> Map_Find_Resu fr.entry_index = m.hashes[fr.hash_index] for fr.entry_index != MAP_SENTINEL { entry := __dynamic_map_get_entry(h, fr.entry_index) - entry_hash := __get_map_hash_from_entry(h, entry) + entry_hash: Map_Hash + __get_map_hash_from_entry(h, entry, &entry_hash) if __dynamic_map_hash_equal(h, entry_hash, hash) { return fr } @@ -409,7 +413,8 @@ __dynamic_map_erase :: proc(using h: Map_Header, fr: Map_Find_Result) #no_bounds end := __dynamic_map_get_entry(h, last_index) __dynamic_map_copy_entry(h, old, end) - old_hash := __get_map_hash_from_entry(h, old) + old_hash: Map_Hash + __get_map_hash_from_entry(h, old, &old_hash) if last := __dynamic_map_find(h, old_hash); last.entry_prev != MAP_SENTINEL { last_entry := __dynamic_map_get_entry(h, last.entry_prev) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index aa901d22f..126bcef11 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -595,14 +595,12 @@ lbValue lb_const_hash(lbModule *m, lbValue key, Type *key_type) { return hashed_key; } -lbValue lb_gen_map_hash(lbProcedure *p, lbValue key, Type *key_type) { - lbAddr v = lb_add_local_generated(p, t_map_hash, true); - lbValue vp = lb_addr_get_ptr(p, v); - key = lb_emit_conv(p, key, key_type); - +lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue *key_ptr_) { lbValue key_ptr = lb_address_from_load_or_generate_local(p, key); key_ptr = lb_emit_conv(p, key_ptr, t_rawptr); + if (key_ptr_) *key_ptr_ = key_ptr; + lbValue hashed_key = lb_const_hash(p->module, key, key_type); if (hashed_key.value == nullptr) { lbValue hasher = lb_get_hasher_proc_for_type(p->module, key_type); @@ -613,10 +611,7 @@ lbValue lb_gen_map_hash(lbProcedure *p, lbValue key, Type *key_type) { hashed_key = lb_emit_call(p, hasher, args); } - lb_emit_store(p, lb_emit_struct_ep(p, vp, 0), hashed_key); - lb_emit_store(p, lb_emit_struct_ep(p, vp, 1), key_ptr); - - return lb_addr_load(p, v); + return hashed_key; } void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbAddr addr, Type *map_type, @@ -625,17 +620,19 @@ void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbAddr addr, Type *map_ GB_ASSERT(map_type->kind == Type_Map); lbValue h = lb_gen_map_header(p, addr.addr, map_type); - lbValue key = lb_gen_map_hash(p, map_key, map_type->Map.key); + lbValue key_ptr = {}; + lbValue key_hash = lb_gen_map_key_hash(p, map_key, map_type->Map.key, &key_ptr); lbValue v = lb_emit_conv(p, map_value, map_type->Map.value); lbAddr value_addr = lb_add_local_generated(p, v.type, false); lb_addr_store(p, value_addr, v); - auto args = array_make(permanent_allocator(), 4); + auto args = array_make(permanent_allocator(), 5); args[0] = h; - args[1] = key; - args[2] = lb_emit_conv(p, value_addr.addr, t_rawptr); - args[3] = lb_emit_source_code_location(p, node); + args[1] = key_hash; + args[2] = key_ptr; + args[3] = lb_emit_conv(p, value_addr.addr, t_rawptr); + args[4] = lb_emit_source_code_location(p, node); lb_emit_runtime_call(p, "__dynamic_map_set", args); } diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index 79f0f37e7..e5bb9455f 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -444,7 +444,7 @@ String lb_get_const_string(lbModule *m, lbValue value); lbValue lb_generate_local_array(lbProcedure *p, Type *elem_type, i64 count, bool zero_init=true); lbValue lb_generate_global_array(lbModule *m, Type *elem_type, i64 count, String prefix, i64 id); lbValue lb_gen_map_header(lbProcedure *p, lbValue map_val_ptr, Type *map_type); -lbValue lb_gen_map_hash(lbProcedure *p, lbValue key, Type *key_type); +lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue *key_ptr_); void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbAddr addr, Type *map_type, lbValue map_key, lbValue map_value, Ast *node); lbValue lb_find_procedure_value_from_entity(lbModule *m, Entity *e); diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 7c92c517c..3ab73a27b 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -1423,15 +1423,9 @@ lbValue lb_build_binary_expr(lbProcedure *p, Ast *expr) { switch (rt->kind) { case Type_Map: { - lbValue addr = lb_address_from_load_or_generate_local(p, right); - lbValue h = lb_gen_map_header(p, addr, rt); - lbValue key = lb_gen_map_hash(p, left, rt->Map.key); - - auto args = array_make(permanent_allocator(), 2); - args[0] = h; - args[1] = key; - - lbValue ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); + lbValue map_ptr = lb_address_from_load_or_generate_local(p, right); + lbValue key = left; + lbValue ptr = lb_internal_dynamic_map_get_ptr(p, map_ptr, key); if (be->op.kind == Token_in) { return lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_NotEq, ptr), t_bool); } else { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 1f8fccdcb..55b09cbfc 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -383,6 +383,21 @@ Type *lb_addr_type(lbAddr const &addr) { return type_deref(addr.addr.type); } +lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key) { + Type *map_type = base_type(type_deref(map_ptr.type)); + lbValue h = lb_gen_map_header(p, map_ptr, map_type); + + lbValue key_ptr = {}; + auto args = array_make(permanent_allocator(), 3); + args[0] = h; + args[1] = lb_gen_map_key_hash(p, key, map_type->Map.key, &key_ptr); + args[2] = key_ptr; + + lbValue ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); + + return lb_emit_conv(p, ptr, alloc_type_pointer(map_type->Map.value)); +} + lbValue lb_addr_get_ptr(lbProcedure *p, lbAddr const &addr) { if (addr.addr.value == nullptr) { GB_PANIC("Illegal addr -> nullptr"); @@ -390,19 +405,8 @@ lbValue lb_addr_get_ptr(lbProcedure *p, lbAddr const &addr) { } switch (addr.kind) { - case lbAddr_Map: { - Type *map_type = base_type(addr.map.type); - lbValue h = lb_gen_map_header(p, addr.addr, map_type); - lbValue key = lb_gen_map_hash(p, addr.map.key, map_type->Map.key); - - auto args = array_make(permanent_allocator(), 2); - args[0] = h; - args[1] = key; - - lbValue ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); - - return lb_emit_conv(p, ptr, alloc_type_pointer(map_type->Map.value)); - } + case lbAddr_Map: + return lb_internal_dynamic_map_get_ptr(p, addr.addr, addr.map.key); case lbAddr_RelativePointer: { Type *rel_ptr = base_type(lb_addr_type(addr)); @@ -1059,16 +1063,11 @@ lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { } else if (addr.kind == lbAddr_Map) { - Type *map_type = base_type(addr.map.type); + Type *map_type = base_type(type_deref(addr.addr.type)); + GB_ASSERT(map_type->kind == Type_Map); lbAddr v = lb_add_local_generated(p, map_type->Map.lookup_result_type, true); - lbValue h = lb_gen_map_header(p, addr.addr, map_type); - lbValue key = lb_gen_map_hash(p, addr.map.key, map_type->Map.key); - auto args = array_make(permanent_allocator(), 2); - args[0] = h; - args[1] = key; - - lbValue ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); + lbValue ptr = lb_internal_dynamic_map_get_ptr(p, addr.addr, addr.map.key); lbValue ok = lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_NotEq, ptr), t_bool); lb_emit_store(p, lb_emit_struct_ep(p, v.addr, 1), ok); From 0ebc2add030d224d59c0b8961dfa75a1f9d13b91 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 12:56:03 +0100 Subject: [PATCH 04/13] Use a cache when generating the map header to minimize stack wastage --- src/llvm_backend.cpp | 84 ++++++++++++++++++++++----------------- src/llvm_backend.hpp | 1 + src/llvm_backend_proc.cpp | 5 ++- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 126bcef11..142ecc348 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -502,48 +502,58 @@ lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, A lbValue lb_gen_map_header(lbProcedure *p, lbValue map_val_ptr, Type *map_type) { GB_ASSERT_MSG(is_type_pointer(map_val_ptr.type), "%s", type_to_string(map_val_ptr.type)); - lbAddr h = lb_add_local_generated(p, t_map_header, false); // all the values will be initialzed later map_type = base_type(map_type); GB_ASSERT(map_type->kind == Type_Map); - Type *key_type = map_type->Map.key; - Type *val_type = map_type->Map.value; - gb_unused(val_type); + lbAddr h = {}; + lbAddr *found = map_get(&p->map_header_cache, map_val_ptr.value); + if (found != nullptr) { + h = *found; + } else { + h = lb_add_local_generated(p, t_map_header, false); // all the values will be initialzed later - GB_ASSERT(map_type->Map.entry_type->kind == Type_Struct); - map_type->Map.entry_type->cached_size = -1; - map_type->Map.entry_type->Struct.are_offsets_set = false; - - i64 entry_size = type_size_of (map_type->Map.entry_type); - i64 entry_align = type_align_of (map_type->Map.entry_type); - - i64 key_offset = type_offset_of(map_type->Map.entry_type, 2); - i64 key_size = type_size_of (map_type->Map.key); + Type *key_type = map_type->Map.key; + Type *val_type = map_type->Map.value; + gb_unused(val_type); - i64 value_offset = type_offset_of(map_type->Map.entry_type, 3); - i64 value_size = type_size_of (map_type->Map.value); - - - Type *map_header_base = base_type(t_map_header); - GB_ASSERT(map_header_base->Struct.fields.count == 8); - Type *raw_map_ptr_type = map_header_base->Struct.fields[0]->type; - LLVMValueRef const_values[8] = {}; - const_values[0] = LLVMConstNull(lb_type(p->module, raw_map_ptr_type)); - const_values[1] = lb_get_equal_proc_for_type(p->module, key_type) .value; - const_values[2] = lb_const_int(p->module, t_int, entry_size) .value; - const_values[3] = lb_const_int(p->module, t_int, entry_align) .value; - const_values[4] = lb_const_int(p->module, t_uintptr, key_offset) .value; - const_values[5] = lb_const_int(p->module, t_int, key_size) .value; - const_values[6] = lb_const_int(p->module, t_uintptr, value_offset).value; - const_values[7] = lb_const_int(p->module, t_int, value_size) .value; - - LLVMValueRef const_value = llvm_const_named_struct(p->module, t_map_header, const_values, gb_count_of(const_values)); - LLVMBuildStore(p->builder, const_value, h.addr.value); - - // NOTE(bill): Removes unnecessary allocation if split gep - lbValue gep0 = lb_emit_struct_ep(p, h.addr, 0); - lbValue m = lb_emit_conv(p, map_val_ptr, type_deref(gep0.type)); - lb_emit_store(p, gep0, m); + GB_ASSERT(map_type->Map.entry_type->kind == Type_Struct); + map_type->Map.entry_type->cached_size = -1; + map_type->Map.entry_type->Struct.are_offsets_set = false; + + i64 entry_size = type_size_of (map_type->Map.entry_type); + i64 entry_align = type_align_of (map_type->Map.entry_type); + + i64 key_offset = type_offset_of(map_type->Map.entry_type, 2); + i64 key_size = type_size_of (map_type->Map.key); + + i64 value_offset = type_offset_of(map_type->Map.entry_type, 3); + i64 value_size = type_size_of (map_type->Map.value); + + + Type *map_header_base = base_type(t_map_header); + GB_ASSERT(map_header_base->Struct.fields.count == 8); + Type *raw_map_ptr_type = map_header_base->Struct.fields[0]->type; + LLVMValueRef const_values[8] = {}; + const_values[0] = LLVMConstNull(lb_type(p->module, raw_map_ptr_type)); + const_values[1] = lb_get_equal_proc_for_type(p->module, key_type) .value; + const_values[2] = lb_const_int(p->module, t_int, entry_size) .value; + const_values[3] = lb_const_int(p->module, t_int, entry_align) .value; + const_values[4] = lb_const_int(p->module, t_uintptr, key_offset) .value; + const_values[5] = lb_const_int(p->module, t_int, key_size) .value; + const_values[6] = lb_const_int(p->module, t_uintptr, value_offset).value; + const_values[7] = lb_const_int(p->module, t_int, value_size) .value; + + LLVMValueRef const_value = llvm_const_named_struct(p->module, t_map_header, const_values, gb_count_of(const_values)); + LLVMBuildStore(p->builder, const_value, h.addr.value); + + // NOTE(bill): Removes unnecessary allocation if split gep + lbValue gep0 = lb_emit_struct_ep(p, h.addr, 0); + lbValue m = lb_emit_conv(p, map_val_ptr, type_deref(gep0.type)); + lb_emit_store(p, gep0, m); + + + map_set(&p->map_header_cache, map_val_ptr.value, h); + } return lb_addr_load(p, h); } diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index e5bb9455f..d622f3661 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -308,6 +308,7 @@ struct lbProcedure { PtrMap selector_values; PtrMap selector_addr; + PtrMap map_header_cache; }; diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 8bbbb0c56..9e8158a88 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -121,8 +121,9 @@ lbProcedure *lb_create_procedure(lbModule *m, Entity *entity, bool ignore_body) p->branch_blocks.allocator = a; p->context_stack.allocator = a; p->scope_stack.allocator = a; - map_init(&p->selector_values, a, 0); - map_init(&p->selector_addr, a, 0); + map_init(&p->selector_values, a, 0); + map_init(&p->selector_addr, a, 0); + map_init(&p->map_header_cache, a, 0); if (p->is_foreign) { lb_add_foreign_library_path(p->module, entity->Procedure.foreign_library); From 8ee6bb5d4b102158c6f4771a3104610bb8e6fd7f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 13:00:19 +0100 Subject: [PATCH 05/13] Add `contextless` where possible in dynamic_map_internal.odin --- core/runtime/dynamic_map_internal.odin | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 3c4b60938..d330d4808 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -205,7 +205,7 @@ __slice_resize :: proc(array_: ^$T/[]$E, new_count: int, allocator: Allocator, l return false } -__dynamic_map_reset_entries :: proc(using header: Map_Header, loc := #caller_location) { +__dynamic_map_reset_entries :: proc "contextless" (using header: Map_Header, loc := #caller_location) { for i in 0.. rawptr { +__dynamic_map_get :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr) -> rawptr { index := __dynamic_map_find(h, {key_hash, key_ptr}).entry_index if index != MAP_SENTINEL { data := uintptr(__dynamic_map_get_entry(h, index)) @@ -348,7 +348,7 @@ __dynamic_map_hash_equal :: proc "contextless" (h: Map_Header, a, b: Map_Hash) - return a.hash == b.hash && h.equal(a.key_ptr, b.key_ptr) } -__dynamic_map_find :: proc(using h: Map_Header, hash: Map_Hash) -> Map_Find_Result #no_bounds_check { +__dynamic_map_find :: proc "contextless" (using h: Map_Header, hash: Map_Hash) -> Map_Find_Result #no_bounds_check { fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL} if n := uintptr(len(m.hashes)); n != 0 { fr.hash_index = Map_Index(hash.hash & (n-1)) @@ -381,23 +381,22 @@ __dynamic_map_add_entry :: proc(using h: Map_Header, hash: Map_Hash, loc := #cal return prev } -__dynamic_map_delete_key :: proc(using h: Map_Header, hash: Map_Hash) { +__dynamic_map_delete_key :: proc "contextless" (using h: Map_Header, hash: Map_Hash) { fr := __dynamic_map_find(h, hash) if fr.entry_index != MAP_SENTINEL { __dynamic_map_erase(h, fr) } } -__dynamic_map_get_entry :: proc(using h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { - // assert(0 <= index && index < m.entries.len) +__dynamic_map_get_entry :: proc "contextless" (using h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { return (^Map_Entry_Header)(uintptr(m.entries.data) + uintptr(index*Map_Index(entry_size))) } -__dynamic_map_copy_entry :: proc(h: Map_Header, new, old: ^Map_Entry_Header) { +__dynamic_map_copy_entry :: proc "contextless" (h: Map_Header, new, old: ^Map_Entry_Header) { mem_copy(new, old, h.entry_size) } -__dynamic_map_erase :: proc(using h: Map_Header, fr: Map_Find_Result) #no_bounds_check { +__dynamic_map_erase :: proc "contextless" (using h: Map_Header, fr: Map_Find_Result) #no_bounds_check { if fr.entry_prev == MAP_SENTINEL { m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next } else { From bfe0ffd6e6e7940d189b12df1697591f98564d60 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 13:02:06 +0100 Subject: [PATCH 06/13] Minor clean up --- core/runtime/dynamic_map_internal.odin | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index d330d4808..7326d761d 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -334,7 +334,6 @@ ceil_to_pow2 :: proc "contextless" (n: int) -> int { } __dynamic_map_grow :: proc(using h: Map_Header, loc := #caller_location) { - // TODO(bill): Determine an efficient growing rate new_count := max(m.entries.cap * 2, INITIAL_MAP_CAP) __dynamic_map_rehash(h, new_count, loc) } @@ -344,7 +343,7 @@ __dynamic_map_full :: #force_inline proc "contextless" (using h: Map_Header) -> } -__dynamic_map_hash_equal :: proc "contextless" (h: Map_Header, a, b: Map_Hash) -> bool { +__dynamic_map_hash_equal :: #force_inline proc "contextless" (h: Map_Header, a, b: Map_Hash) -> bool { return a.hash == b.hash && h.equal(a.key_ptr, b.key_ptr) } @@ -388,7 +387,7 @@ __dynamic_map_delete_key :: proc "contextless" (using h: Map_Header, hash: Map_H } } -__dynamic_map_get_entry :: proc "contextless" (using h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { +__dynamic_map_get_entry :: #force_inline proc "contextless" (using h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { return (^Map_Entry_Header)(uintptr(m.entries.data) + uintptr(index*Map_Index(entry_size))) } From 40bcfc7c8df2e4e76a75f543c43f97cf9d050925 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 13:05:14 +0100 Subject: [PATCH 07/13] Update json/unmarshal.odin for the new `runtime.__dynamic_map_set` --- core/encoding/json/unmarshal.odin | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index 792b1edc6..85d3303c2 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -422,19 +422,17 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm delete(key, p.allocator) return err } - - hash := runtime.Map_Hash { - hash = runtime.default_hasher_string(&key, 0), - key_ptr = &key, - } - + + key_hash := runtime.default_hasher_string(&key, 0) + key_ptr := rawptr(&key) + key_cstr: cstring if reflect.is_cstring(t.key) { key_cstr = cstring(raw_data(key)) - hash.key_ptr = &key_cstr + key_ptr = &key_cstr } - set_ptr := runtime.__dynamic_map_set(header, hash, map_backing_value.data) + set_ptr := runtime.__dynamic_map_set(header, key_hash, key_ptr, map_backing_value.data) if set_ptr == nil { delete(key, p.allocator) } From fbf036a654f54b4104f6252cac8fce6c9375daaf Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 13:11:29 +0100 Subject: [PATCH 08/13] Wrap `__dynamic_map_find` for certain cases --- core/runtime/core_builtin.odin | 8 +++----- core/runtime/dynamic_map_internal.odin | 13 +++++++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 92b3e9677..558a04bf9 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -325,8 +325,7 @@ delete_key :: proc(m: ^$T/map[$K]$V, key: K) -> (deleted_key: K, deleted_value: if m != nil { key := key h := __get_map_header(m) - hash := __get_map_hash(&key) - fr := __dynamic_map_find(h, hash) + fr := __map_find(h, &key) if fr.entry_index >= 0 { entry := __dynamic_map_get_entry(h, fr.entry_index) deleted_key = (^K)(uintptr(entry)+h.key_offset)^ @@ -674,9 +673,8 @@ shrink_dynamic_array :: proc(array: ^$T/[dynamic]$E, new_cap := -1, loc := #call map_insert :: proc(m: ^$T/map[$K]$V, key: K, value: V, loc := #caller_location) -> (ptr: ^V) { key, value := key, value h := __get_map_header(m) - hash := __get_map_hash(&key) - - data := uintptr(__dynamic_map_set(h, hash, &value, loc)) + + data := uintptr(__dynamic_map_set(h, __get_map_key_hash(&key), &key, &value, loc)) return (^V)(data + h.value_offset) } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 7326d761d..be7f73339 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -11,11 +11,9 @@ Map_Hash :: struct { key_ptr: rawptr, // address of Map_Entry_Header.key } -__get_map_hash :: proc "contextless" (k: ^$K) -> (map_hash: Map_Hash) { +__get_map_key_hash :: proc "contextless" (k: ^$K) -> uintptr { hasher := intrinsics.type_hasher_proc(K) - map_hash.key_ptr = k - map_hash.hash = hasher(k, 0) - return + return hasher(k, 0) } __get_map_hash_from_entry :: proc "contextless" (h: Map_Header, entry: ^Map_Entry_Header, hash: ^Map_Hash) { @@ -347,6 +345,13 @@ __dynamic_map_hash_equal :: #force_inline proc "contextless" (h: Map_Header, a, return a.hash == b.hash && h.equal(a.key_ptr, b.key_ptr) } + +__map_find :: proc "contextless" (h: Map_Header, key_ptr: ^$K) -> Map_Find_Result #no_bounds_check { + hash := __get_map_key_hash(key_ptr) + return __dynamic_map_find(h, {hash, key_ptr}) +} + + __dynamic_map_find :: proc "contextless" (using h: Map_Header, hash: Map_Hash) -> Map_Find_Result #no_bounds_check { fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL} if n := uintptr(len(m.hashes)); n != 0 { From 81f10f53ad08e641e2fa800757c802d8eee24f30 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 13:22:23 +0100 Subject: [PATCH 09/13] Correct `delete_key` --- core/runtime/core_builtin.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 558a04bf9..f65fd37d2 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -326,7 +326,7 @@ delete_key :: proc(m: ^$T/map[$K]$V, key: K) -> (deleted_key: K, deleted_value: key := key h := __get_map_header(m) fr := __map_find(h, &key) - if fr.entry_index >= 0 { + if fr.entry_index != MAP_SENTINEL { entry := __dynamic_map_get_entry(h, fr.entry_index) deleted_key = (^K)(uintptr(entry)+h.key_offset)^ deleted_value = (^V)(uintptr(entry)+h.value_offset)^ From 4d512c2cf647bdd46f30746cd3544632bfaa8273 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 13:40:29 +0100 Subject: [PATCH 10/13] Correct `lb_gen_map_header` initialization --- src/llvm_backend_proc.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 9e8158a88..17ed9c2a6 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -381,6 +381,8 @@ lbProcedure *lb_create_dummy_procedure(lbModule *m, String link_name, Type *type lb_add_proc_attribute_at_index(p, offset+parameter_index, "nocapture"); } + map_init(&p->map_header_cache, heap_allocator(), 0); + return p; } From c37de9459e4bf782e86e4c0687bba9aef92c58d8 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 14:46:52 +0100 Subject: [PATCH 11/13] Minor refactor of the dynamic_map_internal.odin stuff --- core/runtime/dynamic_array_internal.odin | 2 + core/runtime/dynamic_map_internal.odin | 124 +++++++++++------------ 2 files changed, 60 insertions(+), 66 deletions(-) diff --git a/core/runtime/dynamic_array_internal.odin b/core/runtime/dynamic_array_internal.odin index b6a685fcf..267ee0785 100644 --- a/core/runtime/dynamic_array_internal.odin +++ b/core/runtime/dynamic_array_internal.odin @@ -59,6 +59,8 @@ __dynamic_array_shrink :: proc(array_: rawptr, elem_size, elem_align: int, new_c return } + new_cap := new_cap + new_cap = max(new_cap, 0) old_size := array.cap * elem_size new_size := new_cap * elem_size allocator := array.allocator diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index be7f73339..dd18f6272 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -181,7 +181,7 @@ __get_map_header_runtime :: proc "contextless" (m: ^Raw_Map, ti: Type_Info_Map) } -__slice_resize :: proc(array_: ^$T/[]$E, new_count: int, allocator: Allocator, loc := #caller_location) -> bool { +__slice_resize :: proc "odin" (array_: ^$T/[]$E, new_count: int, allocator: Allocator, loc := #caller_location) -> bool { array := (^Raw_Slice)(array_) if new_count < array.len { @@ -203,59 +203,55 @@ __slice_resize :: proc(array_: ^$T/[]$E, new_count: int, allocator: Allocator, l return false } -__dynamic_map_reset_entries :: proc "contextless" (using header: Map_Header, loc := #caller_location) { - for i in 0.. (did_shrink: bool) { +__dynamic_map_shrink :: proc "odin" (h: Map_Header, cap: int, loc := #caller_location) -> (did_shrink: bool) { c := context - if m.entries.allocator.procedure != nil { - c.allocator = m.entries.allocator + if h.m.entries.allocator.procedure != nil { + c.allocator = h.m.entries.allocator } context = c - return __dynamic_array_shrink(&m.entries, entry_size, entry_align, cap, loc) -} - -__dynamic_map_rehash :: proc(using header: Map_Header, new_count: int, loc := #caller_location) { - #force_inline __dynamic_map_reserve(header, new_count, loc) + return __dynamic_array_shrink(&h.m.entries, h.entry_size, h.entry_align, cap, loc) } // USED INTERNALLY BY THE COMPILER @@ -269,8 +265,19 @@ __dynamic_map_get :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_p } // USED INTERNALLY BY THE COMPILER -__dynamic_map_set :: proc(h: Map_Header, key_hash: uintptr, key_ptr: rawptr, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check { - hash := Map_Hash{key_hash, key_ptr} +__dynamic_map_set :: proc "odin" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check { + add_entry :: proc "odin" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr, loc := #caller_location) -> Map_Index { + prev := Map_Index(h.m.entries.len) + c := Map_Index(__dynamic_array_append_nothing(&h.m.entries, h.entry_size, h.entry_align, loc)) + if c != prev { + end := __dynamic_map_get_entry(h, c-1) + end.hash = key_hash + mem_copy(rawptr(uintptr(end) + h.key_offset), key_ptr, h.key_size) + end.next = MAP_SENTINEL + } + return prev + } + index := MAP_SENTINEL if len(h.m.hashes) == 0 { @@ -278,11 +285,11 @@ __dynamic_map_set :: proc(h: Map_Header, key_hash: uintptr, key_ptr: rawptr, val __dynamic_map_grow(h, loc) } - fr := __dynamic_map_find(h, hash) + fr := __dynamic_map_find(h, {key_hash, key_ptr}) if fr.entry_index != MAP_SENTINEL { index = fr.entry_index } else { - index = __dynamic_map_add_entry(h, hash, loc) + index = add_entry(h, key_hash, key_ptr, loc) if fr.entry_prev != MAP_SENTINEL { entry := __dynamic_map_get_entry(h, fr.entry_prev) entry.next = index @@ -294,10 +301,10 @@ __dynamic_map_set :: proc(h: Map_Header, key_hash: uintptr, key_ptr: rawptr, val } e := __dynamic_map_get_entry(h, index) - e.hash = hash.hash + e.hash = key_hash key := rawptr(uintptr(e) + h.key_offset) - mem_copy(key, hash.key_ptr, h.key_size) + mem_copy(key, key_ptr, h.key_size) val := rawptr(uintptr(e) + h.value_offset) mem_copy(val, value, h.value_size) @@ -331,18 +338,14 @@ ceil_to_pow2 :: proc "contextless" (n: int) -> int { return n } -__dynamic_map_grow :: proc(using h: Map_Header, loc := #caller_location) { - new_count := max(m.entries.cap * 2, INITIAL_MAP_CAP) - __dynamic_map_rehash(h, new_count, loc) +__dynamic_map_grow :: proc "odin" (h: Map_Header, loc := #caller_location) { + new_count := max(h.m.entries.cap * 2, INITIAL_MAP_CAP) + // Rehash through Reserve + __dynamic_map_reserve(h, new_count, loc) } -__dynamic_map_full :: #force_inline proc "contextless" (using h: Map_Header) -> bool { - return int(0.75 * f64(len(m.hashes))) <= m.entries.len -} - - -__dynamic_map_hash_equal :: #force_inline proc "contextless" (h: Map_Header, a, b: Map_Hash) -> bool { - return a.hash == b.hash && h.equal(a.key_ptr, b.key_ptr) +__dynamic_map_full :: #force_inline proc "contextless" (h: Map_Header) -> bool { + return int(0.75 * f64(len(h.m.hashes))) <= h.m.entries.len } @@ -352,16 +355,17 @@ __map_find :: proc "contextless" (h: Map_Header, key_ptr: ^$K) -> Map_Find_Resul } -__dynamic_map_find :: proc "contextless" (using h: Map_Header, hash: Map_Hash) -> Map_Find_Result #no_bounds_check { +__dynamic_map_find :: proc "contextless" (h: Map_Header, hash: Map_Hash) -> Map_Find_Result #no_bounds_check { fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL} - if n := uintptr(len(m.hashes)); n != 0 { + if n := uintptr(len(h.m.hashes)); n != 0 { fr.hash_index = Map_Index(hash.hash & (n-1)) - fr.entry_index = m.hashes[fr.hash_index] + fr.entry_index = h.m.hashes[fr.hash_index] for fr.entry_index != MAP_SENTINEL { entry := __dynamic_map_get_entry(h, fr.entry_index) entry_hash: Map_Hash __get_map_hash_from_entry(h, entry, &entry_hash) - if __dynamic_map_hash_equal(h, entry_hash, hash) { + + if entry_hash.hash == hash.hash && h.equal(entry_hash.key_ptr, hash.key_ptr) { return fr } // assert(entry.next < m.entries.len) @@ -373,48 +377,36 @@ __dynamic_map_find :: proc "contextless" (using h: Map_Header, hash: Map_Hash) - return fr } -__dynamic_map_add_entry :: proc(using h: Map_Header, hash: Map_Hash, loc := #caller_location) -> Map_Index { - prev := Map_Index(m.entries.len) - c := Map_Index(__dynamic_array_append_nothing(&m.entries, entry_size, entry_align, loc)) - if c != prev { - end := __dynamic_map_get_entry(h, c-1) - end.hash = hash.hash - mem_copy(rawptr(uintptr(end) + key_offset), hash.key_ptr, key_size) - end.next = MAP_SENTINEL - } - return prev -} - -__dynamic_map_delete_key :: proc "contextless" (using h: Map_Header, hash: Map_Hash) { +__dynamic_map_delete_key :: proc "contextless" (h: Map_Header, hash: Map_Hash) { fr := __dynamic_map_find(h, hash) if fr.entry_index != MAP_SENTINEL { __dynamic_map_erase(h, fr) } } -__dynamic_map_get_entry :: #force_inline proc "contextless" (using h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { - return (^Map_Entry_Header)(uintptr(m.entries.data) + uintptr(index*Map_Index(entry_size))) +__dynamic_map_get_entry :: #force_inline proc "contextless" (h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { + return (^Map_Entry_Header)(uintptr(h.m.entries.data) + uintptr(index*Map_Index(h.entry_size))) } __dynamic_map_copy_entry :: proc "contextless" (h: Map_Header, new, old: ^Map_Entry_Header) { mem_copy(new, old, h.entry_size) } -__dynamic_map_erase :: proc "contextless" (using h: Map_Header, fr: Map_Find_Result) #no_bounds_check { +__dynamic_map_erase :: proc "contextless" (h: Map_Header, fr: Map_Find_Result) #no_bounds_check { if fr.entry_prev == MAP_SENTINEL { - m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next + h.m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next } else { prev := __dynamic_map_get_entry(h, fr.entry_prev) curr := __dynamic_map_get_entry(h, fr.entry_index) prev.next = curr.next } - last_index := Map_Index(m.entries.len-1) + last_index := Map_Index(h.m.entries.len-1) if fr.entry_index == last_index { // NOTE(bill): No need to do anything else, just pop } else { old := __dynamic_map_get_entry(h, fr.entry_index) end := __dynamic_map_get_entry(h, last_index) - __dynamic_map_copy_entry(h, old, end) + mem_copy(old, end, h.entry_size) old_hash: Map_Hash __get_map_hash_from_entry(h, old, &old_hash) @@ -423,9 +415,9 @@ __dynamic_map_erase :: proc "contextless" (using h: Map_Header, fr: Map_Find_Res last_entry := __dynamic_map_get_entry(h, last.entry_prev) last_entry.next = fr.entry_index } else { - m.hashes[last.hash_index] = fr.entry_index + h.m.hashes[last.hash_index] = fr.entry_index } } - m.entries.len -= 1 + h.m.entries.len -= 1 } From 9e3ea92829fe0f377220f38119e55c18420b58d0 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 14:59:16 +0100 Subject: [PATCH 12/13] Inline many calls and delete unused procedures --- core/runtime/core_builtin.odin | 3 +- core/runtime/dynamic_map_internal.odin | 51 ++++++++++---------------- 2 files changed, 20 insertions(+), 34 deletions(-) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index f65fd37d2..f8e39b8b2 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -296,7 +296,7 @@ clear_map :: proc "contextless" (m: ^$T/map[$K]$V) { @builtin reserve_map :: proc(m: ^$T/map[$K]$V, capacity: int, loc := #caller_location) { if m != nil { - __dynamic_map_reserve(__get_map_header(m), capacity, loc) + __dynamic_map_reserve(__get_map_header(m), uint(capacity), loc) } } @@ -334,7 +334,6 @@ delete_key :: proc(m: ^$T/map[$K]$V, key: K) -> (deleted_key: K, deleted_value: __dynamic_map_erase(h, fr) } } - return } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index dd18f6272..5dd3df001 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -214,7 +214,7 @@ __dynamic_map_reset_entries :: proc "contextless" (h: Map_Header, loc := #caller __get_map_hash_from_entry(h, entry_header, &entry_hash) entry_header.next = MAP_SENTINEL - fr := __dynamic_map_find(h, entry_hash) + fr := __dynamic_map_find(h, entry_hash.hash, entry_hash.key_ptr) if fr.entry_prev == MAP_SENTINEL { h.m.hashes[fr.hash_index] = i } else { @@ -224,7 +224,7 @@ __dynamic_map_reset_entries :: proc "contextless" (h: Map_Header, loc := #caller } } -__dynamic_map_reserve :: proc "odin" (h: Map_Header, cap: int, loc := #caller_location) { +__dynamic_map_reserve :: proc "odin" (h: Map_Header, cap: uint, loc := #caller_location) { c := context if h.m.entries.allocator.procedure != nil { c.allocator = h.m.entries.allocator @@ -234,12 +234,12 @@ __dynamic_map_reserve :: proc "odin" (h: Map_Header, cap: int, loc := #caller_lo cap := cap cap = ceil_to_pow2(cap) - __dynamic_array_reserve(&h.m.entries, h.entry_size, h.entry_align, cap, loc) + __dynamic_array_reserve(&h.m.entries, h.entry_size, h.entry_align, int(cap), loc) if h.m.entries.len*2 < len(h.m.hashes) { return } - if __slice_resize(&h.m.hashes, cap*2, h.m.entries.allocator, loc) { + if __slice_resize(&h.m.hashes, int(cap*2), h.m.entries.allocator, loc) { __dynamic_map_reset_entries(h, loc) } } @@ -256,7 +256,7 @@ __dynamic_map_shrink :: proc "odin" (h: Map_Header, cap: int, loc := #caller_loc // USED INTERNALLY BY THE COMPILER __dynamic_map_get :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr) -> rawptr { - index := __dynamic_map_find(h, {key_hash, key_ptr}).entry_index + index := __dynamic_map_find(h, key_hash, key_ptr).entry_index if index != MAP_SENTINEL { data := uintptr(__dynamic_map_get_entry(h, index)) return rawptr(data + h.value_offset) @@ -285,7 +285,7 @@ __dynamic_map_set :: proc "odin" (h: Map_Header, key_hash: uintptr, key_ptr: raw __dynamic_map_grow(h, loc) } - fr := __dynamic_map_find(h, {key_hash, key_ptr}) + fr := __dynamic_map_find(h, key_hash, key_ptr) if fr.entry_index != MAP_SENTINEL { index = fr.entry_index } else { @@ -318,13 +318,11 @@ __dynamic_map_set :: proc "odin" (h: Map_Header, key_hash: uintptr, key_ptr: raw @(private="file") -ceil_to_pow2 :: proc "contextless" (n: int) -> int { - n := n - if n <= 0 { - return 0 - } else if n <= 2 { +ceil_to_pow2 :: proc "contextless" (n: uint) -> uint { + if n <= 2 { return n } + n := n n -= 1 n |= n >> 1 n |= n >> 2 @@ -339,7 +337,7 @@ ceil_to_pow2 :: proc "contextless" (n: int) -> int { } __dynamic_map_grow :: proc "odin" (h: Map_Header, loc := #caller_location) { - new_count := max(h.m.entries.cap * 2, INITIAL_MAP_CAP) + new_count := max(uint(h.m.entries.cap) * 2, INITIAL_MAP_CAP) // Rehash through Reserve __dynamic_map_reserve(h, new_count, loc) } @@ -348,27 +346,19 @@ __dynamic_map_full :: #force_inline proc "contextless" (h: Map_Header) -> bool { return int(0.75 * f64(len(h.m.hashes))) <= h.m.entries.len } - -__map_find :: proc "contextless" (h: Map_Header, key_ptr: ^$K) -> Map_Find_Result #no_bounds_check { - hash := __get_map_key_hash(key_ptr) - return __dynamic_map_find(h, {hash, key_ptr}) -} - - -__dynamic_map_find :: proc "contextless" (h: Map_Header, hash: Map_Hash) -> Map_Find_Result #no_bounds_check { +__dynamic_map_find :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr) -> Map_Find_Result #no_bounds_check { fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL} if n := uintptr(len(h.m.hashes)); n != 0 { - fr.hash_index = Map_Index(hash.hash & (n-1)) + fr.hash_index = Map_Index(key_hash & (n-1)) fr.entry_index = h.m.hashes[fr.hash_index] for fr.entry_index != MAP_SENTINEL { entry := __dynamic_map_get_entry(h, fr.entry_index) entry_hash: Map_Hash __get_map_hash_from_entry(h, entry, &entry_hash) - if entry_hash.hash == hash.hash && h.equal(entry_hash.key_ptr, hash.key_ptr) { + if entry_hash.hash == key_hash && h.equal(entry_hash.key_ptr, key_ptr) { return fr } - // assert(entry.next < m.entries.len) fr.entry_prev = fr.entry_index fr.entry_index = entry.next @@ -377,11 +367,10 @@ __dynamic_map_find :: proc "contextless" (h: Map_Header, hash: Map_Hash) -> Map_ return fr } -__dynamic_map_delete_key :: proc "contextless" (h: Map_Header, hash: Map_Hash) { - fr := __dynamic_map_find(h, hash) - if fr.entry_index != MAP_SENTINEL { - __dynamic_map_erase(h, fr) - } +// Utility procedure used by other runtime procedures +__map_find :: proc "contextless" (h: Map_Header, key_ptr: ^$K) -> Map_Find_Result #no_bounds_check { + hash := __get_map_key_hash(key_ptr) + return __dynamic_map_find(h, hash, key_ptr) } __dynamic_map_get_entry :: #force_inline proc "contextless" (h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { @@ -401,9 +390,7 @@ __dynamic_map_erase :: proc "contextless" (h: Map_Header, fr: Map_Find_Result) # prev.next = curr.next } last_index := Map_Index(h.m.entries.len-1) - if fr.entry_index == last_index { - // NOTE(bill): No need to do anything else, just pop - } else { + if fr.entry_index != last_index { old := __dynamic_map_get_entry(h, fr.entry_index) end := __dynamic_map_get_entry(h, last_index) mem_copy(old, end, h.entry_size) @@ -411,7 +398,7 @@ __dynamic_map_erase :: proc "contextless" (h: Map_Header, fr: Map_Find_Result) # old_hash: Map_Hash __get_map_hash_from_entry(h, old, &old_hash) - if last := __dynamic_map_find(h, old_hash); last.entry_prev != MAP_SENTINEL { + if last := __dynamic_map_find(h, old_hash.hash, old_hash.key_ptr); last.entry_prev != MAP_SENTINEL { last_entry := __dynamic_map_get_entry(h, last.entry_prev) last_entry.next = fr.entry_index } else { From cd484979a840a093967dcd7076e7cc39cb900096 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 17 Sep 2022 15:09:39 +0100 Subject: [PATCH 13/13] General clean up minor with rearrangements and removing unneeded procedures --- core/runtime/dynamic_map_internal.odin | 57 ++++++++++++-------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 5dd3df001..6ca9455ef 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -11,14 +11,13 @@ Map_Hash :: struct { key_ptr: rawptr, // address of Map_Entry_Header.key } -__get_map_key_hash :: proc "contextless" (k: ^$K) -> uintptr { +__get_map_key_hash :: #force_inline proc "contextless" (k: ^$K) -> uintptr { hasher := intrinsics.type_hasher_proc(K) return hasher(k, 0) } -__get_map_hash_from_entry :: proc "contextless" (h: Map_Header, entry: ^Map_Entry_Header, hash: ^Map_Hash) { - hash.hash = entry.hash - hash.key_ptr = rawptr(uintptr(entry) + h.key_offset) +__get_map_entry_key_ptr :: #force_inline proc "contextless" (h: Map_Header, entry: ^Map_Entry_Header) -> rawptr { + return rawptr(uintptr(entry) + h.key_offset) } Map_Index :: distinct uint @@ -210,16 +209,14 @@ __dynamic_map_reset_entries :: proc "contextless" (h: Map_Header, loc := #caller for i in 0.. bool { return int(0.75 * f64(len(h.m.hashes))) <= h.m.entries.len } +__dynamic_map_find_from_entry :: proc "contextless" (h: Map_Header, e: ^Map_Entry_Header) -> Map_Find_Result #no_bounds_check { + key_ptr := __get_map_entry_key_ptr(h, e) + return __dynamic_map_find(h, e.hash, key_ptr) + +} + __dynamic_map_find :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr) -> Map_Find_Result #no_bounds_check { fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL} if n := uintptr(len(h.m.hashes)); n != 0 { @@ -353,10 +356,8 @@ __dynamic_map_find :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ fr.entry_index = h.m.hashes[fr.hash_index] for fr.entry_index != MAP_SENTINEL { entry := __dynamic_map_get_entry(h, fr.entry_index) - entry_hash: Map_Hash - __get_map_hash_from_entry(h, entry, &entry_hash) - - if entry_hash.hash == key_hash && h.equal(entry_hash.key_ptr, key_ptr) { + entry_key_ptr := __get_map_entry_key_ptr(h, entry) + if entry.hash == key_hash && h.equal(entry_key_ptr, key_ptr) { return fr } @@ -370,24 +371,20 @@ __dynamic_map_find :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ // Utility procedure used by other runtime procedures __map_find :: proc "contextless" (h: Map_Header, key_ptr: ^$K) -> Map_Find_Result #no_bounds_check { hash := __get_map_key_hash(key_ptr) - return __dynamic_map_find(h, hash, key_ptr) + return #force_inline __dynamic_map_find(h, hash, key_ptr) } __dynamic_map_get_entry :: #force_inline proc "contextless" (h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { return (^Map_Entry_Header)(uintptr(h.m.entries.data) + uintptr(index*Map_Index(h.entry_size))) } -__dynamic_map_copy_entry :: proc "contextless" (h: Map_Header, new, old: ^Map_Entry_Header) { - mem_copy(new, old, h.entry_size) -} - __dynamic_map_erase :: proc "contextless" (h: Map_Header, fr: Map_Find_Result) #no_bounds_check { - if fr.entry_prev == MAP_SENTINEL { - h.m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next - } else { + if fr.entry_prev != MAP_SENTINEL { prev := __dynamic_map_get_entry(h, fr.entry_prev) curr := __dynamic_map_get_entry(h, fr.entry_index) prev.next = curr.next + } else { + h.m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next } last_index := Map_Index(h.m.entries.len-1) if fr.entry_index != last_index { @@ -395,12 +392,10 @@ __dynamic_map_erase :: proc "contextless" (h: Map_Header, fr: Map_Find_Result) # end := __dynamic_map_get_entry(h, last_index) mem_copy(old, end, h.entry_size) - old_hash: Map_Hash - __get_map_hash_from_entry(h, old, &old_hash) - - if last := __dynamic_map_find(h, old_hash.hash, old_hash.key_ptr); last.entry_prev != MAP_SENTINEL { - last_entry := __dynamic_map_get_entry(h, last.entry_prev) - last_entry.next = fr.entry_index + last := __dynamic_map_find_from_entry(h, old) + if last.entry_prev != MAP_SENTINEL { + e := __dynamic_map_get_entry(h, last.entry_prev) + e.next = fr.entry_index } else { h.m.hashes[last.hash_index] = fr.entry_index }