diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 882c21e75..dc2691d80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,11 @@ jobs: cd tests/vendor make timeout-minutes: 10 + - name: Odin internals tests + run: | + cd tests/internal + make + timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 timeout-minutes: 10 @@ -87,6 +92,11 @@ jobs: cd tests/vendor make timeout-minutes: 10 + - name: Odin internals tests + run: | + cd tests/internal + make + timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 timeout-minutes: 10 @@ -146,6 +156,13 @@ jobs: cd tests\vendor call build.bat timeout-minutes: 10 + - name: Odin internals tests + shell: cmd + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat + cd tests\internal + call build.bat + timeout-minutes: 10 - name: core:math/big tests shell: cmd run: | diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 10fb4c348..1dadc8ef0 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -257,21 +257,18 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: opt_write_start(w, opt, '{') or_return if m != nil { - if info.generated_struct == nil { + if info.map_info == nil { return .Unsupported_Type } - entries := &m.entries - gs := runtime.type_info_base(info.generated_struct).variant.(runtime.Type_Info_Struct) - ed := runtime.type_info_base(gs.types[1]).variant.(runtime.Type_Info_Dynamic_Array) - entry_type := ed.elem.variant.(runtime.Type_Info_Struct) - entry_size := ed.elem_size + map_cap := uintptr(runtime.map_cap(m^)) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(m^, info.map_info) + for bucket_index in 0.. (err: name: string #partial switch info in ti.variant { - case runtime.Type_Info_String: + case runtime.Type_Info_String: switch s in a { case string: name = s case cstring: name = string(s) } opt_write_key(w, opt, name) or_return - + case: return .Unsupported_Type } } diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index 19288a294..11cae6160 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -399,12 +399,10 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm return UNSUPPORTED_TYPE } raw_map := (^mem.Raw_Map)(v.data) - if raw_map.entries.allocator.procedure == nil { - raw_map.entries.allocator = p.allocator + if raw_map.allocator.procedure == nil { + raw_map.allocator = p.allocator } - header := runtime.__get_map_header_table_runtime(t) - elem_backing := bytes_make(t.value.size, t.value.align, p.allocator) or_return defer delete(elem_backing, p.allocator) @@ -421,7 +419,6 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm return err } - key_hash := runtime.default_hasher_string(&key, 0) key_ptr := rawptr(&key) key_cstr: cstring @@ -430,7 +427,7 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm key_ptr = &key_cstr } - set_ptr := runtime.__dynamic_map_set(raw_map, header, key_hash, key_ptr, map_backing_value.data) + set_ptr := runtime.__dynamic_map_set_without_hash(raw_map, t.map_info, key_ptr, map_backing_value.data) if set_ptr == nil { delete(key, p.allocator) } diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 2047aaf75..9dd38eb29 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -2069,41 +2069,27 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { m := (^mem.Raw_Map)(v.data) if m != nil { - if info.generated_struct == nil { + if info.map_info == nil { return } - entries := &m.entries - gs := runtime.type_info_base(info.generated_struct).variant.(runtime.Type_Info_Struct) - ed := runtime.type_info_base(gs.types[1]).variant.(runtime.Type_Info_Dynamic_Array) - entry_type := ed.elem.variant.(runtime.Type_Info_Struct) - entry_size := ed.elem_size - /* - NOTE: The layout of a `map` is as follows: - - map[Key]Value - - ## Internal Layout - struct { - hashes: []int, - entries: [dynamic]struct{ - hash: uintptr, - next: int, - key: Key, - value: Value, - }, + map_cap := uintptr(runtime.map_cap(m^)) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(m^, info.map_info) + j := 0 + for bucket_index in 0.. 0 { io.write_string(fi.writer, ", ", &fi.n) } - data := uintptr(entries.data) + uintptr(i*entry_size) + if j > 0 { + io.write_string(fi.writer, ", ", &fi.n) + } + j += 1 + + key := runtime.map_cell_index_dynamic(ks, info.map_info.ks, bucket_index) + value := runtime.map_cell_index_dynamic(vs, info.map_info.vs, bucket_index) - key := data + entry_type.offsets[2] // key: Key fmt_arg(&Info{writer = fi.writer}, any{rawptr(key), info.key.id}, 'v') - io.write_string(fi.writer, "=", &fi.n) - - value := data + entry_type.offsets[3] // value: Value fmt_arg(fi, any{rawptr(value), info.value.id}, 'v') } } diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 69f77cea2..38542d2fc 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -188,6 +188,9 @@ type_field_index_of :: proc($T: typeid, $name: string) -> uintptr --- type_equal_proc :: proc($T: typeid) -> (equal: proc "contextless" (rawptr, rawptr) -> bool) where type_is_comparable(T) --- type_hasher_proc :: proc($T: typeid) -> (hasher: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr) where type_is_comparable(T) --- +type_map_info :: proc($T: typeid/map[$K]$V) -> ^runtime.Map_Info --- +type_map_cell_info :: proc($T: typeid) -> ^runtime.Map_Cell_Info --- + type_convert_variants_to_pointers :: proc($T: typeid) -> typeid where type_is_union(T) --- constant_utf16_cstring :: proc($literal: string) -> [^]u16 --- diff --git a/core/mem/alloc.odin b/core/mem/alloc.odin index 551906bed..c7a21dcd4 100644 --- a/core/mem/alloc.odin +++ b/core/mem/alloc.odin @@ -77,6 +77,14 @@ free :: proc(ptr: rawptr, allocator := context.allocator, loc := #caller_locatio return runtime.mem_free(ptr, allocator, loc) } +free_with_size :: proc(ptr: rawptr, byte_count: int, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { + if ptr == nil || allocator.procedure == nil { + return nil + } + _, err := allocator.procedure(allocator.data, .Free, 0, 0, ptr, byte_count, loc) + return err +} + free_bytes :: proc(bytes: []byte, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { return runtime.mem_free_bytes(bytes, allocator, loc) } @@ -112,22 +120,20 @@ query_info :: proc(pointer: rawptr, allocator: Allocator, loc := #caller_locatio -delete_string :: proc(str: string, allocator := context.allocator, loc := #caller_location) { - free(raw_data(str), allocator, loc) +delete_string :: proc(str: string, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { + return free_with_size(raw_data(str), len(str), allocator, loc) } -delete_cstring :: proc(str: cstring, allocator := context.allocator, loc := #caller_location) { - free((^byte)(str), allocator, loc) +delete_cstring :: proc(str: cstring, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { + return free((^byte)(str), allocator, loc) } -delete_dynamic_array :: proc(array: $T/[dynamic]$E, loc := #caller_location) { - free(raw_data(array), array.allocator, loc) +delete_dynamic_array :: proc(array: $T/[dynamic]$E, loc := #caller_location) -> Allocator_Error { + return free_with_size(raw_data(array), cap(array)*size_of(E), array.allocator, loc) } -delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #caller_location) { - free(raw_data(array), allocator, loc) +delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { + return free_with_size(raw_data(array), len(array)*size_of(E), allocator, loc) } -delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) { - raw := transmute(Raw_Map)m - delete_slice(raw.hashes, raw.entries.allocator, loc) - free(raw.entries.data, raw.entries.allocator, loc) +delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) -> Allocator_Error { + return runtime.map_free_dynamic(transmute(Raw_Map)m, runtime.map_info(T), loc) } @@ -158,8 +164,6 @@ new_clone :: proc(data: $T, allocator := context.allocator, loc := #caller_locat return nil, .Out_Of_Memory } -DEFAULT_RESERVE_CAPACITY :: 16 - make_aligned :: proc($T: typeid/[]$E, #any_int len: int, alignment: int, allocator := context.allocator, loc := #caller_location) -> (slice: T, err: Allocator_Error) { runtime.make_slice_error_loc(loc, len) data := alloc_bytes(size_of(E)*len, alignment, allocator, loc) or_return @@ -173,7 +177,7 @@ make_slice :: proc($T: typeid/[]$E, #any_int len: int, allocator := context.allo return make_aligned(T, len, align_of(E), allocator, loc) } make_dynamic_array :: proc($T: typeid/[dynamic]$E, allocator := context.allocator, loc := #caller_location) -> (T, Allocator_Error) { - return make_dynamic_array_len_cap(T, 0, DEFAULT_RESERVE_CAPACITY, allocator, loc) + return make_dynamic_array_len_cap(T, 0, 16, allocator, loc) } make_dynamic_array_len :: proc($T: typeid/[dynamic]$E, #any_int len: int, allocator := context.allocator, loc := #caller_location) -> (T, Allocator_Error) { return make_dynamic_array_len_cap(T, len, len, allocator, loc) @@ -188,12 +192,12 @@ make_dynamic_array_len_cap :: proc($T: typeid/[dynamic]$E, #any_int len: int, #a array = transmute(T)s return } -make_map :: proc($T: typeid/map[$K]$E, #any_int cap: int = DEFAULT_RESERVE_CAPACITY, allocator := context.allocator, loc := #caller_location) -> T { +make_map :: proc($T: typeid/map[$K]$E, #any_int cap: int = 1< T { runtime.make_map_expr_error_loc(loc, cap) context.allocator = allocator m: T - reserve_map(&m, cap) + reserve_map(&m, cap, loc) return m } make_multi_pointer :: proc($T: typeid/[^]$E, #any_int len: int, allocator := context.allocator, loc := #caller_location) -> (mp: T, err: Allocator_Error) { diff --git a/core/mem/allocators.odin b/core/mem/allocators.odin index 98db9eeb2..fc009621b 100644 --- a/core/mem/allocators.odin +++ b/core/mem/allocators.odin @@ -867,6 +867,10 @@ tracking_allocator_init :: proc(t: ^Tracking_Allocator, backing_allocator: Alloc t.backing = backing_allocator t.allocation_map.allocator = internals_allocator t.bad_free_array.allocator = internals_allocator + + if .Free_All in query_features(t.backing) { + t.clear_on_free_all = true + } } tracking_allocator_destroy :: proc(t: ^Tracking_Allocator) { @@ -874,6 +878,13 @@ tracking_allocator_destroy :: proc(t: ^Tracking_Allocator) { delete(t.bad_free_array) } + +tracking_allocator_clear :: proc(t: ^Tracking_Allocator) { + clear(&t.allocation_map) + clear(&t.bad_free_array) +} + + tracking_allocator :: proc(data: ^Tracking_Allocator) -> Allocator { return Allocator{ data = data, @@ -883,7 +894,7 @@ tracking_allocator :: proc(data: ^Tracking_Allocator) -> Allocator { tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, size, alignment: int, - old_memory: rawptr, old_size: int, loc := #caller_location) -> ([]byte, Allocator_Error) { + old_memory: rawptr, old_size: int, loc := #caller_location) -> (result: []byte, err: Allocator_Error) { data := (^Tracking_Allocator)(allocator_data) if mode == .Query_Info { info := (^Allocator_Query_Info)(old_memory) @@ -895,21 +906,16 @@ tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, info.pointer = nil } - return nil, nil + return } - result: []byte - err: Allocator_Error if mode == .Free && old_memory != nil && old_memory not_in data.allocation_map { append(&data.bad_free_array, Tracking_Allocator_Bad_Free_Entry{ memory = old_memory, location = loc, }) } else { - result, err = data.backing.procedure(data.backing.data, mode, size, alignment, old_memory, old_size, loc) - if err != nil { - return result, err - } + result = data.backing.procedure(data.backing.data, mode, size, alignment, old_memory, old_size, loc) or_return } result_ptr := raw_data(result) @@ -957,6 +963,6 @@ tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, unreachable() } - return result, err + return } diff --git a/core/reflect/iterator.odin b/core/reflect/iterator.odin new file mode 100644 index 000000000..9fb6489ca --- /dev/null +++ b/core/reflect/iterator.odin @@ -0,0 +1,76 @@ +package reflect + +import "core:runtime" + +iterate_array :: proc(val: any, it: ^int) -> (elem: any, index: int, ok: bool) { + if val == nil || it == nil { + return + } + + ti := type_info_base(type_info_of(val.id)) + #partial switch info in ti.variant { + case Type_Info_Pointer: + if ptr := (^rawptr)(val.data)^; ptr != nil { + return iterate_array(any{ptr, info.elem.id}, it) + } + case Type_Info_Array: + if it^ < info.count { + elem.data = rawptr(uintptr(val.data) + uintptr(it^ * info.elem_size)) + elem.id = info.elem.id + ok = true + it^ += 1 + } + case Type_Info_Slice: + array := (^runtime.Raw_Slice)(val.data) + if it^ < array.len { + elem.data = rawptr(uintptr(array.data) + uintptr(it^ * info.elem_size)) + elem.id = info.elem.id + ok = true + it^ += 1 + } + case Type_Info_Dynamic_Array: + array := (^runtime.Raw_Dynamic_Array)(val.data) + if it^ < array.len { + elem.data = rawptr(uintptr(array.data) + uintptr(it^ * info.elem_size)) + elem.id = info.elem.id + ok = true + it^ += 1 + } + } + + return +} + +iterate_map :: proc(val: any, it: ^int) -> (key, value: any, ok: bool) { + if val == nil || it == nil { + return + } + ti := type_info_base(type_info_of(val.id)) + #partial switch info in ti.variant { + case Type_Info_Pointer: + if ptr := (^rawptr)(val.data)^; ptr != nil { + return iterate_map(any{ptr, info.elem.id}, it) + } + case Type_Info_Map: + if info.map_info == nil { + break + } + rm := (^runtime.Raw_Map)(val.data) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(rm^, info.map_info) + for /**/ ; it^ < int(runtime.map_cap(rm^)); it^ += 1 { + if hash := hs[it^]; runtime.map_hash_is_valid(hash) { + key_ptr := runtime.map_cell_index_dynamic(ks, info.map_info.ks, uintptr(it^)) + value_ptr := runtime.map_cell_index_dynamic(vs, info.map_info.vs, uintptr(it^)) + + key.data = rawptr(key_ptr) + value.data = rawptr(value_ptr) + key.id = info.key.id + value.id = info.value.id + ok = true + break + } + + } + } + return +} \ No newline at end of file diff --git a/core/reflect/map.odin b/core/reflect/map.odin deleted file mode 100644 index c56173074..000000000 --- a/core/reflect/map.odin +++ /dev/null @@ -1,42 +0,0 @@ -package reflect - -import "core:runtime" -import "core:mem" -_ :: runtime -_ :: mem - -Map_Entry_Info :: struct($Key, $Value: typeid) { - hash: uintptr, - key: Key, - value: Value, -} - -map_entry_info_slice :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (entries: []Map_Entry_Info(K, V)) #no_bounds_check { - m := m - rm := (^mem.Raw_Map)(&m) - - info := type_info_base(type_info_of(M)).variant.(Type_Info_Map) - gs := type_info_base(info.generated_struct).variant.(Type_Info_Struct) - ed := type_info_base(gs.types[1]).variant.(Type_Info_Dynamic_Array) - entry_type := ed.elem.variant.(Type_Info_Struct) - key_offset := entry_type.offsets[2] - value_offset := entry_type.offsets[3] - entry_size := uintptr(ed.elem_size) - - entries = make(type_of(entries), rm.entries.len) - - data := uintptr(rm.entries.data) - for i in 0.. int { return (^runtime.Raw_Dynamic_Array)(val.data).len case Type_Info_Map: - return (^runtime.Raw_Map)(val.data).entries.len + return runtime.map_len((^runtime.Raw_Map)(val.data)^) case Type_Info_String: if a.is_cstring { @@ -305,7 +305,7 @@ capacity :: proc(val: any) -> int { return (^runtime.Raw_Dynamic_Array)(val.data).cap case Type_Info_Map: - return (^runtime.Raw_Map)(val.data).entries.cap + return runtime.map_cap((^runtime.Raw_Map)(val.data)^) } return 0 } diff --git a/core/runtime/core.odin b/core/runtime/core.odin index 385a03b71..108609f78 100644 --- a/core/runtime/core.odin +++ b/core/runtime/core.odin @@ -143,11 +143,9 @@ Type_Info_Enum :: struct { values: []Type_Info_Enum_Value, } Type_Info_Map :: struct { - key: ^Type_Info, - value: ^Type_Info, - generated_struct: ^Type_Info, - key_equal: Equal_Proc, - key_hasher: Hasher_Proc, + key: ^Type_Info, + value: ^Type_Info, + map_info: ^Map_Info, } Type_Info_Bit_Set :: struct { elem: ^Type_Info, @@ -394,9 +392,32 @@ Raw_Dynamic_Array :: struct { allocator: Allocator, } +// The raw, type-erased representation of a map. +// +// 32-bytes on 64-bit +// 16-bytes on 32-bit Raw_Map :: struct { - hashes: []Map_Index, - entries: Raw_Dynamic_Array, + // A single allocation spanning all keys, values, and hashes. + // { + // k: Map_Cell(K) * (capacity / ks_per_cell) + // v: Map_Cell(V) * (capacity / vs_per_cell) + // h: Map_Cell(H) * (capacity / hs_per_cell) + // } + // + // The data is allocated assuming 64-byte alignment, meaning the address is + // always a multiple of 64. This means we have 6 bits of zeros in the pointer + // to store the capacity. We can store a value as large as 2^6-1 or 63 in + // there. This conveniently is the maximum log2 capacity we can have for a map + // as Odin uses signed integers to represent capacity. + // + // Since the hashes are backed by Map_Hash, which is just a 64-bit unsigned + // integer, the cell structure for hashes is unnecessary because 64/8 is 8 and + // requires no padding, meaning it can be indexed as a regular array of + // Map_Hash directly, though for consistency sake it's written as if it were + // an array of Map_Cell(Map_Hash). + data: uintptr, // 8-bytes on 64-bits, 4-bytes on 32-bits + len: int, // 8-bytes on 64-bits, 4-bytes on 32-bits + allocator: Allocator, // 16-bytes on 64-bits, 8-bytes on 32-bits } Raw_Any :: struct { diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index b0f4cb25c..6792734d2 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -159,20 +159,7 @@ delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #cal } @builtin delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) -> Allocator_Error { - Entry :: struct { - hash: uintptr, - next: int, - key: K, - value: V, - } - - raw := transmute(Raw_Map)m - err := delete_slice(raw.hashes, raw.entries.allocator, loc) - err1 := mem_free_with_size(raw.entries.data, raw.entries.cap*size_of(Entry), raw.entries.allocator, loc) - if err == nil { - err = err1 - } - return err + return map_free_dynamic(transmute(Raw_Map)m, map_info(T), loc) } @@ -244,12 +231,12 @@ make_dynamic_array_len_cap :: proc($T: typeid/[dynamic]$E, #any_int len: int, #a return } @(builtin) -make_map :: proc($T: typeid/map[$K]$E, #any_int cap: int = DEFAULT_RESERVE_CAPACITY, allocator := context.allocator, loc := #caller_location) -> T { - make_map_expr_error_loc(loc, cap) +make_map :: proc($T: typeid/map[$K]$E, #any_int capacity: int = 1< T { + make_map_expr_error_loc(loc, capacity) context.allocator = allocator m: T - reserve_map(&m, cap) + reserve_map(&m, capacity, loc) return m } @(builtin) @@ -285,36 +272,24 @@ clear_map :: proc "contextless" (m: ^$T/map[$K]$V) { if m == nil { return } - raw_map := (^Raw_Map)(m) - entries := (^Raw_Dynamic_Array)(&raw_map.entries) - entries.len = 0 - for _, i in raw_map.hashes { - raw_map.hashes[i] = MAP_SENTINEL - } + map_clear_dynamic((^Raw_Map)(m), map_info(T)) } @builtin reserve_map :: proc(m: ^$T/map[$K]$V, capacity: int, loc := #caller_location) { if m != nil { - h := __get_map_header_table(T) - __dynamic_map_reserve(m, h, uint(capacity), loc) + __dynamic_map_reserve((^Raw_Map)(m), map_info(T), uint(capacity), loc) } } /* - Shrinks the capacity of a map down to the current length, or the given capacity. - - If `new_cap` is negative, then `len(m)` is used. - - Returns false if `cap(m) < new_cap`, or the allocator report failure. - - If `len(m) < new_cap`, then `len(m)` will be left unchanged. + Shrinks the capacity of a map down to the current length. */ @builtin -shrink_map :: proc(m: ^$T/map[$K]$V, new_cap := -1, loc := #caller_location) -> (did_shrink: bool) { +shrink_map :: proc(m: ^$T/map[$K]$V, loc := #caller_location) -> (did_shrink: bool) { if m != nil { - new_cap := new_cap if new_cap >= 0 else len(m) - return __dynamic_map_shrink(__get_map_header(m), new_cap, loc) + err := map_shrink_dynamic((^Raw_Map)(m), map_info(T), loc) + did_shrink = err == nil } return } @@ -325,14 +300,10 @@ shrink_map :: proc(m: ^$T/map[$K]$V, new_cap := -1, loc := #caller_location) -> delete_key :: proc(m: ^$T/map[$K]$V, key: K) -> (deleted_key: K, deleted_value: V) { if m != nil { key := key - h := __get_map_header(m) - fr := __map_find(h, &key) - if fr.entry_index != MAP_SENTINEL { - entry := __dynamic_map_get_entry(h, fr.entry_index) - deleted_key = (^K)(uintptr(entry)+h.key_offset)^ - deleted_value = (^V)(uintptr(entry)+h.value_offset)^ - - __dynamic_map_erase(h, fr) + old_k, old_v, ok := map_erase_dynamic((^Raw_Map)(m), map_info(T), uintptr(&key)) + if ok { + deleted_key = (^K)(old_k)^ + deleted_value = (^V)(old_v)^ } } return @@ -573,10 +544,7 @@ reserve_dynamic_array :: proc(array: ^$T/[dynamic]$E, capacity: int, loc := #cal new_size := capacity * size_of(E) allocator := a.allocator - new_data, err := allocator.procedure( - allocator.data, .Resize, new_size, align_of(E), - a.data, old_size, loc, - ) + new_data, err := mem_resize(a.data, old_size, new_size, align_of(E), allocator, loc) if new_data == nil || err != nil { return false } @@ -607,10 +575,7 @@ resize_dynamic_array :: proc(array: ^$T/[dynamic]$E, length: int, loc := #caller new_size := length * size_of(E) allocator := a.allocator - new_data, err := allocator.procedure( - allocator.data, .Resize, new_size, align_of(E), - a.data, old_size, loc, - ) + new_data, err := mem_resize(a.data, old_size, new_size, align_of(E), allocator, loc) if new_data == nil || err != nil { return false } @@ -650,15 +615,7 @@ shrink_dynamic_array :: proc(array: ^$T/[dynamic]$E, new_cap := -1, loc := #call old_size := a.cap * size_of(E) new_size := new_cap * size_of(E) - new_data, err := a.allocator.procedure( - a.allocator.data, - .Resize, - new_size, - align_of(E), - a.data, - old_size, - loc, - ) + new_data, err := mem_resize(a.data, old_size, new_size, align_of(E), allocator, loc) if err != nil { return } @@ -672,10 +629,7 @@ shrink_dynamic_array :: proc(array: ^$T/[dynamic]$E, new_cap := -1, loc := #call @builtin map_insert :: proc(m: ^$T/map[$K]$V, key: K, value: V, loc := #caller_location) -> (ptr: ^V) { key, value := key, value - h := __get_map_header_table(T) - - e := __dynamic_map_set(m, h, __get_map_key_hash(&key), &key, &value, loc) - return (^V)(uintptr(e) + h.value_offset) + return (^V)(__dynamic_map_set_without_hash((^Raw_Map)(m), map_info(T), rawptr(&key), rawptr(&value), loc)) } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index abe58fc5a..0ecd0b73b 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -3,426 +3,793 @@ package runtime import "core:intrinsics" _ :: intrinsics -INITIAL_MAP_CAP :: 16 +// High performance, cache-friendly, open-addressed Robin Hood hashing hash map +// data structure with various optimizations for Odin. +// +// Copyright 2022 (c) Dale Weiler +// +// The core of the hash map data structure is the Raw_Map struct which is a +// type-erased representation of the map. This type-erased representation is +// used in two ways: static and dynamic. When static type information is known, +// the procedures suffixed with _static should be used instead of _dynamic. The +// static procedures are optimized since they have type information. Hashing of +// keys, comparison of keys, and data lookup are all optimized. When type +// information is not known, the procedures suffixed with _dynamic should be +// used. The representation of the map is the same for both static and dynamic, +// and procedures of each can be mixed and matched. The purpose of the dynamic +// representation is to enable reflection and runtime manipulation of the map. +// The dynamic procedures all take an additional Map_Info structure parameter +// which carries runtime values describing the size, alignment, and offset of +// various traits of a given key and value type pair. The Map_Info value can +// be created by calling map_info(K, V) with the key and value typeids. +// +// This map implementation makes extensive use of uintptr for representing +// sizes, lengths, capacities, masks, pointers, offsets, and addresses to avoid +// expensive sign extension and masking that would be generated if types were +// casted all over. The only place regular ints show up is in the cap() and +// len() implementations. +// +// To make this map cache-friendly it uses a novel strategy to ensure keys and +// values of the map are always cache-line aligned and that no single key or +// value of any type ever straddles a cache-line. This cache efficiency makes +// for quick lookups because the linear-probe always addresses data in a cache +// friendly way. This is enabled through the use of a special meta-type called +// a Map_Cell which packs as many values of a given type into a local array adding +// internal padding to round to MAP_CACHE_LINE_SIZE. One other benefit to storing +// the internal data in this manner is false sharing no longer occurs when using +// a map, enabling efficient concurrent access of the map data structure with +// minimal locking if desired. -// Temporary data structure for comparing hashes and keys -Map_Hash :: struct { - hash: uintptr, - key_ptr: rawptr, // address of Map_Entry_Header.key +// With Robin Hood hashing a maximum load factor of 75% is ideal. +MAP_LOAD_FACTOR :: 75 + +// Minimum log2 capacity. +MAP_MIN_LOG2_CAPACITY :: 6 // 64 elements + +// Has to be less than 100% though. +#assert(MAP_LOAD_FACTOR < 100) + +// This is safe to change. The log2 size of a cache-line. At minimum it has to +// be six though. Higher cache line sizes are permitted. +MAP_CACHE_LINE_LOG2 :: 6 + +// The size of a cache-line. +MAP_CACHE_LINE_SIZE :: 1 << MAP_CACHE_LINE_LOG2 + +// The minimum cache-line size allowed by this implementation is 64 bytes since +// we need 6 bits in the base pointer to store the integer log2 capacity, which +// at maximum is 63. Odin uses signed integers to represent length and capacity, +// so only 63 bits are needed in the maximum case. +#assert(MAP_CACHE_LINE_SIZE >= 64) + +// Map_Cell type that packs multiple T in such a way to ensure that each T stays +// aligned by align_of(T) and such that align_of(Map_Cell(T)) % MAP_CACHE_LINE_SIZE == 0 +// +// This means a value of type T will never straddle a cache-line. +// +// When multiple Ts can fit in a single cache-line the data array will have more +// than one element. When it cannot, the data array will have one element and +// an array of Map_Cell(T) will be padded to stay a multiple of MAP_CACHE_LINE_SIZE. +// +// We rely on the type system to do all the arithmetic and padding for us here. +// +// The usual array[index] indexing for []T backed by a []Map_Cell(T) becomes a bit +// more involved as there now may be internal padding. The indexing now becomes +// +// N :: len(Map_Cell(T){}.data) +// i := index / N +// j := index % N +// cell[i].data[j] +// +// However, since len(Map_Cell(T){}.data) is a compile-time constant, there are some +// optimizations we can do to eliminate the need for any divisions as N will +// be bounded by [1, 64). +// +// In the optimal case, len(Map_Cell(T){}.data) = 1 so the cell array can be treated +// as a regular array of T, which is the case for hashes. +Map_Cell :: struct($T: typeid) #align MAP_CACHE_LINE_SIZE { + data: [MAP_CACHE_LINE_SIZE / size_of(T) when 0 < size_of(T) && size_of(T) < MAP_CACHE_LINE_SIZE else 1]T, } -__get_map_key_hash :: #force_inline proc "contextless" (k: ^$K) -> uintptr { - hasher := intrinsics.type_hasher_proc(K) - return hasher(k, 0) +// So we can operate on a cell data structure at runtime without any type +// information, we have a simple table that stores some traits about the cell. +// +// 32-bytes on 64-bit +// 16-bytes on 32-bit +Map_Cell_Info :: struct { + size_of_type: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits + align_of_type: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits + size_of_cell: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits + elements_per_cell: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits } -__get_map_entry_key_ptr :: #force_inline proc "contextless" (h: Map_Header_Table, entry: ^Map_Entry_Header) -> rawptr { - return rawptr(uintptr(entry) + h.key_offset) +// map_cell_info :: proc "contextless" ($T: typeid) -> ^Map_Cell_Info {...} +map_cell_info :: intrinsics.type_map_cell_info + +// Same as the above procedure but at runtime with the cell Map_Cell_Info value. +@(require_results) +map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, #no_alias info: ^Map_Cell_Info, index: uintptr) -> uintptr { + // Micro-optimize the common cases to save on integer division. + elements_per_cell := uintptr(info.elements_per_cell) + size_of_cell := uintptr(info.size_of_cell) + switch elements_per_cell { + case 1: + return base + (index * size_of_cell) + case 2: + cell_index := index >> 1 + data_index := index & 1 + size_of_type := uintptr(info.size_of_type) + return base + (cell_index * size_of_cell) + (data_index * size_of_type) + case: + cell_index := index / elements_per_cell + data_index := index % elements_per_cell + size_of_type := uintptr(info.size_of_type) + return base + (cell_index * size_of_cell) + (data_index * size_of_type) + } } -Map_Index :: distinct uint -MAP_SENTINEL :: ~Map_Index(0) - -Map_Find_Result :: struct { - hash_index: Map_Index, - entry_prev: Map_Index, - entry_index: Map_Index, +// Same as above procedure but with compile-time constant index. +@(require_results) +map_cell_index_dynamic_const :: proc "contextless" (base: uintptr, #no_alias info: ^Map_Cell_Info, $INDEX: uintptr) -> uintptr { + elements_per_cell := uintptr(info.elements_per_cell) + size_of_cell := uintptr(info.size_of_cell) + size_of_type := uintptr(info.size_of_type) + cell_index := INDEX / elements_per_cell + data_index := INDEX % elements_per_cell + return base + (cell_index * size_of_cell) + (data_index * size_of_type) } -Map_Entry_Header :: struct { - hash: uintptr, - next: Map_Index, -/* - key: Key_Value, - value: Value_Type, -*/ -} +// We always round the capacity to a power of two so this becomes [16]Foo, which +// works out to [4]Cell(Foo). +// +// The following compile-time procedure indexes such a [N]Cell(T) structure as +// if it were a flat array accounting for the internal padding introduced by the +// Cell structure. +@(require_results) +map_cell_index_static :: #force_inline proc "contextless" (cells: [^]Map_Cell($T), index: uintptr) -> ^T #no_bounds_check { + N :: size_of(Map_Cell(T){}.data) / size_of(T) when size_of(T) > 0 else 1 -Map_Header_Table :: struct { - equal: Equal_Proc, + #assert(N <= MAP_CACHE_LINE_SIZE) - entry_size: int, - entry_align: int, + when size_of(Map_Cell(T)) == size_of([N]T) { + // No padding case, can treat as a regular array of []T. - key_offset: uintptr, - key_size: int, + return &([^]T)(cells)[index] + } else when (N & (N - 1)) == 0 && N <= 8*size_of(uintptr) { + // Likely case, N is a power of two because T is a power of two. - value_offset: uintptr, - value_size: int, -} - -Map_Header :: struct { - m: ^Raw_Map, - using table: Map_Header_Table, -} - -// USED INTERNALLY BY THE COMPILER -__dynamic_map_get :: proc "contextless" (m: rawptr, table: Map_Header_Table, key_hash: uintptr, key_ptr: rawptr) -> rawptr { - if m != nil { - h := Map_Header{(^Raw_Map)(m), table} - index := __dynamic_map_find(h, key_hash, key_ptr).entry_index - if index != MAP_SENTINEL { - data := uintptr(__dynamic_map_get_entry(h, index)) - return rawptr(data + h.value_offset) + // Compute the integer log 2 of N, this is the shift amount to index the + // correct cell. Odin's intrinsics.count_leading_zeros does not produce a + // constant, hence this approach. We only need to check up to N = 64. + SHIFT :: 1 when N < 2 else + 2 when N < 4 else + 3 when N < 8 else + 4 when N < 16 else + 5 when N < 32 else 6 + #assert(SHIFT <= MAP_CACHE_LINE_LOG2) + // Unique case, no need to index data here since only one element. + when N == 1 { + return &cells[index >> SHIFT].data[0] + } else { + return &cells[index >> SHIFT].data[index & (N - 1)] } + } else { + // Least likely (and worst case), we pay for a division operation but we + // assume the compiler does not actually generate a division. N will be in the + // range [1, CACHE_LINE_SIZE) and not a power of two. + return &cells[index / N].data[index % N] + } +} + +// len() for map +@(require_results) +map_len :: #force_inline proc "contextless" (m: Raw_Map) -> int { + return m.len +} + +// cap() for map +@(require_results) +map_cap :: #force_inline proc "contextless" (m: Raw_Map) -> int { + // The data uintptr stores the capacity in the lower six bits which gives the + // a maximum value of 2^6-1, or 63. We store the integer log2 of capacity + // since our capacity is always a power of two. We only need 63 bits as Odin + // represents length and capacity as a signed integer. + return 0 if m.data == 0 else 1 << map_log2_cap(m) +} + +// Query the load factor of the map. This is not actually configurable, but +// some math is needed to compute it. Compute it as a fixed point percentage to +// avoid floating point operations. This division can be optimized out by +// multiplying by the multiplicative inverse of 100. +@(require_results) +map_load_factor :: #force_inline proc "contextless" (log2_capacity: uintptr) -> uintptr { + return ((uintptr(1) << log2_capacity) * MAP_LOAD_FACTOR) / 100 +} + +@(require_results) +map_resize_threshold :: #force_inline proc "contextless" (m: Raw_Map) -> int { + return int(map_load_factor(map_log2_cap(m))) +} + +// The data stores the log2 capacity in the lower six bits. This is primarily +// used in the implementation rather than map_cap since the check for data = 0 +// isn't necessary in the implementation. cap() on the otherhand needs to work +// when called on an empty map. +@(require_results) +map_log2_cap :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr { + return m.data & (64 - 1) +} + +// Canonicalize the data by removing the tagged capacity stored in the lower six +// bits of the data uintptr. +@(require_results) +map_data :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr { + return m.data &~ uintptr(64 - 1) +} + + +Map_Hash :: uintptr + +// Procedure to check if a slot is empty for a given hash. This is represented +// by the zero value to make the zero value useful. This is a procedure just +// for prose reasons. +@(require_results) +map_hash_is_empty :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { + return hash == 0 +} + +@(require_results) +map_hash_is_deleted :: #force_no_inline proc "contextless" (hash: Map_Hash) -> bool { + // The MSB indicates a tombstone + N :: size_of(Map_Hash)*8 - 1 + return hash >> N != 0 +} +@(require_results) +map_hash_is_valid :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { + // The MSB indicates a tombstone + N :: size_of(Map_Hash)*8 - 1 + return (hash != 0) & (hash >> N == 0) +} + + +// Computes the desired position in the array. This is just index % capacity, +// but a procedure as there's some math involved here to recover the capacity. +@(require_results) +map_desired_position :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Hash) -> uintptr { + // We do not use map_cap since we know the capacity will not be zero here. + capacity := uintptr(1) << map_log2_cap(m) + return uintptr(hash & Map_Hash(capacity - 1)) +} + +@(require_results) +map_probe_distance :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Hash, slot: uintptr) -> uintptr { + // We do not use map_cap since we know the capacity will not be zero here. + capacity := uintptr(1) << map_log2_cap(m) + return (slot + capacity - map_desired_position(m, hash)) & (capacity - 1) +} + +// When working with the type-erased structure at runtime we need information +// about the map to make working with it possible. This info structure stores +// that. +// +// `Map_Info` and `Map_Cell_Info` are read only data structures and cannot be +// modified after creation +// +// 32-bytes on 64-bit +// 16-bytes on 32-bit +Map_Info :: struct { + ks: ^Map_Cell_Info, // 8-bytes on 64-bit, 4-bytes on 32-bit + vs: ^Map_Cell_Info, // 8-bytes on 64-bit, 4-bytes on 32-bit + key_hasher: proc "contextless" (key: rawptr, seed: Map_Hash) -> Map_Hash, // 8-bytes on 64-bit, 4-bytes on 32-bit + key_equal: proc "contextless" (lhs, rhs: rawptr) -> bool, // 8-bytes on 64-bit, 4-bytes on 32-bit +} + + +// The Map_Info structure is basically a pseudo-table of information for a given K and V pair. +// map_info :: proc "contextless" ($T: typeid/map[$K]$V) -> ^Map_Info {...} +map_info :: intrinsics.type_map_info + +@(require_results) +map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (ks: uintptr, vs: uintptr, hs: [^]Map_Hash, sk: uintptr, sv: uintptr) { + INFO_HS := intrinsics.type_map_cell_info(Map_Hash) + + capacity := uintptr(1) << map_log2_cap(m) + ks = map_data(m) + vs = map_cell_index_dynamic(ks, info.ks, capacity) // Skip past ks to get start of vs + hs_ := map_cell_index_dynamic(vs, info.vs, capacity) // Skip past vs to get start of hs + sk = map_cell_index_dynamic(hs_, INFO_HS, capacity) // Skip past hs to get start of sk + // Need to skip past two elements in the scratch key space to get to the start + // of the scratch value space, of which there's only two elements as well. + sv = map_cell_index_dynamic_const(sk, info.ks, 2) + + hs = ([^]Map_Hash)(hs_) + return +} + +@(require_results) +map_kvh_data_values_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (vs: uintptr) { + capacity := uintptr(1) << map_log2_cap(m) + return map_cell_index_dynamic(map_data(m), info.ks, capacity) // Skip past ks to get start of vs +} + + +@(private, require_results) +map_total_allocation_size :: #force_inline proc "contextless" (capacity: uintptr, info: ^Map_Info) -> uintptr { + round :: #force_inline proc "contextless" (value: uintptr) -> uintptr { + CACHE_MASK :: MAP_CACHE_LINE_SIZE - 1 + return (value + CACHE_MASK) &~ CACHE_MASK + } + INFO_HS := intrinsics.type_map_cell_info(Map_Hash) + + size := uintptr(0) + size = round(map_cell_index_dynamic(size, info.ks, capacity)) + size = round(map_cell_index_dynamic(size, info.vs, capacity)) + size = round(map_cell_index_dynamic(size, INFO_HS, capacity)) + size = round(map_cell_index_dynamic(size, info.ks, 2)) // Two additional ks for scratch storage + size = round(map_cell_index_dynamic(size, info.vs, 2)) // Two additional vs for scratch storage + return size +} + +// The only procedure which needs access to the context is the one which allocates the map. +@(require_results) +map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator, loc := #caller_location) -> (result: Raw_Map, err: Allocator_Error) { + result.allocator = allocator // set the allocator always + if log2_capacity == 0 { + return + } + + if log2_capacity >= 64 { + // Overflowed, would be caused by log2_capacity > 64 + return {}, .Out_Of_Memory + } + + capacity := uintptr(1) << max(log2_capacity, MAP_MIN_LOG2_CAPACITY) + + CACHE_MASK :: MAP_CACHE_LINE_SIZE - 1 + + size := map_total_allocation_size(capacity, info) + + data := mem_alloc_non_zeroed(int(size), MAP_CACHE_LINE_SIZE, allocator, loc) or_return + data_ptr := uintptr(raw_data(data)) + if data_ptr == 0 { + err = .Out_Of_Memory + return + } + if intrinsics.expect(data_ptr & CACHE_MASK != 0, false) { + panic("allocation not aligned to a cache line", loc) + } else { + result.data = data_ptr | log2_capacity // Tagged pointer representation for capacity. + result.len = 0 + + map_clear_dynamic(&result, info) + } + return +} + +// This procedure has to stack allocate storage to store local keys during the +// Robin Hood hashing technique where elements are swapped in the backing +// arrays to reduce variance. This swapping can only be done with memcpy since +// there is no type information. +// +// This procedure returns the address of the just inserted value. +@(require_results) +map_insert_hash_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { + h := h + pos := map_desired_position(m^, h) + distance := uintptr(0) + mask := (uintptr(1) << map_log2_cap(m^)) - 1 + + ks, vs, hs, sk, sv := map_kvh_data_dynamic(m^, info) + + // Avoid redundant loads of these values + size_of_k := info.ks.size_of_type + size_of_v := info.vs.size_of_type + + k := map_cell_index_dynamic(sk, info.ks, 0) + v := map_cell_index_dynamic(sv, info.vs, 0) + intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(ik), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(iv), size_of_v) + + // Temporary k and v dynamic storage for swap below + tk := map_cell_index_dynamic(sk, info.ks, 1) + tv := map_cell_index_dynamic(sv, info.vs, 1) + + + for { + hp := &hs[pos] + element_hash := hp^ + + if map_hash_is_empty(element_hash) { + k_dst := map_cell_index_dynamic(ks, info.ks, pos) + v_dst := map_cell_index_dynamic(vs, info.vs, pos) + intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) + hp^ = h + + return result if result != 0 else v_dst + } + + if probe_distance := map_probe_distance(m^, element_hash, pos); distance > probe_distance { + if map_hash_is_deleted(element_hash) { + k_dst := map_cell_index_dynamic(ks, info.ks, pos) + v_dst := map_cell_index_dynamic(vs, info.vs, pos) + intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) + hp^ = h + + return result if result != 0 else v_dst + } + + if result == 0 { + result = map_cell_index_dynamic(vs, info.vs, pos) + } + + kp := map_cell_index_dynamic(ks, info.ks, pos) + vp := map_cell_index_dynamic(vs, info.vs, pos) + + intrinsics.mem_copy_non_overlapping(rawptr(tk), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(kp), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(kp), rawptr(tk), size_of_k) + + intrinsics.mem_copy_non_overlapping(rawptr(tv), rawptr(v), size_of_v) + intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(vp), size_of_v) + intrinsics.mem_copy_non_overlapping(rawptr(vp), rawptr(tv), size_of_v) + + th := h + h = hp^ + hp^ = th + + distance = probe_distance + } + + pos = (pos + 1) & mask + distance += 1 + } +} + +@(require_results) +map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { + log2_capacity := map_log2_cap(m^) + new_capacity := uintptr(1) << max(log2_capacity + 1, MAP_MIN_LOG2_CAPACITY) + return map_reserve_dynamic(m, info, new_capacity, loc) +} + + +@(require_results) +map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr, loc := #caller_location) -> Allocator_Error { + @(require_results) + ceil_log2 :: #force_inline proc "contextless" (x: uintptr) -> uintptr { + z := intrinsics.count_leading_zeros(x) + if z > 0 && x & (x-1) != 0 { + z -= 1 + } + return size_of(uintptr)*8 - 1 - z + } + + if m.allocator.procedure == nil { + m.allocator = context.allocator + } + + new_capacity := new_capacity + old_capacity := uintptr(map_cap(m^)) + + if old_capacity >= new_capacity { + return nil + } + + // ceiling nearest power of two + log2_new_capacity := ceil_log2(new_capacity) + + log2_min_cap := max(MAP_MIN_LOG2_CAPACITY, log2_new_capacity) + + if m.data == 0 { + m^ = map_alloc_dynamic(info, log2_min_cap, m.allocator, loc) or_return + return nil + } + + resized := map_alloc_dynamic(info, log2_min_cap, m.allocator, loc) or_return + + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) + + // Cache these loads to avoid hitting them in the for loop. + n := m.len + for i in 0.. Allocator_Error { + if m.allocator.procedure == nil { + m.allocator = context.allocator + } + + // Cannot shrink the capacity if the number of items in the map would exceed + // one minus the current log2 capacity's resize threshold. That is the shrunk + // map needs to be within the max load factor. + log2_capacity := map_log2_cap(m^) + if uintptr(m.len) >= map_load_factor(log2_capacity - 1) { + return nil + } + + shrunk := map_alloc_dynamic(info, log2_capacity - 1, m.allocator) or_return + + capacity := uintptr(1) << log2_capacity + + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) + + n := m.len + for i in 0.. Allocator_Error { + ptr := rawptr(map_data(m)) + size := int(map_total_allocation_size(uintptr(map_cap(m)), info)) + return mem_free_with_size(ptr, size, m.allocator, loc) +} + +@(require_results) +map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (index: uintptr, ok: bool) { + if map_len(m) == 0 { + return 0, false + } + h := info.key_hasher(rawptr(k), 0) + p := map_desired_position(m, h) + d := uintptr(0) + c := (uintptr(1) << map_log2_cap(m)) - 1 + ks, _, hs, _, _ := map_kvh_data_dynamic(m, info) + for { + element_hash := hs[p] + if map_hash_is_empty(element_hash) { + return 0, false + } else if d > map_probe_distance(m, element_hash, p) { + return 0, false + } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, p))) { + return p, true + } + p = (p + 1) & c + d += 1 + } +} +@(require_results) +map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (ok: bool) { + if map_len(m) == 0 { + return false + } + h := info.key_hasher(rawptr(k), 0) + p := map_desired_position(m, h) + d := uintptr(0) + c := (uintptr(1) << map_log2_cap(m)) - 1 + ks, _, hs, _, _ := map_kvh_data_dynamic(m, info) + for { + element_hash := hs[p] + if map_hash_is_empty(element_hash) { + return false + } else if d > map_probe_distance(m, element_hash, p) { + return false + } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, p))) { + return true + } + p = (p + 1) & c + d += 1 + } +} + + + +@(require_results) +map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (old_k, old_v: uintptr, ok: bool) { + MASK :: 1 << (size_of(Map_Hash)*8 - 1) + + index := map_lookup_dynamic(m^, info, k) or_return + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) + hs[index] |= MASK + old_k = map_cell_index_dynamic(ks, info.ks, index) + old_v = map_cell_index_dynamic(vs, info.vs, index) + m.len -= 1 + ok = true + return +} + +map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) { + if m.data == 0 { + return + } + _, _, hs, _, _ := map_kvh_data_dynamic(m^, info) + intrinsics.mem_zero(rawptr(hs), map_cap(m^) * size_of(Map_Hash)) + m.len = 0 +} + + +@(require_results) +map_kvh_data_static :: #force_inline proc "contextless" (m: $T/map[$K]$V) -> (ks: [^]Map_Cell(K), vs: [^]Map_Cell(V), hs: [^]Map_Hash) { + capacity := uintptr(cap(m)) + ks = ([^]Map_Cell(K))(map_data(transmute(Raw_Map)m)) + vs = ([^]Map_Cell(V))(map_cell_index_static(ks, capacity)) + hs = ([^]Map_Hash)(map_cell_index_static(vs, capacity)) + return +} + + +@(require_results) +map_get :: proc "contextless" (m: $T/map[$K]$V, key: K) -> (stored_key: K, stored_value: V, ok: bool) { + rm := transmute(Raw_Map)m + if rm.len == 0 { + return + } + info := intrinsics.type_map_info(T) + key := key + + h := info.key_hasher(&key, 0) + pos := map_desired_position(rm, h) + distance := uintptr(0) + mask := (uintptr(1) << map_log2_cap(rm)) - 1 + ks, vs, hs := map_kvh_data_static(m) + for { + element_hash := hs[pos] + if map_hash_is_empty(element_hash) { + return + } else if distance > map_probe_distance(rm, element_hash, pos) { + return + } else if element_hash == h { + element_key := map_cell_index_static(ks, pos) + if info.key_equal(&key, rawptr(element_key)) { + element_value := map_cell_index_static(vs, pos) + stored_key = (^K)(element_key)^ + stored_value = (^V)(element_value)^ + ok = true + return + } + + } + pos = (pos + 1) & mask + distance += 1 + } +} + +// IMPORTANT: USED WITHIN THE COMPILER +__dynamic_map_get :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, key: rawptr) -> (ptr: rawptr) { + if m.len == 0 { + return nil + } + pos := map_desired_position(m^, h) + distance := uintptr(0) + mask := (uintptr(1) << map_log2_cap(m^)) - 1 + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) + for { + element_hash := hs[pos] + if map_hash_is_empty(element_hash) { + return nil + } else if distance > map_probe_distance(m^, element_hash, pos) { + return nil + } else if element_hash == h && info.key_equal(key, rawptr(map_cell_index_dynamic(ks, info.ks, pos))) { + return rawptr(map_cell_index_dynamic(vs, info.vs, pos)) + } + pos = (pos + 1) & mask + distance += 1 + } +} + +// IMPORTANT: USED WITHIN THE COMPILER +__dynamic_map_check_grow :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { + if m.len >= map_resize_threshold(m^) { + return map_grow_dynamic(m, info, loc) } return nil } -// USED INTERNALLY BY THE COMPILER -__dynamic_map_set :: proc "odin" (m: rawptr, table: Map_Header_Table, key_hash: uintptr, key_ptr: rawptr, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check { - add_entry :: proc "odin" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr, loc := #caller_location) -> Map_Index { - prev := Map_Index(h.m.entries.len) - c := Map_Index(__dynamic_array_append_nothing(&h.m.entries, h.entry_size, h.entry_align, loc)) - if c != prev { - end := __dynamic_map_get_entry(h, c-1) - end.hash = key_hash - mem_copy(rawptr(uintptr(end) + h.key_offset), key_ptr, h.key_size) - end.next = MAP_SENTINEL - } - return prev - } - - h := Map_Header{(^Raw_Map)(m), table} - - index := MAP_SENTINEL - - if len(h.m.hashes) == 0 { - __dynamic_map_reserve(m, table, INITIAL_MAP_CAP, loc) - __dynamic_map_grow(h, loc) - } - - fr := __dynamic_map_find(h, key_hash, key_ptr) - if fr.entry_index != MAP_SENTINEL { - index = fr.entry_index - } else { - index = add_entry(h, key_hash, key_ptr, loc) - if fr.entry_prev != MAP_SENTINEL { - entry := __dynamic_map_get_entry(h, fr.entry_prev) - entry.next = index - } else if fr.hash_index != MAP_SENTINEL { - h.m.hashes[fr.hash_index] = index - } else { - return nil - } - } - - e := __dynamic_map_get_entry(h, index) - e.hash = key_hash - - key := rawptr(uintptr(e) + h.key_offset) - val := rawptr(uintptr(e) + h.value_offset) - - mem_copy(key, key_ptr, h.key_size) - mem_copy(val, value, h.value_size) - - if __dynamic_map_full(h) { - __dynamic_map_grow(h, loc) - } - - return __dynamic_map_get_entry(h, index) +__dynamic_map_set_without_hash :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { + return __dynamic_map_set(m, info, info.key_hasher(key, 0), key, value, loc) } -// USED INTERNALLY BY THE COMPILER -__dynamic_map_reserve :: proc "odin" (m: rawptr, table: Map_Header_Table, cap: uint, loc := #caller_location) { - h := Map_Header{(^Raw_Map)(m), table} - c := context - if h.m.entries.allocator.procedure != nil { - c.allocator = h.m.entries.allocator +// IMPORTANT: USED WITHIN THE COMPILER +__dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, hash: Map_Hash, key, value: rawptr, loc := #caller_location) -> rawptr { + if found := __dynamic_map_get(m, info, hash, key); found != nil { + intrinsics.mem_copy_non_overlapping(found, value, info.vs.size_of_type) + return found } - context = c - cap := cap - cap = ceil_to_pow2(cap) - - __dynamic_array_reserve(&h.m.entries, h.entry_size, h.entry_align, int(cap), loc) - - if h.m.entries.len*2 < len(h.m.hashes) { - return - } - if __slice_resize(&h.m.hashes, int(cap*2), h.m.entries.allocator, loc) { - __dynamic_map_reset_entries(h, loc) + if __dynamic_map_check_grow(m, info, loc) != nil { + return nil } + + result := map_insert_hash_dynamic(m, info, hash, uintptr(key), uintptr(value)) + m.len += 1 + return rawptr(result) } +// IMPORTANT: USED WITHIN THE COMPILER +@(private) +__dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) -> Allocator_Error { + return map_reserve_dynamic(m, info, uintptr(new_capacity), loc) +} + + + +// NOTE: the default hashing algorithm derives from fnv64a, with some minor modifications to work for `map` type: +// +// * Convert a `0` result to `1` +// * "empty entry" +// * Prevent the top bit from being set +// * "deleted entry" +// +// Both of these modification are necessary for the implementation of the `map` INITIAL_HASH_SEED :: 0xcbf29ce484222325 -_fnv64a :: proc "contextless" (data: []byte, seed: u64 = INITIAL_HASH_SEED) -> u64 { - h: u64 = seed - for b in data { - h = (h ~ u64(b)) * 0x100000001b3 - } - return h -} +HASH_MASK :: 1 << (8*size_of(uintptr) - 1) -1 -default_hash :: #force_inline proc "contextless" (data: []byte) -> uintptr { - return uintptr(_fnv64a(data)) -} -default_hash_string :: #force_inline proc "contextless" (s: string) -> uintptr { - return default_hash(transmute([]byte)(s)) -} -default_hash_ptr :: #force_inline proc "contextless" (data: rawptr, size: int) -> uintptr { - s := Raw_Slice{data, size} - return default_hash(transmute([]byte)(s)) -} - -@(private) -_default_hasher_const :: #force_inline proc "contextless" (data: rawptr, seed: uintptr, $N: uint) -> uintptr where N <= 16 { - h := u64(seed) + 0xcbf29ce484222325 - p := uintptr(data) - #unroll for _ in 0.. uintptr { - h := u64(seed) + 0xcbf29ce484222325 - p := uintptr(data) +default_hasher :: #force_inline proc "contextless" (data: rawptr, seed: uintptr, N: int) -> uintptr { + h := u64(seed) + INITIAL_HASH_SEED + p := ([^]byte)(data) for _ in 0.. uintptr { return #force_inline _default_hasher_const(data, seed, 1) } -default_hasher2 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 2) } -default_hasher3 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 3) } -default_hasher4 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 4) } -default_hasher5 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 5) } -default_hasher6 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 6) } -default_hasher7 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 7) } -default_hasher8 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 8) } -default_hasher9 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 9) } -default_hasher10 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 10) } -default_hasher11 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 11) } -default_hasher12 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 12) } -default_hasher13 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 13) } -default_hasher14 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 14) } -default_hasher15 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 15) } -default_hasher16 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 16) } - default_hasher_string :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { - h := u64(seed) + 0xcbf29ce484222325 - str := (^[]byte)(data)^ - for b in str { - h = (h ~ u64(b)) * 0x100000001b3 - } - return uintptr(h) + str := (^[]byte)(data) + return default_hasher(raw_data(str^), seed, len(str)) } default_hasher_cstring :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { - h := u64(seed) + 0xcbf29ce484222325 - ptr := (^uintptr)(data)^ - for (^byte)(ptr)^ != 0 { - b := (^byte)(ptr)^ - h = (h ~ u64(b)) * 0x100000001b3 - ptr += 1 - } - return uintptr(h) -} - - -__get_map_header :: proc "contextless" (m: ^$T/map[$K]$V) -> (header: Map_Header) { - header.m = (^Raw_Map)(m) - header.table = #force_inline __get_map_header_table(T) - return -} - -__get_map_header_runtime :: proc "contextless" (m: ^Raw_Map, ti: Type_Info_Map) -> (header: Map_Header) { - header.m = m - header.table = #force_inline __get_map_header_table_runtime(ti) - return -} - -__get_map_header_table :: proc "contextless" ($T: typeid/map[$K]$V) -> (header: Map_Header_Table) { - Entry :: struct { - hash: uintptr, - next: Map_Index, - key: K, - value: V, - } - - header.equal = intrinsics.type_equal_proc(K) - - header.entry_size = size_of(Entry) - header.entry_align = align_of(Entry) - - header.key_offset = offset_of(Entry, key) - header.key_size = size_of(K) - - header.value_offset = offset_of(Entry, value) - header.value_size = size_of(V) - - return -} - -__get_map_header_table_runtime :: proc "contextless" (ti: Type_Info_Map) -> (header: Map_Header) { - header.equal = ti.key_equal - - entries := ti.generated_struct.variant.(Type_Info_Struct).types[1] - entry := entries.variant.(Type_Info_Dynamic_Array).elem - e := entry.variant.(Type_Info_Struct) - - header.entry_size = entry.size - header.entry_align = entry.align - - header.key_offset = e.offsets[2] - header.key_size = e.types[2].size - - header.value_offset = e.offsets[3] - header.value_size = e.types[3].size - - return -} - - - -__slice_resize :: proc "odin" (array_: ^$T/[]$E, new_count: int, allocator: Allocator, loc := #caller_location) -> bool { - array := (^Raw_Slice)(array_) - - if new_count < array.len { - return true - } - - old_size := array.len*size_of(T) - new_size := new_count*size_of(T) - - new_data, err := mem_resize(array.data, old_size, new_size, align_of(T), allocator, loc) - if err != nil { - return false - } - if new_data != nil || size_of(E) == 0 { - array.data = raw_data(new_data) - array.len = new_count - return true - } - return false -} - -__dynamic_map_reset_entries :: proc "contextless" (h: Map_Header, loc := #caller_location) { - for i in 0.. (did_shrink: bool) { - c := context - if h.m.entries.allocator.procedure != nil { - c.allocator = h.m.entries.allocator - } - context = c - - return __dynamic_array_shrink(&h.m.entries, h.entry_size, h.entry_align, cap, loc) -} - - -@(private="file") -ceil_to_pow2 :: proc "contextless" (n: uint) -> uint { - if n <= 2 { - return n - } - n := n - n -= 1 - n |= n >> 1 - n |= n >> 2 - n |= n >> 4 - n |= n >> 8 - n |= n >> 16 - when size_of(int) == 8 { - n |= n >> 32 - } - n += 1 - return n -} - -__dynamic_map_grow :: proc "odin" (h: Map_Header, loc := #caller_location) { - new_count := max(uint(h.m.entries.cap) * 2, INITIAL_MAP_CAP) - // Rehash through Reserve - __dynamic_map_reserve(h.m, h.table, new_count, loc) -} - -__dynamic_map_full :: #force_inline proc "contextless" (h: Map_Header) -> bool { - return int(0.75 * f64(len(h.m.hashes))) <= h.m.entries.len -} - -__dynamic_map_find_from_entry :: proc "contextless" (h: Map_Header, e: ^Map_Entry_Header) -> Map_Find_Result #no_bounds_check { - key_ptr := __get_map_entry_key_ptr(h, e) - return __dynamic_map_find(h, e.hash, key_ptr) - -} - -__dynamic_map_find :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr) -> Map_Find_Result #no_bounds_check { - fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL} - if n := uintptr(len(h.m.hashes)); n != 0 { - fr.hash_index = Map_Index(key_hash & (n-1)) - fr.entry_index = h.m.hashes[fr.hash_index] - for fr.entry_index != MAP_SENTINEL { - entry := __dynamic_map_get_entry(h, fr.entry_index) - entry_key_ptr := __get_map_entry_key_ptr(h, entry) - if entry.hash == key_hash && h.equal(entry_key_ptr, key_ptr) { - return fr - } - - fr.entry_prev = fr.entry_index - fr.entry_index = entry.next - } - } - return fr -} - -// Utility procedure used by other runtime procedures -__map_find :: proc "contextless" (h: Map_Header, key_ptr: ^$K) -> Map_Find_Result #no_bounds_check { - hash := __get_map_key_hash(key_ptr) - return #force_inline __dynamic_map_find(h, hash, key_ptr) -} - -__dynamic_map_get_entry :: #force_inline proc "contextless" (h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { - return (^Map_Entry_Header)(uintptr(h.m.entries.data) + uintptr(index*Map_Index(h.entry_size))) -} - -__dynamic_map_erase :: proc "contextless" (h: Map_Header, fr: Map_Find_Result) #no_bounds_check { - if fr.entry_prev != MAP_SENTINEL { - prev := __dynamic_map_get_entry(h, fr.entry_prev) - curr := __dynamic_map_get_entry(h, fr.entry_index) - prev.next = curr.next - } else { - h.m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next - } - last_index := Map_Index(h.m.entries.len-1) - if fr.entry_index != last_index { - old := __dynamic_map_get_entry(h, fr.entry_index) - end := __dynamic_map_get_entry(h, last_index) - mem_copy(old, end, h.entry_size) - - last := __dynamic_map_find_from_entry(h, old) - if last.entry_prev != MAP_SENTINEL { - e := __dynamic_map_get_entry(h, last.entry_prev) - e.next = fr.entry_index - } else { - h.m.hashes[last.hash_index] = fr.entry_index - } - } - - h.m.entries.len -= 1 + h &= HASH_MASK + return uintptr(h) | uintptr(uintptr(h) == 0) } diff --git a/core/slice/map.odin b/core/slice/map.odin index 9de00b174..22f0db9b9 100644 --- a/core/slice/map.odin +++ b/core/slice/map.odin @@ -6,8 +6,8 @@ import "core:runtime" _ :: intrinsics _ :: runtime -map_keys :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (keys: []K) { - keys = make(type_of(keys), len(m), allocator) +map_keys :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (keys: []K, err: runtime.Allocator_Error) { + keys = make(type_of(keys), len(m), allocator) or_return i := 0 for key in m { keys[i] = key @@ -15,8 +15,8 @@ map_keys :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (keys: []K) } return } -map_values :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (values: []V) { - values = make(type_of(values), len(m), allocator) +map_values :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (values: []V, err: runtime.Allocator_Error) { + values = make(type_of(values), len(m), allocator) or_return i := 0 for _, value in m { values[i] = value @@ -37,8 +37,8 @@ Map_Entry_Info :: struct($Key, $Value: typeid) { } -map_entries :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (entries: []Map_Entry(K, V)) { - entries = make(type_of(entries), len(m), allocator) +map_entries :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (entries: []Map_Entry(K, V), err: runtime.Allocator) { + entries = make(type_of(entries), len(m), allocator) or_return i := 0 for key, value in m { entries[i].key = key @@ -52,28 +52,24 @@ map_entry_infos :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (ent m := m rm := (^runtime.Raw_Map)(&m) - info := runtime.type_info_base(type_info_of(M)).variant.(runtime.Type_Info_Map) - gs := runtime.type_info_base(info.generated_struct).variant.(runtime.Type_Info_Struct) - ed := runtime.type_info_base(gs.types[1]).variant.(runtime.Type_Info_Dynamic_Array) - entry_type := ed.elem.variant.(runtime.Type_Info_Struct) - key_offset := entry_type.offsets[2] - value_offset := entry_type.offsets[3] - entry_size := uintptr(ed.elem_size) + info := type_info_base(type_info_of(M)).variant.(Type_Info_Map) + if info.map_info != nil { + entries = make(type_of(entries), len(m), allocator) or_return - entries = make(type_of(entries), rm.entries.len) + map_cap := uintptr(cap(m)) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(rm^, info.map_info) + entry_index := 0 + for bucket_index in 0..args[0]); + Type *type = base_type(bt); + if (type == nullptr || type == t_invalid) { + error(ce->args[0], "Expected a type for '%.*s'", LIT(builtin_name)); + return false; + } + if (!is_type_map(type)) { + gbString t = type_to_string(type); + error(ce->args[0], "Expected a map type for '%.*s', got %s", LIT(builtin_name), t); + gb_string_free(t); + return false; + } + + add_map_key_type_dependencies(c, type); + + operand->mode = Addressing_Value; + operand->type = t_map_info_ptr; + break; + } + case BuiltinProc_type_map_cell_info: + { + Operand op = {}; + Type *bt = check_type(c, ce->args[0]); + Type *type = base_type(bt); + if (type == nullptr || type == t_invalid) { + error(ce->args[0], "Expected a type for '%.*s'", LIT(builtin_name)); + return false; + } + + operand->mode = Addressing_Value; + operand->type = t_map_cell_info_ptr; + break; + } + case BuiltinProc_constant_utf16_cstring: { String value = {}; diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 9e48fd8ad..c58aac609 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -285,6 +285,37 @@ void error_operand_no_value(Operand *o) { } } +void add_map_get_dependencies(CheckerContext *c) { + if (build_context.use_static_map_calls) { + add_package_dependency(c, "runtime", "map_desired_position"); + add_package_dependency(c, "runtime", "map_probe_distance"); + } else { + add_package_dependency(c, "runtime", "__dynamic_map_get"); + } +} + +void add_map_set_dependencies(CheckerContext *c) { + init_core_source_code_location(c->checker); + + if (t_map_set_proc == nullptr) { + Type *map_set_args[5] = {/*map*/t_rawptr, /*hash*/t_uintptr, /*key*/t_rawptr, /*value*/t_rawptr, /*#caller_location*/t_source_code_location}; + t_map_set_proc = alloc_type_proc_from_types(map_set_args, gb_count_of(map_set_args), t_rawptr, false, ProcCC_Odin); + } + + if (build_context.use_static_map_calls) { + add_package_dependency(c, "runtime", "__dynamic_map_check_grow"); + add_package_dependency(c, "runtime", "map_insert_hash_dynamic"); + } else { + add_package_dependency(c, "runtime", "__dynamic_map_set"); + } +} + +void add_map_reserve_dependencies(CheckerContext *c) { + init_core_source_code_location(c->checker); + add_package_dependency(c, "runtime", "__dynamic_map_reserve"); +} + + void check_scope_decls(CheckerContext *c, Slice const &nodes, isize reserve_size) { Scope *s = c->scope; @@ -1364,8 +1395,6 @@ bool is_polymorphic_type_assignable(CheckerContext *c, Type *poly, Type *source, bool key = is_polymorphic_type_assignable(c, poly->Map.key, source->Map.key, true, modify_type); bool value = is_polymorphic_type_assignable(c, poly->Map.value, source->Map.value, true, modify_type); if (key || value) { - poly->Map.entry_type = nullptr; - poly->Map.internal_type = nullptr; poly->Map.lookup_result_type = nullptr; init_map_internal_types(poly); return true; @@ -3246,7 +3275,7 @@ void check_binary_expr(CheckerContext *c, Operand *x, Ast *node, Type *type_hint check_assignment(c, x, yt->Map.key, str_lit("map 'not_in'")); } - add_package_dependency(c, "runtime", "__dynamic_map_get"); + add_map_get_dependencies(c); } else if (is_type_bit_set(rhs_type)) { Type *yt = base_type(rhs_type); @@ -8557,8 +8586,8 @@ ExprKind check_compound_literal(CheckerContext *c, Operand *o, Ast *node, Type * if (build_context.no_dynamic_literals && cl->elems.count) { error(node, "Compound literals of dynamic types have been disabled"); } else { - add_package_dependency(c, "runtime", "__dynamic_map_reserve"); - add_package_dependency(c, "runtime", "__dynamic_map_set"); + add_map_reserve_dependencies(c); + add_map_set_dependencies(c); } break; } @@ -8994,8 +9023,8 @@ ExprKind check_index_expr(CheckerContext *c, Operand *o, Ast *node, Type *type_h o->type = t->Map.value; o->expr = node; - add_package_dependency(c, "runtime", "__dynamic_map_get"); - add_package_dependency(c, "runtime", "__dynamic_map_set"); + add_map_get_dependencies(c); + add_map_set_dependencies(c); return Expr_Expr; } diff --git a/src/check_type.cpp b/src/check_type.cpp index 2ffe04342..4d94fce6c 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2176,70 +2176,36 @@ Type *make_optional_ok_type(Type *value, bool typed) { return t; } -void init_map_entry_type(Type *type) { - GB_ASSERT(type->kind == Type_Map); - if (type->Map.entry_type != nullptr) return; - // NOTE(bill): The preload types may have not been set yet - GB_ASSERT(t_map_hash != nullptr); +// IMPORTANT NOTE(bill): This must match the definition in dynamic_map_internal.odin +enum : i64 { + MAP_CACHE_LINE_LOG2 = 6, + MAP_CACHE_LINE_SIZE = 1 << MAP_CACHE_LINE_LOG2 +}; +GB_STATIC_ASSERT(MAP_CACHE_LINE_SIZE >= 64); +void map_cell_size_and_len(Type *type, i64 *size_, i64 *len_) { + i64 elem_sz = type_size_of(type); - /* - struct { - hash: uintptr, - next: int, - key: Key, - value: Value, + i64 len = 1; + if (0 < elem_sz && elem_sz < MAP_CACHE_LINE_SIZE) { + len = MAP_CACHE_LINE_SIZE / elem_sz; } - */ - Scope *s = create_scope(nullptr, builtin_pkg->scope); - - auto fields = slice_make(permanent_allocator(), 4); - fields[0] = alloc_entity_field(s, make_token_ident(str_lit("hash")), t_uintptr, false, 0, EntityState_Resolved); - fields[1] = alloc_entity_field(s, make_token_ident(str_lit("next")), t_int, false, 1, EntityState_Resolved); - fields[2] = alloc_entity_field(s, make_token_ident(str_lit("key")), type->Map.key, false, 2, EntityState_Resolved); - fields[3] = alloc_entity_field(s, make_token_ident(str_lit("value")), type->Map.value, false, 3, EntityState_Resolved); - - Type *entry_type = alloc_type_struct(); - entry_type->Struct.fields = fields; - entry_type->Struct.tags = gb_alloc_array(permanent_allocator(), String, fields.count); - - type_set_offsets(entry_type); - type->Map.entry_type = entry_type; + i64 size = align_formula(elem_sz * len, MAP_CACHE_LINE_SIZE); + if (size_) *size_ = size; + if (len_) *len_ = len; } void init_map_internal_types(Type *type) { GB_ASSERT(type->kind == Type_Map); - init_map_entry_type(type); - if (type->Map.internal_type != nullptr) return; + GB_ASSERT(t_allocator != nullptr); + if (type->Map.lookup_result_type != nullptr) return; Type *key = type->Map.key; Type *value = type->Map.value; GB_ASSERT(key != nullptr); GB_ASSERT(value != nullptr); - Type *generated_struct_type = alloc_type_struct(); - - /* - struct { - hashes: []int; - entries: [dynamic]EntryType; - } - */ - Scope *s = create_scope(nullptr, builtin_pkg->scope); - - Type *hashes_type = alloc_type_slice(t_int); - Type *entries_type = alloc_type_dynamic_array(type->Map.entry_type); - - - auto fields = slice_make(permanent_allocator(), 2); - fields[0] = alloc_entity_field(s, make_token_ident(str_lit("hashes")), hashes_type, false, 0, EntityState_Resolved); - fields[1] = alloc_entity_field(s, make_token_ident(str_lit("entries")), entries_type, false, 1, EntityState_Resolved); - - generated_struct_type->Struct.fields = fields; - type_set_offsets(generated_struct_type); - - type->Map.internal_type = generated_struct_type; - type->Map.lookup_result_type = make_optional_ok_type(value); + type->Map.lookup_result_type = make_optional_ok_type(value); } void add_map_key_type_dependencies(CheckerContext *ctx, Type *key) { @@ -2255,35 +2221,27 @@ void add_map_key_type_dependencies(CheckerContext *ctx, Type *key) { } if (is_type_simple_compare(key)) { - i64 sz = type_size_of(key); - if (1 <= sz && sz <= 16) { - char buf[20] = {}; - gb_snprintf(buf, 20, "default_hasher%d", cast(i32)sz); - add_package_dependency(ctx, "runtime", buf); - return; - } else { - add_package_dependency(ctx, "runtime", "default_hasher_n"); - return; - } + add_package_dependency(ctx, "runtime", "default_hasher"); + return; } if (key->kind == Type_Struct) { - add_package_dependency(ctx, "runtime", "default_hasher_n"); + add_package_dependency(ctx, "runtime", "default_hasher"); for_array(i, key->Struct.fields) { Entity *field = key->Struct.fields[i]; add_map_key_type_dependencies(ctx, field->type); } } else if (key->kind == Type_Union) { - add_package_dependency(ctx, "runtime", "default_hasher_n"); + add_package_dependency(ctx, "runtime", "default_hasher"); for_array(i, key->Union.variants) { Type *v = key->Union.variants[i]; add_map_key_type_dependencies(ctx, v); } } else if (key->kind == Type_EnumeratedArray) { - add_package_dependency(ctx, "runtime", "default_hasher_n"); + add_package_dependency(ctx, "runtime", "default_hasher"); add_map_key_type_dependencies(ctx, key->EnumeratedArray.elem); } else if (key->kind == Type_Array) { - add_package_dependency(ctx, "runtime", "default_hasher_n"); + add_package_dependency(ctx, "runtime", "default_hasher"); add_map_key_type_dependencies(ctx, key->Array.elem); } } diff --git a/src/checker.cpp b/src/checker.cpp index dd81e2a48..4d1ef4614 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -922,10 +922,13 @@ void init_universal(void) { { Type *equal_args[2] = {t_rawptr, t_rawptr}; - t_equal_proc = alloc_type_proc_from_types(equal_args, 2, t_bool, false, ProcCC_Contextless); + t_equal_proc = alloc_type_proc_from_types(equal_args, gb_count_of(equal_args), t_bool, false, ProcCC_Contextless); Type *hasher_args[2] = {t_rawptr, t_uintptr}; - t_hasher_proc = alloc_type_proc_from_types(hasher_args, 2, t_uintptr, false, ProcCC_Contextless); + t_hasher_proc = alloc_type_proc_from_types(hasher_args, gb_count_of(hasher_args), t_uintptr, false, ProcCC_Contextless); + + Type *map_get_args[3] = {/*map*/t_rawptr, /*hash*/t_uintptr, /*key*/t_rawptr}; + t_map_get_proc = alloc_type_proc_from_types(map_get_args, gb_count_of(map_get_args), t_rawptr, false, ProcCC_Contextless); } // Constants @@ -1933,7 +1936,8 @@ void add_type_info_type_internal(CheckerContext *c, Type *t) { init_map_internal_types(bt); add_type_info_type_internal(c, bt->Map.key); add_type_info_type_internal(c, bt->Map.value); - add_type_info_type_internal(c, bt->Map.internal_type); + add_type_info_type_internal(c, t_uintptr); // hash value + add_type_info_type_internal(c, t_allocator); break; case Type_Tuple: @@ -2155,7 +2159,8 @@ void add_min_dep_type_info(Checker *c, Type *t) { init_map_internal_types(bt); add_min_dep_type_info(c, bt->Map.key); add_min_dep_type_info(c, bt->Map.value); - add_min_dep_type_info(c, bt->Map.internal_type); + add_min_dep_type_info(c, t_uintptr); // hash value + add_min_dep_type_info(c, t_allocator); break; case Type_Tuple: @@ -2838,16 +2843,21 @@ void init_core_source_code_location(Checker *c) { return; } t_source_code_location = find_core_type(c, str_lit("Source_Code_Location")); - t_source_code_location_ptr = alloc_type_pointer(t_allocator); + t_source_code_location_ptr = alloc_type_pointer(t_source_code_location); } void init_core_map_type(Checker *c) { - if (t_map_hash != nullptr) { + if (t_map_info != nullptr) { return; } - t_map_hash = find_core_type(c, str_lit("Map_Hash")); - t_map_header = find_core_type(c, str_lit("Map_Header")); - t_map_header_table = find_core_type(c, str_lit("Map_Header_Table")); + init_mem_allocator(c); + t_map_info = find_core_type(c, str_lit("Map_Info")); + t_map_cell_info = find_core_type(c, str_lit("Map_Cell_Info")); + t_raw_map = find_core_type(c, str_lit("Raw_Map")); + + t_map_info_ptr = alloc_type_pointer(t_map_info); + t_map_cell_info_ptr = alloc_type_pointer(t_map_cell_info); + t_raw_map_ptr = alloc_type_pointer(t_raw_map); } void init_preload(Checker *c) { diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index c59ae7867..e03e94ab4 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -277,6 +277,8 @@ BuiltinProc__type_simple_boolean_end, BuiltinProc_type_equal_proc, BuiltinProc_type_hasher_proc, + BuiltinProc_type_map_info, + BuiltinProc_type_map_cell_info, BuiltinProc__type_end, @@ -570,8 +572,10 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("type_field_index_of"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("type_equal_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("type_hasher_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_equal_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_hasher_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_map_info"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_map_cell_info"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 1d2c00700..594224e6a 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -140,7 +140,7 @@ lbContextData *lb_push_context_onto_stack(lbProcedure *p, lbAddr ctx) { } -lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type) { +lbValue lb_equal_proc_for_type(lbModule *m, Type *type) { type = base_type(type); GB_ASSERT(is_type_comparable(type)); @@ -157,8 +157,8 @@ lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type) { static u32 proc_index = 0; - char buf[16] = {}; - isize n = gb_snprintf(buf, 16, "__$equal%u", ++proc_index); + char buf[32] = {}; + isize n = gb_snprintf(buf, 32, "__$equal%u", ++proc_index); char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); String proc_name = make_string_c(str); @@ -166,6 +166,9 @@ lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type) { map_set(&m->equal_procs, type, p); lb_begin_procedure_body(p); + lb_add_attribute_to_proc(m, p->value, "readonly"); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + LLVMValueRef x = LLVMGetParam(p->value, 0); LLVMValueRef y = LLVMGetParam(p->value, 1); x = LLVMBuildPointerCast(p->builder, x, ptr_type, ""); @@ -173,6 +176,8 @@ lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type) { lbValue lhs = {x, pt}; lbValue rhs = {y, pt}; + lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+1, "nonnull"); lbBlock *block_same_ptr = lb_create_block(p, "same_ptr"); lbBlock *block_diff_ptr = lb_create_block(p, "diff_ptr"); @@ -277,28 +282,20 @@ lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type) { lbValue lb_simple_compare_hash(lbProcedure *p, Type *type, lbValue data, lbValue seed) { GB_ASSERT_MSG(is_type_simple_compare(type), "%s", type_to_string(type)); - i64 sz = type_size_of(type); - - if (1 <= sz && sz <= 16) { - char name[20] = {}; - gb_snprintf(name, 20, "default_hasher%d", cast(i32)sz); - - auto args = array_make(permanent_allocator(), 2); - args[0] = data; - args[1] = seed; - return lb_emit_runtime_call(p, name, args); - } - auto args = array_make(permanent_allocator(), 3); args[0] = data; args[1] = seed; args[2] = lb_const_int(p->module, t_int, type_size_of(type)); - return lb_emit_runtime_call(p, "default_hasher_n", args); + return lb_emit_runtime_call(p, "default_hasher", args); } -lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { +void lb_add_callsite_force_inline(lbProcedure *p, lbValue ret_value) { + LLVMAddCallSiteAttribute(ret_value.value, LLVMAttributeIndex_FunctionIndex, lb_create_enum_attribute(p->module->ctx, "alwaysinline")); +} + +lbValue lb_hasher_proc_for_type(lbModule *m, Type *type) { type = core_type(type); - GB_ASSERT(is_type_valid_for_keys(type)); + GB_ASSERT_MSG(is_type_valid_for_keys(type), "%s", type_to_string(type)); Type *pt = alloc_type_pointer(type); @@ -310,8 +307,8 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { static u32 proc_index = 0; - char buf[16] = {}; - isize n = gb_snprintf(buf, 16, "__$hasher%u", ++proc_index); + char buf[32] = {}; + isize n = gb_snprintf(buf, 32, "__$hasher%u", ++proc_index); char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); String proc_name = make_string_c(str); @@ -320,16 +317,20 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { lb_begin_procedure_body(p); defer (lb_end_procedure_body(p)); + lb_add_attribute_to_proc(m, p->value, "readonly"); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + LLVMValueRef x = LLVMGetParam(p->value, 0); LLVMValueRef y = LLVMGetParam(p->value, 1); lbValue data = {x, t_rawptr}; lbValue seed = {y, t_uintptr}; - LLVMAttributeRef nonnull_attr = lb_create_enum_attribute(m->ctx, "nonnull"); - LLVMAddAttributeAtIndex(p->value, 1+0, nonnull_attr); + lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+0, "readonly"); if (is_type_simple_compare(type)) { lbValue res = lb_simple_compare_hash(p, type, data, seed); + lb_add_callsite_force_inline(p, res); LLVMBuildRet(p->builder, res.value); return {p->value, p->type}; } @@ -343,7 +344,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { GB_ASSERT(type->Struct.offsets != nullptr); i64 offset = type->Struct.offsets[i]; Entity *field = type->Struct.fields[i]; - lbValue field_hasher = lb_get_hasher_proc_for_type(m, field->type); + lbValue field_hasher = lb_hasher_proc_for_type(m, field->type); lbValue ptr = lb_emit_ptr_offset(p, data, lb_const_int(m, t_uintptr, offset)); args[0] = ptr; @@ -356,11 +357,12 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { if (is_type_union_maybe_pointer(type)) { Type *v = type->Union.variants[0]; - lbValue variant_hasher = lb_get_hasher_proc_for_type(m, v); + lbValue variant_hasher = lb_hasher_proc_for_type(m, v); args[0] = data; args[1] = seed; lbValue res = lb_emit_call(p, variant_hasher, args); + lb_add_callsite_force_inline(p, res); LLVMBuildRet(p->builder, res.value); } @@ -379,7 +381,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { Type *v = type->Union.variants[i]; lbValue case_tag = lb_const_union_tag(p->module, type, v); - lbValue variant_hasher = lb_get_hasher_proc_for_type(m, v); + lbValue variant_hasher = lb_hasher_proc_for_type(m, v); args[0] = data; args[1] = seed; @@ -397,7 +399,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { lb_addr_store(p, pres, seed); auto args = array_make(permanent_allocator(), 2); - lbValue elem_hasher = lb_get_hasher_proc_for_type(m, type->Array.elem); + lbValue elem_hasher = lb_hasher_proc_for_type(m, type->Array.elem); auto loop_data = lb_loop_start(p, cast(isize)type->Array.count, t_i32); @@ -418,7 +420,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { lb_addr_store(p, res, seed); auto args = array_make(permanent_allocator(), 2); - lbValue elem_hasher = lb_get_hasher_proc_for_type(m, type->EnumeratedArray.elem); + lbValue elem_hasher = lb_hasher_proc_for_type(m, type->EnumeratedArray.elem); auto loop_data = lb_loop_start(p, cast(isize)type->EnumeratedArray.count, t_i32); @@ -439,12 +441,14 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { args[0] = data; args[1] = seed; lbValue res = lb_emit_runtime_call(p, "default_hasher_cstring", args); + lb_add_callsite_force_inline(p, res); LLVMBuildRet(p->builder, res.value); } else if (is_type_string(type)) { auto args = array_make(permanent_allocator(), 2); args[0] = data; args[1] = seed; lbValue res = lb_emit_runtime_call(p, "default_hasher_string", args); + lb_add_callsite_force_inline(p, res); LLVMBuildRet(p->builder, res.value); } else { GB_PANIC("Unhandled type for hasher: %s", type_to_string(type)); @@ -454,6 +458,279 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { } +lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { + GB_ASSERT(build_context.use_static_map_calls); + type = base_type(type); + GB_ASSERT(type->kind == Type_Map); + + + lbProcedure **found = map_get(&m->map_get_procs, type); + if (found) { + GB_ASSERT(*found != nullptr); + return {(*found)->value, (*found)->type}; + } + static u32 proc_index = 0; + + char buf[32] = {}; + isize n = gb_snprintf(buf, 32, "__$map_get-%u", ++proc_index); + char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); + String proc_name = make_string_c(str); + + lbProcedure *p = lb_create_dummy_procedure(m, proc_name, t_map_get_proc); + map_set(&m->map_get_procs, type, p); + lb_begin_procedure_body(p); + defer (lb_end_procedure_body(p)); + + LLVMSetLinkage(p->value, LLVMInternalLinkage); + lb_add_attribute_to_proc(m, p->value, "readonly"); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + if (build_context.ODIN_DEBUG) { + lb_add_attribute_to_proc(m, p->value, "noinline"); + } + + LLVMValueRef x = LLVMGetParam(p->value, 0); + LLVMValueRef y = LLVMGetParam(p->value, 1); + LLVMValueRef z = LLVMGetParam(p->value, 2); + lbValue map_ptr = {x, t_rawptr}; + lbValue h = {y, t_uintptr}; + lbValue key_ptr = {z, t_rawptr}; + + lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+0, "noalias"); + lb_add_proc_attribute_at_index(p, 1+0, "readonly"); + + lb_add_proc_attribute_at_index(p, 1+2, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+2, "noalias"); + lb_add_proc_attribute_at_index(p, 1+2, "readonly"); + + lbBlock *loop_block = lb_create_block(p, "loop"); + lbBlock *hash_block = lb_create_block(p, "hash"); + lbBlock *probe_block = lb_create_block(p, "probe"); + lbBlock *increment_block = lb_create_block(p, "increment"); + lbBlock *hash_compare_block = lb_create_block(p, "hash_compare"); + lbBlock *key_compare_block = lb_create_block(p, "key_compare"); + lbBlock *value_block = lb_create_block(p, "value"); + lbBlock *nil_block = lb_create_block(p, "nil"); + + map_ptr = lb_emit_conv(p, map_ptr, t_raw_map_ptr); + lbValue map = lb_emit_load(p, map_ptr); + + lbValue length = lb_map_len(p, map); + + lb_emit_if(p, lb_emit_comp(p, Token_CmpEq, length, lb_const_nil(m, t_int)), nil_block, hash_block); + lb_start_block(p, hash_block); + + key_ptr = lb_emit_conv(p, key_ptr, alloc_type_pointer(type->Map.key)); + lbValue key = lb_emit_load(p, key_ptr); + + lbAddr pos = lb_add_local_generated(p, t_uintptr, false); + lbAddr distance = lb_add_local_generated(p, t_uintptr, true); + lbValue capacity = lb_map_cap(p, map); + lbValue mask = lb_emit_conv(p, lb_emit_arith(p, Token_Sub, capacity, lb_const_int(m, t_int, 1), t_int), t_uintptr); + + { + auto args = array_make(heap_allocator(), 2); + args[0] = map; + args[1] = h; + lb_addr_store(p, pos, lb_emit_runtime_call(p, "map_desired_position", args)); + } + lbValue zero_uintptr = lb_const_int(m, t_uintptr, 0); + lbValue one_uintptr = lb_const_int(m, t_uintptr, 1); + + lbValue ks = lb_map_data_uintptr(p, map); + lbValue vs = lb_map_cell_index_static(p, type->Map.key, ks, capacity); + lbValue hs = lb_map_cell_index_static(p, type->Map.value, vs, capacity); + + ks = lb_emit_conv(p, ks, alloc_type_pointer(type->Map.key)); + vs = lb_emit_conv(p, vs, alloc_type_pointer(type->Map.value)); + hs = lb_emit_conv(p, hs, alloc_type_pointer(t_uintptr)); + + lb_emit_jump(p, loop_block); + lb_start_block(p, loop_block); + + lbValue element_hash = lb_emit_load(p, lb_emit_ptr_offset(p, hs, lb_addr_load(p, pos))); + { + // if element_hash == 0 { return nil } + lb_emit_if(p, lb_emit_comp(p, Token_CmpEq, element_hash, zero_uintptr), nil_block, probe_block); + } + + lb_start_block(p, probe_block); + { + auto args = array_make(heap_allocator(), 3); + args[0] = map; + args[1] = element_hash; + args[2] = lb_addr_load(p, pos); + lbValue probe_distance = lb_emit_runtime_call(p, "map_probe_distance", args); + lbValue cond = lb_emit_comp(p, Token_Gt, lb_addr_load(p, distance), probe_distance); + lb_emit_if(p, cond, nil_block, hash_compare_block); + } + + lb_start_block(p, hash_compare_block); + { + lb_emit_if(p, lb_emit_comp(p, Token_CmpEq, element_hash, h), key_compare_block, increment_block); + } + + lb_start_block(p, key_compare_block); + { + lbValue element_key = lb_map_cell_index_static(p, type->Map.key, ks, lb_addr_load(p, pos)); + element_key = lb_emit_conv(p, element_key, ks.type); + lbValue cond = lb_emit_comp(p, Token_CmpEq, lb_emit_load(p, element_key), key); + lb_emit_if(p, cond, value_block, increment_block); + } + + lb_start_block(p, value_block); + { + lbValue element_value = lb_map_cell_index_static(p, type->Map.value, vs, lb_addr_load(p, pos)); + element_value = lb_emit_conv(p, element_value, t_rawptr); + LLVMBuildRet(p->builder, element_value.value); + } + + lb_start_block(p, increment_block); + { + lbValue pp = lb_addr_load(p, pos); + pp = lb_emit_arith(p, Token_Add, pp, one_uintptr, t_uintptr); + pp = lb_emit_arith(p, Token_And, pp, mask, t_uintptr); + lb_addr_store(p, pos, pp); + lb_emit_increment(p, distance.addr); + } + lb_emit_jump(p, loop_block); + + lb_start_block(p, nil_block); + { + lbValue res = lb_const_nil(m, t_rawptr); + LLVMBuildRet(p->builder, res.value); + } + + + return {p->value, p->type}; +} + +void lb_debug_print(lbProcedure *p, String const &str) { + auto args = array_make(heap_allocator(), 1); + args[0] = lb_const_string(p->module, str); + lb_emit_runtime_call(p, "print_string", args); +} + +lbValue lb_map_set_proc_for_type(lbModule *m, Type *type) { + GB_ASSERT(build_context.use_static_map_calls); + type = base_type(type); + GB_ASSERT(type->kind == Type_Map); + + + lbProcedure **found = map_get(&m->map_set_procs, type); + if (found) { + GB_ASSERT(*found != nullptr); + return {(*found)->value, (*found)->type}; + } + static u32 proc_index = 0; + + char buf[32] = {}; + isize n = gb_snprintf(buf, 32, "__$map_set-%u", ++proc_index); + char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); + String proc_name = make_string_c(str); + + lbProcedure *p = lb_create_dummy_procedure(m, proc_name, t_map_set_proc); + map_set(&m->map_set_procs, type, p); + lb_begin_procedure_body(p); + defer (lb_end_procedure_body(p)); + + LLVMSetLinkage(p->value, LLVMInternalLinkage); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + if (build_context.ODIN_DEBUG) { + lb_add_attribute_to_proc(m, p->value, "noinline"); + } + + lbValue map_ptr = {LLVMGetParam(p->value, 0), t_rawptr}; + lbValue hash = {LLVMGetParam(p->value, 1), t_uintptr}; + lbValue key_ptr = {LLVMGetParam(p->value, 2), t_rawptr}; + lbValue value_ptr = {LLVMGetParam(p->value, 3), t_rawptr}; + lbValue location_ptr = {LLVMGetParam(p->value, 4), t_source_code_location_ptr}; + + map_ptr = lb_emit_conv(p, map_ptr, alloc_type_pointer(type)); + key_ptr = lb_emit_conv(p, key_ptr, alloc_type_pointer(type->Map.key)); + + lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+0, "noalias"); + + lb_add_proc_attribute_at_index(p, 1+2, "nonnull"); + if (!are_types_identical(type->Map.key, type->Map.value)) { + lb_add_proc_attribute_at_index(p, 1+2, "noalias"); + } + lb_add_proc_attribute_at_index(p, 1+2, "readonly"); + + lb_add_proc_attribute_at_index(p, 1+3, "nonnull"); + if (!are_types_identical(type->Map.key, type->Map.value)) { + lb_add_proc_attribute_at_index(p, 1+3, "noalias"); + } + lb_add_proc_attribute_at_index(p, 1+3, "readonly"); + + lb_add_proc_attribute_at_index(p, 1+4, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+4, "noalias"); + lb_add_proc_attribute_at_index(p, 1+4, "readonly"); + + //// + lbValue found_ptr = {}; + { + lbValue map_get_proc = lb_map_get_proc_for_type(m, type); + + auto args = array_make(permanent_allocator(), 3); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = hash; + args[2] = key_ptr; + + found_ptr = lb_emit_call(p, map_get_proc, args); + } + + + lbBlock *found_block = lb_create_block(p, "found"); + lbBlock *check_grow_block = lb_create_block(p, "check-grow"); + lbBlock *grow_fail_block = lb_create_block(p, "grow-fail"); + lbBlock *insert_block = lb_create_block(p, "insert"); + + lb_emit_if(p, lb_emit_comp_against_nil(p, Token_NotEq, found_ptr), found_block, check_grow_block); + lb_start_block(p, found_block); + { + lb_mem_copy_non_overlapping(p, found_ptr, value_ptr, lb_const_int(m, t_int, type_size_of(type->Map.value))); + LLVMBuildRet(p->builder, lb_emit_conv(p, found_ptr, t_rawptr).value); + } + lb_start_block(p, check_grow_block); + + + lbValue map_info = lb_gen_map_info_ptr(p->module, type); + + { + auto args = array_make(permanent_allocator(), 3); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = map_info; + args[2] = lb_emit_load(p, location_ptr); + lbValue grow_err = lb_emit_runtime_call(p, "__dynamic_map_check_grow", args); + + lb_emit_if(p, lb_emit_comp_against_nil(p, Token_NotEq, grow_err), grow_fail_block, insert_block); + + lb_start_block(p, grow_fail_block); + LLVMBuildRet(p->builder, LLVMConstNull(lb_type(m, t_rawptr))); + } + + lb_start_block(p, insert_block); + { + auto args = array_make(permanent_allocator(), 5); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = map_info; + args[2] = hash; + args[3] = lb_emit_conv(p, key_ptr, t_uintptr); + args[4] = lb_emit_conv(p, value_ptr, t_uintptr); + + lbValue result = lb_emit_runtime_call(p, "map_insert_hash_dynamic", args); + + lb_emit_increment(p, lb_map_len_ptr(p, map_ptr)); + + LLVMBuildRet(p->builder, lb_emit_conv(p, result, t_rawptr).value); + } + + return {p->value, p->type}; +} + + lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, Ast *expr, lbProcedure *parent) { lbProcedure **found = map_get(&m->gen->anonymous_proc_lits, expr); if (found) { @@ -500,51 +777,60 @@ lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, A return value; } -lbValue lb_gen_map_header_table_internal(lbProcedure *p, Type *map_type) { - lbModule *m = p->module; +lbValue lb_gen_map_cell_info_ptr(lbModule *m, Type *type) { + lbAddr *found = map_get(&m->map_cell_info_map, type); + if (found) { + return found->addr; + } + + i64 size = 0, len = 0; + map_cell_size_and_len(type, &size, &len); + + LLVMValueRef const_values[4] = {}; + const_values[0] = lb_const_int(m, t_uintptr, type_size_of(type)).value; + const_values[1] = lb_const_int(m, t_uintptr, type_align_of(type)).value; + const_values[2] = lb_const_int(m, t_uintptr, size).value; + const_values[3] = lb_const_int(m, t_uintptr, len).value; + LLVMValueRef llvm_res = llvm_const_named_struct(m, t_map_cell_info, const_values, gb_count_of(const_values)); + lbValue res = {llvm_res, t_map_cell_info}; + + lbAddr addr = lb_add_global_generated(m, t_map_cell_info, res, nullptr); + lb_make_global_private_const(addr); + + map_set(&m->map_cell_info_map, type, addr); + + return addr.addr; +} +lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type) { map_type = base_type(map_type); GB_ASSERT(map_type->kind == Type_Map); - lbAddr *found = map_get(&m->map_header_table_map, map_type); + lbAddr *found = map_get(&m->map_info_map, map_type); if (found) { - return lb_addr_load(p, *found); + return found->addr; } - GB_ASSERT(map_type->Map.entry_type->kind == Type_Struct); - i64 entry_size = type_size_of (map_type->Map.entry_type); - i64 entry_align = type_align_of (map_type->Map.entry_type); + GB_ASSERT(t_map_info != nullptr); + GB_ASSERT(t_map_cell_info != nullptr); - i64 key_offset = type_offset_of(map_type->Map.entry_type, 2); - i64 key_size = type_size_of (map_type->Map.key); + LLVMValueRef key_cell_info = lb_gen_map_cell_info_ptr(m, map_type->Map.key).value; + LLVMValueRef value_cell_info = lb_gen_map_cell_info_ptr(m, map_type->Map.value).value; - i64 value_offset = type_offset_of(map_type->Map.entry_type, 3); - i64 value_size = type_size_of (map_type->Map.value); + LLVMValueRef const_values[4] = {}; + const_values[0] = key_cell_info; + const_values[1] = value_cell_info; + const_values[2] = lb_hasher_proc_for_type(m, map_type->Map.key).value; + const_values[3] = lb_equal_proc_for_type(m, map_type->Map.key).value; - Type *key_type = map_type->Map.key; - Type *val_type = map_type->Map.value; - gb_unused(val_type); + LLVMValueRef llvm_res = llvm_const_named_struct(m, t_map_info, const_values, gb_count_of(const_values)); + lbValue res = {llvm_res, t_map_info}; - Type *st = base_type(t_map_header_table); - GB_ASSERT(st->Struct.fields.count == 7); - - LLVMValueRef const_values[7] = {}; - const_values[0] = lb_get_equal_proc_for_type(m, key_type) .value; - const_values[1] = lb_const_int(m, t_int, entry_size) .value; - const_values[2] = lb_const_int(m, t_int, entry_align) .value; - const_values[3] = lb_const_int(m, t_uintptr, key_offset) .value; - const_values[4] = lb_const_int(m, t_int, key_size) .value; - const_values[5] = lb_const_int(m, t_uintptr, value_offset).value; - const_values[6] = lb_const_int(m, t_int, value_size) .value; - - LLVMValueRef llvm_res = llvm_const_named_struct(m, t_map_header_table, const_values, gb_count_of(const_values)); - lbValue res = {llvm_res, t_map_header_table}; - - lbAddr addr = lb_add_global_generated(m, t_map_header_table, res, nullptr); + lbAddr addr = lb_add_global_generated(m, t_map_info, res, nullptr); lb_make_global_private_const(addr); - map_set(&m->map_header_table_map, map_type, addr); - return lb_addr_load(p, addr); + map_set(&m->map_info_map, map_type, addr); + return addr.addr; } lbValue lb_const_hash(lbModule *m, lbValue key, Type *key_type) { @@ -602,7 +888,7 @@ lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue lbValue hashed_key = lb_const_hash(p->module, key, key_type); if (hashed_key.value == nullptr) { - lbValue hasher = lb_get_hasher_proc_for_type(p->module, key_type); + lbValue hasher = lb_hasher_proc_for_type(p->module, key_type); auto args = array_make(permanent_allocator(), 2); args[0] = key_ptr; @@ -615,42 +901,68 @@ lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key) { Type *map_type = base_type(type_deref(map_ptr.type)); + GB_ASSERT(map_type->kind == Type_Map); + lbValue ptr = {}; lbValue key_ptr = {}; - auto args = array_make(permanent_allocator(), 4); - args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_header_table_internal(p, map_type); - args[2] = lb_gen_map_key_hash(p, key, map_type->Map.key, &key_ptr); - args[3] = key_ptr; + lbValue hash = lb_gen_map_key_hash(p, key, map_type->Map.key, &key_ptr); - lbValue ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); + if (build_context.use_static_map_calls) { + lbValue map_get_proc = lb_map_get_proc_for_type(p->module, map_type); + auto args = array_make(permanent_allocator(), 3); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = hash; + args[2] = key_ptr; + + ptr = lb_emit_call(p, map_get_proc, args); + } else { + auto args = array_make(permanent_allocator(), 4); + args[0] = lb_emit_transmute(p, map_ptr, t_raw_map_ptr); + args[1] = lb_gen_map_info_ptr(p->module, map_type); + args[2] = hash; + args[3] = key_ptr; + + ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); + } return lb_emit_conv(p, ptr, alloc_type_pointer(map_type->Map.value)); } -void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, Type *map_type, - lbValue const &map_key, lbValue const &map_value, Ast *node) { +void lb_internal_dynamic_map_set(lbProcedure *p, lbValue const &map_ptr, Type *map_type, + lbValue const &map_key, lbValue const &map_value, Ast *node) { map_type = base_type(map_type); GB_ASSERT(map_type->kind == Type_Map); lbValue key_ptr = {}; - lbValue key_hash = lb_gen_map_key_hash(p, map_key, map_type->Map.key, &key_ptr); + lbValue hash = lb_gen_map_key_hash(p, map_key, map_type->Map.key, &key_ptr); + lbValue v = lb_emit_conv(p, map_value, map_type->Map.value); + lbValue value_ptr = lb_address_from_load_or_generate_local(p, v); - lbAddr value_addr = lb_add_local_generated(p, v.type, false); - lb_addr_store(p, value_addr, v); + if (build_context.use_static_map_calls) { + lbValue map_set_proc = lb_map_set_proc_for_type(p->module, map_type); - auto args = array_make(permanent_allocator(), 6); - args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_header_table_internal(p, map_type); - args[2] = key_hash; - args[3] = key_ptr; - args[4] = lb_emit_conv(p, value_addr.addr, t_rawptr); - args[5] = lb_emit_source_code_location_as_global(p, node); - lb_emit_runtime_call(p, "__dynamic_map_set", args); + auto args = array_make(permanent_allocator(), 5); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = hash; + args[2] = lb_emit_conv(p, key_ptr, t_rawptr); + args[3] = lb_emit_conv(p, value_ptr, t_rawptr); + args[4] = lb_emit_source_code_location_as_global(p, node); + + lb_emit_call(p, map_set_proc, args); + } else { + auto args = array_make(permanent_allocator(), 6); + args[0] = lb_emit_conv(p, map_ptr, t_raw_map_ptr); + args[1] = lb_gen_map_info_ptr(p->module, map_type); + args[2] = hash; + args[3] = lb_emit_conv(p, key_ptr, t_rawptr); + args[4] = lb_emit_conv(p, value_ptr, t_rawptr); + args[5] = lb_emit_source_code_location_as_global(p, node); + lb_emit_runtime_call(p, "__dynamic_map_set", args); + } } -void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos) { +lbValue lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos) { GB_ASSERT(!build_context.no_dynamic_literals); String proc_name = {}; @@ -660,10 +972,10 @@ void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const auto args = array_make(permanent_allocator(), 4); args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_header_table_internal(p, type_deref(map_ptr.type)); - args[2] = lb_const_int(p->module, t_int, capacity); + args[1] = lb_gen_map_info_ptr(p->module, type_deref(map_ptr.type)); + args[2] = lb_const_int(p->module, t_uint, capacity); args[3] = lb_emit_source_code_location_as_global(p, proc_name, pos); - lb_emit_runtime_call(p, "__dynamic_map_reserve", args); + return lb_emit_runtime_call(p, "__dynamic_map_reserve", args); } @@ -688,6 +1000,8 @@ lbProcedure *lb_create_startup_type_info(lbModule *m) { p->is_startup = true; LLVMSetLinkage(p->value, LLVMInternalLinkage); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + lb_begin_procedure_body(p); lb_setup_type_info_data(p); @@ -712,6 +1026,7 @@ lbProcedure *lb_create_objc_names(lbModule *main_module) { } Type *proc_type = alloc_type_proc(nullptr, nullptr, 0, nullptr, 0, false, ProcCC_CDecl); lbProcedure *p = lb_create_dummy_procedure(main_module, str_lit("__$init_objc_names"), proc_type); + lb_add_attribute_to_proc(p->module, p->value, "nounwind"); p->is_startup = true; return p; } @@ -1198,6 +1513,14 @@ WORKER_TASK_PROC(lb_llvm_function_pass_worker_proc) { lbProcedure *p = m->hasher_procs.entries[i].value; lb_run_function_pass_manager(default_function_pass_manager, p); } + for_array(i, m->map_get_procs.entries) { + lbProcedure *p = m->map_get_procs.entries[i].value; + lb_run_function_pass_manager(default_function_pass_manager, p); + } + for_array(i, m->map_set_procs.entries) { + lbProcedure *p = m->map_set_procs.entries[i].value; + lb_run_function_pass_manager(default_function_pass_manager, p); + } return 0; } diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index a8ff1571c..72dfdda54 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -144,6 +144,8 @@ struct lbModule { PtrMap equal_procs; PtrMap hasher_procs; + PtrMap map_get_procs; + PtrMap map_set_procs; u32 nested_type_name_guid; @@ -160,7 +162,8 @@ struct lbModule { StringMap objc_classes; StringMap objc_selectors; - PtrMap map_header_table_map; + PtrMap map_cell_info_map; // address of runtime.Map_Info + PtrMap map_info_map; // address of runtime.Map_Cell_Info }; struct lbGenerator { @@ -422,8 +425,6 @@ lbValue lb_dynamic_array_elem(lbProcedure *p, lbValue da); lbValue lb_dynamic_array_len(lbProcedure *p, lbValue da); lbValue lb_dynamic_array_cap(lbProcedure *p, lbValue da); lbValue lb_dynamic_array_allocator(lbProcedure *p, lbValue da); -lbValue lb_map_entries(lbProcedure *p, lbValue value); -lbValue lb_map_entries_ptr(lbProcedure *p, lbValue value); lbValue lb_map_len(lbProcedure *p, lbValue value); lbValue lb_map_cap(lbProcedure *p, lbValue value); lbValue lb_soa_struct_len(lbProcedure *p, lbValue value); @@ -447,10 +448,12 @@ String lb_get_const_string(lbModule *m, lbValue value); lbValue lb_generate_local_array(lbProcedure *p, Type *elem_type, i64 count, bool zero_init=true); lbValue lb_generate_global_array(lbModule *m, Type *elem_type, i64 count, String prefix, i64 id); lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue *key_ptr_); +lbValue lb_gen_map_cell_info_ptr(lbModule *m, Type *type); +lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type); lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key); -void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, Type *map_type, lbValue const &map_key, lbValue const &map_value, Ast *node); -void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos); +void lb_internal_dynamic_map_set(lbProcedure *p, lbValue const &map_ptr, Type *map_type, lbValue const &map_key, lbValue const &map_value, Ast *node); +lbValue lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos); lbValue lb_find_procedure_value_from_entity(lbModule *m, Entity *e); lbValue lb_find_value_from_entity(lbModule *m, Entity *e); @@ -461,8 +464,8 @@ lbValue lb_emit_source_code_location_const(lbProcedure *p, String const &procedu lbValue lb_handle_param_value(lbProcedure *p, Type *parameter_type, ParameterValue const ¶m_value, TokenPos const &pos); -lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type); -lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type); +lbValue lb_equal_proc_for_type(lbModule *m, Type *type); +lbValue lb_hasher_proc_for_type(lbModule *m, Type *type); lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t); LLVMMetadataRef lb_debug_type(lbModule *m, Type *type); diff --git a/src/llvm_backend_const.cpp b/src/llvm_backend_const.cpp index a8b66a0ea..dff5298c5 100644 --- a/src/llvm_backend_const.cpp +++ b/src/llvm_backend_const.cpp @@ -283,19 +283,28 @@ lbValue lb_emit_source_code_location_const(lbProcedure *p, Ast *node) { return lb_emit_source_code_location_const(p, proc_name, pos); } -lbValue lb_emit_source_code_location_as_global(lbProcedure *p, String const &procedure, TokenPos const &pos) { + +lbValue lb_emit_source_code_location_as_global_ptr(lbProcedure *p, String const &procedure, TokenPos const &pos) { lbValue loc = lb_emit_source_code_location_const(p, procedure, pos); lbAddr addr = lb_add_global_generated(p->module, loc.type, loc, nullptr); lb_make_global_private_const(addr); - return lb_addr_load(p, addr); + return addr.addr; } -lbValue lb_emit_source_code_location_as_global(lbProcedure *p, Ast *node) { +lbValue lb_emit_source_code_location_as_global_ptr(lbProcedure *p, Ast *node) { lbValue loc = lb_emit_source_code_location_const(p, node); lbAddr addr = lb_add_global_generated(p->module, loc.type, loc, nullptr); lb_make_global_private_const(addr); - return lb_addr_load(p, addr); + return addr.addr; +} + +lbValue lb_emit_source_code_location_as_global(lbProcedure *p, String const &procedure, TokenPos const &pos) { + return lb_emit_load(p, lb_emit_source_code_location_as_global_ptr(p, procedure, pos)); +} + +lbValue lb_emit_source_code_location_as_global(lbProcedure *p, Ast *node) { + return lb_emit_load(p, lb_emit_source_code_location_as_global_ptr(p, node)); } diff --git a/src/llvm_backend_debug.cpp b/src/llvm_backend_debug.cpp index 8a98b7f39..e69424929 100644 --- a/src/llvm_backend_debug.cpp +++ b/src/llvm_backend_debug.cpp @@ -671,7 +671,8 @@ void lb_debug_complete_types(lbModule *m) { break; case Type_Map: - bt = bt->Map.internal_type; + GB_ASSERT(t_raw_map != nullptr); + bt = base_type(t_raw_map); /*fallthrough*/ case Type_Struct: if (file == nullptr) { diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 05a9fdfbf..034682855 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -2215,7 +2215,7 @@ lbValue lb_compare_records(lbProcedure *p, TokenKind op_kind, lbValue left, lbVa args[2] = lb_const_int(p->module, t_int, type_size_of(type)); res = lb_emit_runtime_call(p, "memory_equal", args); } else { - lbValue value = lb_get_equal_proc_for_type(p->module, type); + lbValue value = lb_equal_proc_for_type(p->module, type); auto args = array_make(permanent_allocator(), 2); args[0] = lb_emit_conv(p, left_ptr, t_rawptr); args[1] = lb_emit_conv(p, right_ptr, t_rawptr); @@ -4131,7 +4131,8 @@ lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { } GB_ASSERT(!build_context.no_dynamic_literals); - lb_dynamic_map_reserve(p, v.addr, 2*cl->elems.count, pos); + lbValue err = lb_dynamic_map_reserve(p, v.addr, 2*cl->elems.count, pos); + gb_unused(err); for_array(field_index, cl->elems) { Ast *elem = cl->elems[field_index]; @@ -4139,7 +4140,7 @@ lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { lbValue key = lb_build_expr(p, fv->field); lbValue value = lb_build_expr(p, fv->value); - lb_insert_dynamic_map_key_and_value(p, v.addr, type, key, value, elem); + lb_internal_dynamic_map_set(p, v.addr, type, key, value, elem); } break; } diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 87f8afa05..f36dc1842 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -67,6 +67,8 @@ void lb_init_module(lbModule *m, Checker *c) { map_init(&m->function_type_map, a); map_init(&m->equal_procs, a); map_init(&m->hasher_procs, a); + map_init(&m->map_get_procs, a); + map_init(&m->map_set_procs, a); array_init(&m->procedures_to_generate, a, 0, 1024); array_init(&m->missing_procedures_to_check, a, 0, 16); map_init(&m->debug_values, a); @@ -75,7 +77,8 @@ void lb_init_module(lbModule *m, Checker *c) { string_map_init(&m->objc_classes, a); string_map_init(&m->objc_selectors, a); - map_init(&m->map_header_table_map, a, 0); + map_init(&m->map_info_map, a, 0); + map_init(&m->map_cell_info_map, a, 0); } @@ -725,7 +728,7 @@ void lb_addr_store(lbProcedure *p, lbAddr addr, lbValue value) { return; } else if (addr.kind == lbAddr_Map) { - lb_insert_dynamic_map_key_and_value(p, addr.addr, addr.map.type, addr.map.key, value, p->curr_stmt); + lb_internal_dynamic_map_set(p, addr.addr, addr.map.type, addr.map.key, value, p->curr_stmt); return; } else if (addr.kind == lbAddr_Context) { lbAddr old_addr = lb_find_or_generate_context_ptr(p); @@ -1931,38 +1934,8 @@ LLVMTypeRef lb_type_internal(lbModule *m, Type *type) { case Type_Map: init_map_internal_types(type); - { - Type *internal_type = type->Map.internal_type; - GB_ASSERT(internal_type->kind == Type_Struct); - - m->internal_type_level -= 1; - defer (m->internal_type_level += 1); - - unsigned field_count = cast(unsigned)(internal_type->Struct.fields.count); - GB_ASSERT(field_count == 2); - LLVMTypeRef *fields = gb_alloc_array(temporary_allocator(), LLVMTypeRef, field_count); - - LLVMTypeRef entries_fields[] = { - lb_type(m, t_rawptr), // data - lb_type(m, t_int), // len - lb_type(m, t_int), // cap - lb_type(m, t_allocator), // allocator - }; - - fields[0] = lb_type(m, internal_type->Struct.fields[0]->type); - fields[1] = LLVMStructTypeInContext(ctx, entries_fields, gb_count_of(entries_fields), false); - - { // Add this to simplify things - lbStructFieldRemapping entries_field_remapping = {}; - slice_init(&entries_field_remapping, permanent_allocator(), gb_count_of(entries_fields)); - for_array(i, entries_field_remapping) { - entries_field_remapping[i] = cast(i32)i; - } - map_set(&m->struct_field_remapping, cast(void *)fields[1], entries_field_remapping); - } - - return LLVMStructTypeInContext(ctx, fields, field_count, false); - } + GB_ASSERT(t_raw_map != nullptr); + return lb_type_internal(m, t_raw_map); case Type_Struct: { diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index ae8236a71..7c83125ca 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -586,6 +586,7 @@ void lb_begin_procedure_body(lbProcedure *p) { // defer x = ... // defer is executed after the `defer` // return // the values returned should be zeroed // } + // NOTE(bill): REALLY, don't even bother. lbAddr res = lb_add_local(p, e->type, e); if (e->Variable.param_value.kind != ParameterValue_Invalid) { lbValue c = lb_handle_param_value(p, e->type, e->Variable.param_value, e->token.pos); @@ -2319,10 +2320,17 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, case BuiltinProc_type_equal_proc: - return lb_get_equal_proc_for_type(p->module, ce->args[0]->tav.type); + return lb_equal_proc_for_type(p->module, ce->args[0]->tav.type); case BuiltinProc_type_hasher_proc: - return lb_get_hasher_proc_for_type(p->module, ce->args[0]->tav.type); + return lb_hasher_proc_for_type(p->module, ce->args[0]->tav.type); + + case BuiltinProc_type_map_info: + return lb_gen_map_info_ptr(p->module, ce->args[0]->tav.type); + + case BuiltinProc_type_map_cell_info: + return lb_gen_map_cell_info_ptr(p->module, ce->args[0]->tav.type); + case BuiltinProc_fixed_point_mul: case BuiltinProc_fixed_point_div: diff --git a/src/llvm_backend_stmt.cpp b/src/llvm_backend_stmt.cpp index bd622d411..c8f244181 100644 --- a/src/llvm_backend_stmt.cpp +++ b/src/llvm_backend_stmt.cpp @@ -354,16 +354,6 @@ void lb_build_range_indexed(lbProcedure *p, lbValue expr, Type *val_type, lbValu } break; } - case Type_Map: { - lbValue entries = lb_map_entries_ptr(p, expr); - lbValue elem = lb_emit_struct_ep(p, entries, 0); - elem = lb_emit_load(p, elem); - lbValue entry = lb_emit_ptr_offset(p, elem, idx); - idx = lb_emit_load(p, lb_emit_struct_ep(p, entry, 2)); - val = lb_emit_load(p, lb_emit_struct_ep(p, entry, 3)); - - break; - } case Type_Struct: { GB_ASSERT(is_type_soa_struct(expr_type)); break; @@ -380,6 +370,129 @@ void lb_build_range_indexed(lbProcedure *p, lbValue expr, Type *val_type, lbValu if (done_) *done_ = done; } +lbValue lb_map_cell_index_static(lbProcedure *p, Type *type, lbValue cells_ptr, lbValue index) { + i64 size, len; + i64 elem_sz = type_size_of(type); + map_cell_size_and_len(type, &size, &len); + + index = lb_emit_conv(p, index, t_uintptr); + + if (size == len*elem_sz) { + lbValue elems_ptr = lb_emit_conv(p, cells_ptr, alloc_type_pointer(type)); + return lb_emit_ptr_offset(p, elems_ptr, index); + } + + lbValue cell_index = {}; + lbValue data_index = {}; + + lbValue size_const = lb_const_int(p->module, t_uintptr, size); + lbValue len_const = lb_const_int(p->module, t_uintptr, len); + + if (is_power_of_two(len)) { + u64 log2_len = floor_log2(cast(u64)len); + cell_index = log2_len == 0 ? index : lb_emit_arith(p, Token_Shr, index, lb_const_int(p->module, t_uintptr, log2_len), t_uintptr); + data_index = lb_emit_arith(p, Token_And, index, lb_const_int(p->module, t_uintptr, len-1), t_uintptr); + } else { + cell_index = lb_emit_arith(p, Token_Quo, index, len_const, t_uintptr); + data_index = lb_emit_arith(p, Token_Mod, index, len_const, t_uintptr); + } + + lbValue elems_ptr = lb_emit_conv(p, cells_ptr, t_uintptr); + lbValue cell_offset = lb_emit_arith(p, Token_Mul, size_const, cell_index, t_uintptr); + elems_ptr = lb_emit_arith(p, Token_Add, elems_ptr, cell_offset, t_uintptr); + + elems_ptr = lb_emit_conv(p, elems_ptr, alloc_type_pointer(type)); + + return lb_emit_ptr_offset(p, elems_ptr, data_index); +} + +void lb_map_kvh_data_static(lbProcedure *p, lbValue map_value, lbValue *ks_, lbValue *vs_, lbValue *hs_) { + lbValue capacity = lb_map_cap(p, map_value); + lbValue ks = lb_map_data_uintptr(p, map_value); + lbValue vs = {}; + lbValue hs = {}; + if (ks_) *ks_ = ks; + if (vs_) *vs_ = vs; + if (hs_) *hs_ = hs; +} + +lbValue lb_map_hash_is_valid(lbProcedure *p, lbValue hash) { + // N :: size_of(uintptr)*8 - 1 + // (hash != 0) & (hash>>N == 0) + + u64 top_bit_index = cast(u64)(type_size_of(t_uintptr)*8 - 1); + lbValue shift_amount = lb_const_int(p->module, t_uintptr, top_bit_index); + lbValue zero = lb_const_int(p->module, t_uintptr, 0); + + lbValue not_empty = lb_emit_comp(p, Token_NotEq, hash, zero); + + lbValue not_deleted = lb_emit_arith(p, Token_Shr, hash, shift_amount, t_uintptr); + not_deleted = lb_emit_comp(p, Token_CmpEq, not_deleted, zero); + + return lb_emit_arith(p, Token_And, not_deleted, not_empty, t_uintptr); +} + +void lb_build_range_map(lbProcedure *p, lbValue expr, Type *val_type, + lbValue *val_, lbValue *key_, lbBlock **loop_, lbBlock **done_) { + lbModule *m = p->module; + + Type *type = base_type(type_deref(expr.type)); + GB_ASSERT(type->kind == Type_Map); + + lbValue idx = {}; + lbBlock *loop = nullptr; + lbBlock *done = nullptr; + lbBlock *body = nullptr; + lbBlock *hash_check = nullptr; + + + lbAddr index = lb_add_local_generated(p, t_int, false); + lb_addr_store(p, index, lb_const_int(m, t_int, cast(u64)-1)); + + loop = lb_create_block(p, "for.index.loop"); + lb_emit_jump(p, loop); + lb_start_block(p, loop); + + lbValue incr = lb_emit_arith(p, Token_Add, lb_addr_load(p, index), lb_const_int(m, t_int, 1), t_int); + lb_addr_store(p, index, incr); + + hash_check = lb_create_block(p, "for.index.hash_check"); + body = lb_create_block(p, "for.index.body"); + done = lb_create_block(p, "for.index.done"); + + lbValue map_value = lb_emit_load(p, expr); + lbValue capacity = lb_map_cap(p, map_value); + lbValue cond = lb_emit_comp(p, Token_Lt, incr, capacity); + lb_emit_if(p, cond, hash_check, done); + lb_start_block(p, hash_check); + + idx = lb_addr_load(p, index); + + lbValue ks = lb_map_data_uintptr(p, map_value); + lbValue vs = lb_emit_conv(p, lb_map_cell_index_static(p, type->Map.key, ks, capacity), alloc_type_pointer(type->Map.value)); + lbValue hs = lb_emit_conv(p, lb_map_cell_index_static(p, type->Map.value, vs, capacity), alloc_type_pointer(t_uintptr)); + + // NOTE(bill): no need to use lb_map_cell_index_static for that hashes + // since it will always be packed without padding into the cells + lbValue hash = lb_emit_load(p, lb_emit_ptr_offset(p, hs, idx)); + + lbValue hash_cond = lb_map_hash_is_valid(p, hash); + lb_emit_if(p, hash_cond, body, loop); + lb_start_block(p, body); + + + lbValue key_ptr = lb_map_cell_index_static(p, type->Map.key, ks, idx); + lbValue val_ptr = lb_map_cell_index_static(p, type->Map.value, vs, idx); + lbValue key = lb_emit_load(p, key_ptr); + lbValue val = lb_emit_load(p, val_ptr); + + if (val_) *val_ = val; + if (key_) *key_ = key; + if (loop_) *loop_ = loop; + if (done_) *done_ = done; +} + + void lb_build_range_string(lbProcedure *p, lbValue expr, Type *val_type, lbValue *val_, lbValue *idx_, lbBlock **loop_, lbBlock **done_) { @@ -749,9 +862,7 @@ void lb_build_range_stmt(lbProcedure *p, AstRangeStmt *rs, Scope *scope) { if (is_type_pointer(type_deref(map.type))) { map = lb_emit_load(p, map); } - lbValue entries_ptr = lb_map_entries_ptr(p, map); - lbValue count_ptr = lb_emit_struct_ep(p, entries_ptr, 1); - lb_build_range_indexed(p, map, val1_type, count_ptr, &val, &key, &loop, &done); + lb_build_range_map(p, map, val1_type, &val, &key, &loop, &done); break; } case Type_Array: { diff --git a/src/llvm_backend_type.cpp b/src/llvm_backend_type.cpp index 4abf674c5..307d9304b 100644 --- a/src/llvm_backend_type.cpp +++ b/src/llvm_backend_type.cpp @@ -666,7 +666,7 @@ void lb_setup_type_info_data(lbProcedure *p) { // NOTE(bill): Setup type_info da } if (is_type_comparable(t) && !is_type_simple_compare(t)) { - vals[3] = lb_get_equal_proc_for_type(m, t).value; + vals[3] = lb_equal_proc_for_type(m, t).value; } vals[4] = lb_const_bool(m, t_bool, t->Union.custom_align != 0).value; @@ -702,7 +702,7 @@ void lb_setup_type_info_data(lbProcedure *p) { // NOTE(bill): Setup type_info da vals[6] = is_raw_union.value; vals[7] = is_custom_align.value; if (is_type_comparable(t) && !is_type_simple_compare(t)) { - vals[8] = lb_get_equal_proc_for_type(m, t).value; + vals[8] = lb_equal_proc_for_type(m, t).value; } @@ -788,15 +788,11 @@ void lb_setup_type_info_data(lbProcedure *p) { // NOTE(bill): Setup type_info da case Type_Map: { tag = lb_const_ptr_cast(m, variant_ptr, t_type_info_map_ptr); init_map_internal_types(t); - - lbValue gst = lb_type_info(m, t->Map.internal_type); - LLVMValueRef vals[5] = { + LLVMValueRef vals[3] = { lb_type_info(m, t->Map.key).value, lb_type_info(m, t->Map.value).value, - gst.value, - lb_get_equal_proc_for_type(m, t->Map.key).value, - lb_get_hasher_proc_for_type(m, t->Map.key).value + lb_gen_map_info_ptr(p->module, t).value }; lbValue res = {}; diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index a54171b51..101b9dbfb 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -203,26 +203,19 @@ lbValue lb_emit_transmute(lbProcedure *p, lbValue value, Type *t) { if (is_type_uintptr(src) && is_type_internally_pointer_like(dst)) { res.value = LLVMBuildIntToPtr(p->builder, value.value, lb_type(m, t), ""); return res; - } - if (is_type_internally_pointer_like(src) && is_type_uintptr(dst)) { + } else if (is_type_internally_pointer_like(src) && is_type_uintptr(dst)) { res.value = LLVMBuildPtrToInt(p->builder, value.value, lb_type(m, t), ""); return res; - } - - if (is_type_integer(src) && is_type_internally_pointer_like(dst)) { + } else if (is_type_integer(src) && is_type_internally_pointer_like(dst)) { res.value = LLVMBuildIntToPtr(p->builder, value.value, lb_type(m, t), ""); return res; } else if (is_type_internally_pointer_like(src) && is_type_integer(dst)) { res.value = LLVMBuildPtrToInt(p->builder, value.value, lb_type(m, t), ""); return res; - } - - if (is_type_internally_pointer_like(src) && is_type_internally_pointer_like(dst)) { + } else if (is_type_internally_pointer_like(src) && is_type_internally_pointer_like(dst)) { res.value = LLVMBuildPointerCast(p->builder, value.value, lb_type(p->module, t), ""); return res; - } - - if (is_type_simd_vector(src) && is_type_simd_vector(dst)) { + } else if (is_type_simd_vector(src) && is_type_simd_vector(dst)) { res.value = LLVMBuildBitCast(p->builder, value.value, lb_type(p->module, t), ""); return res; } else if (is_type_array_like(src) && is_type_simd_vector(dst)) { @@ -239,9 +232,11 @@ lbValue lb_emit_transmute(lbProcedure *p, lbValue value, Type *t) { ap = lb_emit_conv(p, ap, alloc_type_pointer(value.type)); lb_emit_store(p, ap, value); return lb_addr_load(p, addr); - } - - if (lb_is_type_aggregate(src) || lb_is_type_aggregate(dst)) { + } else if (is_type_map(src) && are_types_identical(t_raw_map, t)) { + res.value = value.value; + res.type = t; + return res; + } else if (lb_is_type_aggregate(src) || lb_is_type_aggregate(dst)) { lbValue s = lb_address_from_load_or_generate_local(p, value); lbValue d = lb_emit_transmute(p, s, alloc_type_pointer(t)); return lb_emit_load(p, d); @@ -990,14 +985,13 @@ lbValue lb_emit_struct_ep(lbProcedure *p, lbValue s, i32 index) { } } else if (is_type_map(t)) { init_map_internal_types(t); - Type *itp = alloc_type_pointer(t->Map.internal_type); + Type *itp = alloc_type_pointer(t_raw_map); s = lb_emit_transmute(p, s, itp); - Type *gst = t->Map.internal_type; - GB_ASSERT(gst->kind == Type_Struct); switch (index) { - case 0: result_type = get_struct_field_type(gst, 0); break; - case 1: result_type = get_struct_field_type(gst, 1); break; + case 0: result_type = get_struct_field_type(t_raw_map, 0); break; + case 1: result_type = get_struct_field_type(t_raw_map, 1); break; + case 2: result_type = get_struct_field_type(t_raw_map, 2); break; } } else if (is_type_array(t)) { return lb_emit_array_epi(p, s, index); @@ -1130,10 +1124,10 @@ lbValue lb_emit_struct_ev(lbProcedure *p, lbValue s, i32 index) { case Type_Map: { init_map_internal_types(t); - Type *gst = t->Map.internal_type; switch (index) { - case 0: result_type = get_struct_field_type(gst, 0); break; - case 1: result_type = get_struct_field_type(gst, 1); break; + case 0: result_type = get_struct_field_type(t_raw_map, 0); break; + case 1: result_type = get_struct_field_type(t_raw_map, 1); break; + case 2: result_type = get_struct_field_type(t_raw_map, 2); break; } } break; @@ -1439,34 +1433,47 @@ lbValue lb_dynamic_array_allocator(lbProcedure *p, lbValue da) { return lb_emit_struct_ev(p, da, 3); } -lbValue lb_map_entries(lbProcedure *p, lbValue value) { - Type *t = base_type(value.type); - GB_ASSERT_MSG(t->kind == Type_Map, "%s", type_to_string(t)); - init_map_internal_types(t); - i32 index = 1; - lbValue entries = lb_emit_struct_ev(p, value, index); - return entries; -} - -lbValue lb_map_entries_ptr(lbProcedure *p, lbValue value) { - Type *t = base_type(type_deref(value.type)); - GB_ASSERT_MSG(t->kind == Type_Map, "%s", type_to_string(t)); - init_map_internal_types(t); - i32 index = 1; - lbValue entries = lb_emit_struct_ep(p, value, index); - return entries; -} - lbValue lb_map_len(lbProcedure *p, lbValue value) { - lbValue entries = lb_map_entries(p, value); - return lb_dynamic_array_len(p, entries); + GB_ASSERT_MSG(is_type_map(value.type) || are_types_identical(value.type, t_raw_map), "%s", type_to_string(value.type)); + lbValue len = lb_emit_struct_ev(p, value, 1); + return lb_emit_conv(p, len, t_int); +} +lbValue lb_map_len_ptr(lbProcedure *p, lbValue map_ptr) { + Type *type = map_ptr.type; + GB_ASSERT(is_type_pointer(type)); + type = type_deref(type); + GB_ASSERT_MSG(is_type_map(type) || are_types_identical(type, t_raw_map), "%s", type_to_string(type)); + return lb_emit_struct_ep(p, map_ptr, 1); } lbValue lb_map_cap(lbProcedure *p, lbValue value) { - lbValue entries = lb_map_entries(p, value); - return lb_dynamic_array_cap(p, entries); + GB_ASSERT_MSG(is_type_map(value.type) || are_types_identical(value.type, t_raw_map), "%s", type_to_string(value.type)); + lbValue zero = lb_const_int(p->module, t_uintptr, 0); + lbValue one = lb_const_int(p->module, t_uintptr, 1); + + lbValue mask = lb_const_int(p->module, t_uintptr, MAP_CACHE_LINE_SIZE-1); + + lbValue data = lb_emit_struct_ev(p, value, 0); + lbValue log2_cap = lb_emit_arith(p, Token_And, data, mask, t_uintptr); + lbValue cap = lb_emit_arith(p, Token_Shl, one, log2_cap, t_uintptr); + lbValue cmp = lb_emit_comp(p, Token_CmpEq, data, zero); + return lb_emit_conv(p, lb_emit_select(p, cmp, zero, cap), t_int); } +lbValue lb_map_data_uintptr(lbProcedure *p, lbValue value) { + GB_ASSERT(is_type_map(value.type) || are_types_identical(value.type, t_raw_map)); + lbValue data = lb_emit_struct_ev(p, value, 0); + u64 mask_value = 0; + if (build_context.word_size == 4) { + mask_value = 0xfffffffful & ~(MAP_CACHE_LINE_SIZE-1); + } else { + mask_value = 0xffffffffffffffffull & ~(MAP_CACHE_LINE_SIZE-1); + } + lbValue mask = lb_const_int(p->module, t_uintptr, mask_value); + return lb_emit_arith(p, Token_And, data, mask, t_uintptr); +} + + lbValue lb_soa_struct_len(lbProcedure *p, lbValue value) { Type *t = base_type(value.type); bool is_ptr = false; diff --git a/src/main.cpp b/src/main.cpp index b75137613..3b0a599db 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -638,6 +638,7 @@ enum BuildFlagKind { BuildFlag_StrictStyleInitOnly, BuildFlag_ForeignErrorProcedures, BuildFlag_DisallowRTTI, + BuildFlag_UseStaticMapCalls, BuildFlag_Compact, BuildFlag_GlobalDefinitions, @@ -814,6 +815,8 @@ bool parse_build_flags(Array args) { add_flag(&build_flags, BuildFlag_DisallowRTTI, str_lit("disallow-rtti"), BuildFlagParam_None, Command__does_check); + add_flag(&build_flags, BuildFlag_UseStaticMapCalls, str_lit("use-static-map-calls"), BuildFlagParam_None, Command__does_check); + add_flag(&build_flags, BuildFlag_Compact, str_lit("compact"), BuildFlagParam_None, Command_query); add_flag(&build_flags, BuildFlag_GlobalDefinitions, str_lit("global-definitions"), BuildFlagParam_None, Command_query); @@ -1414,6 +1417,9 @@ bool parse_build_flags(Array args) { case BuildFlag_DisallowRTTI: build_context.disallow_rtti = true; break; + case BuildFlag_UseStaticMapCalls: + build_context.use_static_map_calls = true; + break; case BuildFlag_DefaultToNilAllocator: build_context.ODIN_DEFAULT_TO_NIL_ALLOCATOR = true; break; diff --git a/src/types.cpp b/src/types.cpp index f9470ce2b..b7bfe1b0f 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -226,8 +226,6 @@ struct TypeProc { TYPE_KIND(Map, struct { \ Type *key; \ Type *value; \ - Type *entry_type; \ - Type *internal_type; \ Type *lookup_result_type; \ }) \ TYPE_KIND(Struct, TypeStruct) \ @@ -685,13 +683,18 @@ gb_global Type *t_allocator_error = nullptr; gb_global Type *t_source_code_location = nullptr; gb_global Type *t_source_code_location_ptr = nullptr; -gb_global Type *t_map_hash = nullptr; -gb_global Type *t_map_header = nullptr; -gb_global Type *t_map_header_table = nullptr; +gb_global Type *t_map_info = nullptr; +gb_global Type *t_map_cell_info = nullptr; +gb_global Type *t_raw_map = nullptr; +gb_global Type *t_map_info_ptr = nullptr; +gb_global Type *t_map_cell_info_ptr = nullptr; +gb_global Type *t_raw_map_ptr = nullptr; gb_global Type *t_equal_proc = nullptr; gb_global Type *t_hasher_proc = nullptr; +gb_global Type *t_map_get_proc = nullptr; +gb_global Type *t_map_set_proc = nullptr; gb_global Type *t_objc_object = nullptr; gb_global Type *t_objc_selector = nullptr; @@ -1926,7 +1929,7 @@ bool is_type_valid_for_keys(Type *t) { if (is_type_untyped(t)) { return false; } - return is_type_comparable(t); + return type_size_of(t) > 0 && is_type_comparable(t); } bool is_type_valid_bit_set_elem(Type *t) { @@ -3333,8 +3336,6 @@ Selection lookup_field_with_selection(Type *type_, String field_name, bool is_ty } } } else if (type->kind == Type_DynamicArray) { - // IMPORTANT TODO(bill): Should these members be available to should I only allow them with - // `Raw_Dynamic_Array` type? GB_ASSERT(t_allocator != nullptr); String allocator_str = str_lit("allocator"); gb_local_persist Entity *entity__allocator = alloc_entity_field(nullptr, make_token_ident(allocator_str), t_allocator, false, 3); @@ -3345,15 +3346,12 @@ Selection lookup_field_with_selection(Type *type_, String field_name, bool is_ty return sel; } } else if (type->kind == Type_Map) { - // IMPORTANT TODO(bill): Should these members be available to should I only allow them with - // `Raw_Map` type? GB_ASSERT(t_allocator != nullptr); String allocator_str = str_lit("allocator"); - gb_local_persist Entity *entity__allocator = alloc_entity_field(nullptr, make_token_ident(allocator_str), t_allocator, false, 3); + gb_local_persist Entity *entity__allocator = alloc_entity_field(nullptr, make_token_ident(allocator_str), t_allocator, false, 2); if (field_name == allocator_str) { - selection_add_index(&sel, 1); - selection_add_index(&sel, 3); + selection_add_index(&sel, 2); sel.entity = entity__allocator; return sel; } @@ -3798,11 +3796,12 @@ i64 type_size_of_internal(Type *t, TypePath *path) { case Type_Map: /* struct { - hashes: []int, // 2 words - entries: [dynamic]Entry_Type, // 5 words + data: uintptr, // 1 word + size: uintptr, // 1 word + allocator: runtime.Allocator, // 2 words } */ - return (2 + (3 + 2))*build_context.word_size; + return (1 + 1 + 2)*build_context.word_size; case Type_Tuple: { i64 count, align, size; diff --git a/tests/internal/Makefile b/tests/internal/Makefile new file mode 100644 index 000000000..7328968f0 --- /dev/null +++ b/tests/internal/Makefile @@ -0,0 +1,6 @@ +ODIN=../../odin + +all: map_test + +map_test: + $(ODIN) run test_map.odin -file -vet -strict-style -o:minimal \ No newline at end of file diff --git a/tests/internal/build.bat b/tests/internal/build.bat new file mode 100644 index 000000000..313e1dbb5 --- /dev/null +++ b/tests/internal/build.bat @@ -0,0 +1,4 @@ +@echo off +set PATH_TO_ODIN==..\..\odin +%PATH_TO_ODIN% run test_map.odin -file -vet -strict-style -o:minimal +rem -define:SEED=42 \ No newline at end of file diff --git a/tests/internal/test_map.odin b/tests/internal/test_map.odin new file mode 100644 index 000000000..781fbad74 --- /dev/null +++ b/tests/internal/test_map.odin @@ -0,0 +1,382 @@ +package test_internal_map + +import "core:fmt" +import "core:intrinsics" +import "core:math/rand" +import "core:mem" +import "core:os" +import "core:testing" + +seed: u64 + +ENTRY_COUNTS := []int{11, 101, 1_001, 10_001, 100_001, 1_000_001} + +@test +map_insert_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) + for entries in ENTRY_COUNTS { + fmt.printf("[map_insert_random_key_value] Testing %v entries.\n", entries) + m: map[i64]i64 + defer delete(m) + + unique_keys := 0 + r := rand.create(seed + seed_incr) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected value. Expected m[%v] = %v, got %v", k, v, m[k])) + } + } + seed_incr += 1 + } +} + +@test +map_update_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) + for entries in ENTRY_COUNTS { + fmt.printf("[map_update_random_key_value] Testing %v entries.\n", entries) + m: map[i64]i64 + defer delete(m) + + unique_keys := 0 + r := rand.create(seed + seed_incr) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected value. Expected m[%v] = %v, got %v", k, v, m[k])) + } + } + seed_incr += 1 + } +} + +@test +map_delete_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) + for entries in ENTRY_COUNTS { + fmt.printf("[map_delete_random_key_value] Testing %v entries.\n", entries) + m: map[i64]i64 + defer delete(m) + + unique_keys := 0 + r := rand.create(seed + seed_incr) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected key present. Expected m[%v] to have been deleted, got %v", k, m[k])) + } + } else { + if k not_in m { + num_fails += 1 + if num_fails > 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Expected key not present. Expected m[%v] = %v", k, v)) + } else if m[k] != v { + num_fails += 1 + if num_fails > 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected value. Expected m[%v] = %v, got %v", k, v, m[k])) + } + } + } + seed_incr += 1 + } +} + +@test +set_insert_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) + for entries in ENTRY_COUNTS { + fmt.printf("[set_insert_random_key_value] Testing %v entries.\n", entries) + m: map[i64]struct{} + defer delete(m) + + unique_keys := 0 + r := rand.create(seed + seed_incr) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected value. Expected m[%v] to exist", k)) + } + } + seed_incr += 1 + } +} + +@test +set_delete_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) + for entries in ENTRY_COUNTS { + fmt.printf("[set_delete_random_key_value] Testing %v entries.\n", entries) + m: map[i64]struct{} + defer delete(m) + + unique_keys := 0 + r := rand.create(seed + seed_incr) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected key present. Expected m[%v] to have been deleted", k)) + } + } else { + if k not_in m { + num_fails += 1 + if num_fails > 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Expected key not present. Expected m[%v] to exist", k)) + } + } + } + seed_incr += 1 + } +} + +// -------- -------- -------- -------- -------- -------- -------- -------- -------- -------- + +main :: proc() { + t := testing.T{} + + // Allow tests to be repeatable + SEED :: #config(SEED, -1) + when SEED > 0 { + seed = u64(SEED) + } else { + seed = u64(intrinsics.read_cycle_counter()) + } + fmt.println("Initialized seed to", seed) + + mem_track_test(&t, map_insert_random_key_value) + mem_track_test(&t, map_update_random_key_value) + mem_track_test(&t, map_delete_random_key_value) + + mem_track_test(&t, set_insert_random_key_value) + mem_track_test(&t, set_delete_random_key_value) + + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + if TEST_fail > 0 { + os.exit(1) + } +} + +mem_track_test :: proc(t: ^testing.T, test: proc(t: ^testing.T)) { + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + test(t) + + expect(t, len(track.allocation_map) == 0, "Expected no leaks.") + expect(t, len(track.bad_free_array) == 0, "Expected no leaks.") + + for _, leak in track.allocation_map { + fmt.printf("%v leaked %v bytes\n", leak.location, leak.size) + } + for bad_free in track.bad_free_array { + fmt.printf("%v allocation %p was freed badly\n", bad_free.location, bad_free.memory) + } +} + +TEST_count := 0 +TEST_fail := 0 + +when ODIN_TEST { + expect :: testing.expect + log :: testing.log +} else { + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] ", loc) + fmt.printf("log: %v\n", v) + } +} \ No newline at end of file