From c96e0afbf1060dc719356b8c82601c45ad688110 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 7 Nov 2022 23:02:21 +0000 Subject: [PATCH 01/62] Begin work on implementing the new `map` internals --- core/fmt/fmt.odin | 2 + core/mem/alloc.odin | 21 +- core/reflect/reflect.odin | 4 +- core/runtime/core.odin | 27 +- core/runtime/core_builtin.odin | 37 +- core/runtime/dynamic_map_internal.odin | 1017 ++++++++++++++++-------- src/check_expr.cpp | 1 - src/check_type.cpp | 49 +- src/checker.cpp | 8 +- src/llvm_backend.cpp | 110 +-- src/llvm_backend.hpp | 2 +- src/llvm_backend_general.cpp | 24 +- src/types.cpp | 22 +- 13 files changed, 832 insertions(+), 492 deletions(-) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 2047aaf75..ce79aab9f 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -2072,6 +2072,7 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { if info.generated_struct == nil { return } + /* entries := &m.entries gs := runtime.type_info_base(info.generated_struct).variant.(runtime.Type_Info_Struct) ed := runtime.type_info_base(gs.types[1]).variant.(runtime.Type_Info_Dynamic_Array) @@ -2106,6 +2107,7 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { value := data + entry_type.offsets[3] // value: Value fmt_arg(fi, any{rawptr(value), info.value.id}, 'v') } + */ } case runtime.Type_Info_Struct: diff --git a/core/mem/alloc.odin b/core/mem/alloc.odin index 551906bed..2d11b523f 100644 --- a/core/mem/alloc.odin +++ b/core/mem/alloc.odin @@ -112,22 +112,21 @@ query_info :: proc(pointer: rawptr, allocator: Allocator, loc := #caller_locatio -delete_string :: proc(str: string, allocator := context.allocator, loc := #caller_location) { - free(raw_data(str), allocator, loc) +delete_string :: proc(str: string, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { + return free(raw_data(str), allocator, loc) } -delete_cstring :: proc(str: cstring, allocator := context.allocator, loc := #caller_location) { - free((^byte)(str), allocator, loc) +delete_cstring :: proc(str: cstring, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { + return free((^byte)(str), allocator, loc) } -delete_dynamic_array :: proc(array: $T/[dynamic]$E, loc := #caller_location) { - free(raw_data(array), array.allocator, loc) +delete_dynamic_array :: proc(array: $T/[dynamic]$E, loc := #caller_location) -> Allocator_Error { + return free(raw_data(array), array.allocator, loc) } -delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #caller_location) { - free(raw_data(array), allocator, loc) +delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { + return free(raw_data(array), allocator, loc) } -delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) { +delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) -> Allocator_Error { raw := transmute(Raw_Map)m - delete_slice(raw.hashes, raw.entries.allocator, loc) - free(raw.entries.data, raw.entries.allocator, loc) + return runtime.map_free(raw, loc) } diff --git a/core/reflect/reflect.odin b/core/reflect/reflect.odin index 896fc4473..8aab77399 100644 --- a/core/reflect/reflect.odin +++ b/core/reflect/reflect.odin @@ -273,7 +273,7 @@ length :: proc(val: any) -> int { return (^runtime.Raw_Dynamic_Array)(val.data).len case Type_Info_Map: - return (^runtime.Raw_Map)(val.data).entries.len + return runtime.map_len((^runtime.Raw_Map)(val.data)^) case Type_Info_String: if a.is_cstring { @@ -305,7 +305,7 @@ capacity :: proc(val: any) -> int { return (^runtime.Raw_Dynamic_Array)(val.data).cap case Type_Info_Map: - return (^runtime.Raw_Map)(val.data).entries.cap + return runtime.map_cap((^runtime.Raw_Map)(val.data)^) } return 0 } diff --git a/core/runtime/core.odin b/core/runtime/core.odin index 385a03b71..ce3aa239b 100644 --- a/core/runtime/core.odin +++ b/core/runtime/core.odin @@ -394,9 +394,32 @@ Raw_Dynamic_Array :: struct { allocator: Allocator, } +// The raw, type-erased representation of a map. +// +// 32-bytes on 64-bit +// 16-bytes on 32-bit Raw_Map :: struct { - hashes: []Map_Index, - entries: Raw_Dynamic_Array, + // A single allocation spanning all keys, values, and hashes. + // { + // k: Map_Cell(K) * (capacity / ks_per_cell) + // v: Map_Cell(V) * (capacity / vs_per_cell) + // h: Map_Cell(H) * (capacity / hs_per_cell) + // } + // + // The data is allocated assuming 64-byte alignment, meaning the address is + // always a multiple of 64. This means we have 6 bits of zeros in the pointer + // to store the capacity. We can store a value as large as 2^6-1 or 63 in + // there. This conveniently is the maximum log2 capacity we can have for a map + // as Odin uses signed integers to represent capacity. + // + // Since the hashes are backed by Map_Hash, which is just a 64-bit unsigned + // integer, the cell structure for hashes is unnecessary because 64/8 is 8 and + // requires no padding, meaning it can be indexed as a regular array of + // Map_Hash directly, though for consistency sake it's written as if it were + // an array of Map_Cell(Map_Hash). + data: uintptr, // 8-bytes on 64-bits, 4-bytes on 32-bits + len: uintptr, // 8-bytes on 64-bits, 4-bytes on 32-bits + allocator: Allocator, // 16-bytes on 64-bits, 8-bytes on 32-bits } Raw_Any :: struct { diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index b0f4cb25c..80a9f2944 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -159,20 +159,7 @@ delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #cal } @builtin delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) -> Allocator_Error { - Entry :: struct { - hash: uintptr, - next: int, - key: K, - value: V, - } - - raw := transmute(Raw_Map)m - err := delete_slice(raw.hashes, raw.entries.allocator, loc) - err1 := mem_free_with_size(raw.entries.data, raw.entries.cap*size_of(Entry), raw.entries.allocator, loc) - if err == nil { - err = err1 - } - return err + return map_free(transmute(Raw_Map)m, loc) } @@ -285,19 +272,13 @@ clear_map :: proc "contextless" (m: ^$T/map[$K]$V) { if m == nil { return } - raw_map := (^Raw_Map)(m) - entries := (^Raw_Dynamic_Array)(&raw_map.entries) - entries.len = 0 - for _, i in raw_map.hashes { - raw_map.hashes[i] = MAP_SENTINEL - } + map_clear_dynamic((^Raw_Map)(m), map_info(K, V)) } @builtin reserve_map :: proc(m: ^$T/map[$K]$V, capacity: int, loc := #caller_location) { if m != nil { - h := __get_map_header_table(T) - __dynamic_map_reserve(m, h, uint(capacity), loc) + __dynamic_map_reserve((^Raw_Map)(m), map_info(K, V), uint(capacity), loc) } } @@ -325,15 +306,9 @@ shrink_map :: proc(m: ^$T/map[$K]$V, new_cap := -1, loc := #caller_location) -> delete_key :: proc(m: ^$T/map[$K]$V, key: K) -> (deleted_key: K, deleted_value: V) { if m != nil { key := key - h := __get_map_header(m) - fr := __map_find(h, &key) - if fr.entry_index != MAP_SENTINEL { - entry := __dynamic_map_get_entry(h, fr.entry_index) - deleted_key = (^K)(uintptr(entry)+h.key_offset)^ - deleted_value = (^V)(uintptr(entry)+h.value_offset)^ - - __dynamic_map_erase(h, fr) - } + info := map_info(K, V) + _ = map_erase_dynamic((^Raw_Map)(m), info, uintptr(&key)) + // TODO(bill) old key and value } return } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index abe58fc5a..07c1fbe7e 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -3,150 +3,727 @@ package runtime import "core:intrinsics" _ :: intrinsics -INITIAL_MAP_CAP :: 16 +// High performance, cache-friendly, open-addressed Robin Hood hashing hash map +// data structure with various optimizations for Odin. +// +// Copyright 2022 (c) Dale Weiler +// +// The core of the hash map data structure is the Raw_Map struct which is a +// type-erased representation of the map. This type-erased representation is +// used in two ways: static and dynamic. When static type information is known, +// the procedures suffixed with _static should be used instead of _dynamic. The +// static procedures are optimized since they have type information. Hashing of +// keys, comparison of keys, and data lookup are all optimized. When type +// information is not known, the procedures suffixed with _dynamic should be +// used. The representation of the map is the same for both static and dynamic, +// and procedures of each can be mixed and matched. The purpose of the dynamic +// representation is to enable reflection and runtime manipulation of the map. +// The dynamic procedures all take an additional Map_Info structure parameter +// which carries runtime values describing the size, alignment, and offset of +// various traits of a given key and value type pair. The Map_Info value can +// be created by calling map_info(K, V) with the key and value typeids. +// +// This map implementation makes extensive use of uintptr for representing +// sizes, lengths, capacities, masks, pointers, offsets, and addresses to avoid +// expensive sign extension and masking that would be generated if types were +// casted all over. The only place regular ints show up is in the cap() and +// len() implementations. +// +// To make this map cache-friendly it uses a novel strategy to ensure keys and +// values of the map are always cache-line aligned and that no single key or +// value of any type ever straddles a cache-line. This cache efficiency makes +// for quick lookups because the linear-probe always addresses data in a cache +// friendly way. This is enabled through the use of a special meta-type called +// a Map_Cell which packs as many values of a given type into a local array adding +// internal padding to round to MAP_CACHE_LINE_SIZE. One other benefit to storing +// the internal data in this manner is false sharing no longer occurs when using +// a map, enabling efficient concurrent access of the map data structure with +// minimal locking if desired. -// Temporary data structure for comparing hashes and keys -Map_Hash :: struct { - hash: uintptr, - key_ptr: rawptr, // address of Map_Entry_Header.key +// With Robin Hood hashing a maximum load factor of 75% is ideal. +MAP_LOAD_FACTOR :: 75 + +// Minimum log2 capacity. +MAP_MIN_LOG2_CAPACITY :: 6 // 64 elements + +// Has to be less than 100% though. +#assert(MAP_LOAD_FACTOR < 100) + +// This is safe to change. The log2 size of a cache-line. At minimum it has to +// be six though. Higher cache line sizes are permitted. +MAP_CACHE_LINE_LOG2 :: 6 + +// The size of a cache-line. +MAP_CACHE_LINE_SIZE :: 1 << MAP_CACHE_LINE_LOG2 + +// The minimum cache-line size allowed by this implementation is 64 bytes since +// we need 6 bits in the base pointer to store the integer log2 capacity, which +// at maximum is 63. Odin uses signed integers to represent length and capacity, +// so only 63 bits are needed in the maximum case. +#assert(MAP_CACHE_LINE_SIZE >= 64) + +// Map_Cell type that packs multiple T in such a way to ensure that each T stays +// aligned by align_of(T) and such that align_of(Map_Cell(T)) % MAP_CACHE_LINE_SIZE == 0 +// +// This means a value of type T will never straddle a cache-line. +// +// When multiple Ts can fit in a single cache-line the data array will have more +// than one element. When it cannot, the data array will have one element and +// an array of Map_Cell(T) will be padded to stay a multiple of MAP_CACHE_LINE_SIZE. +// +// We rely on the type system to do all the arithmetic and padding for us here. +// +// The usual array[index] indexing for []T backed by a []Map_Cell(T) becomes a bit +// more involved as there now may be internal padding. The indexing now becomes +// +// N :: len(Map_Cell(T){}.data) +// i := index / N +// j := index % N +// cell[i].data[j] +// +// However, since len(Map_Cell(T){}.data) is a compile-time constant, there are some +// optimizations we can do to eliminate the need for any divisions as N will +// be bounded by [1, 64). +// +// In the optimal case, len(Map_Cell(T){}.data) = 1 so the cell array can be treated +// as a regular array of T, which is the case for hashes. +Map_Cell :: struct($T: typeid) #align MAP_CACHE_LINE_SIZE { + data: [MAP_CACHE_LINE_SIZE / size_of(T) when size_of(T) < MAP_CACHE_LINE_SIZE else 1]T, } -__get_map_key_hash :: #force_inline proc "contextless" (k: ^$K) -> uintptr { - hasher := intrinsics.type_hasher_proc(K) - return hasher(k, 0) +// So we can operate on a cell data structure at runtime without any type +// information, we have a simple table that stores some traits about the cell. +// +// 32-bytes on 64-bit +// 16-bytes on 32-bit +Map_Cell_Info :: struct { + size_of_type: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits + align_of_type: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits + size_of_cell: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits + elements_per_cell: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits } -__get_map_entry_key_ptr :: #force_inline proc "contextless" (h: Map_Header_Table, entry: ^Map_Entry_Header) -> rawptr { - return rawptr(uintptr(entry) + h.key_offset) -} - -Map_Index :: distinct uint -MAP_SENTINEL :: ~Map_Index(0) - -Map_Find_Result :: struct { - hash_index: Map_Index, - entry_prev: Map_Index, - entry_index: Map_Index, -} - -Map_Entry_Header :: struct { - hash: uintptr, - next: Map_Index, -/* - key: Key_Value, - value: Value_Type, -*/ -} - -Map_Header_Table :: struct { - equal: Equal_Proc, - - entry_size: int, - entry_align: int, - - key_offset: uintptr, - key_size: int, - - value_offset: uintptr, - value_size: int, -} - -Map_Header :: struct { - m: ^Raw_Map, - using table: Map_Header_Table, -} - -// USED INTERNALLY BY THE COMPILER -__dynamic_map_get :: proc "contextless" (m: rawptr, table: Map_Header_Table, key_hash: uintptr, key_ptr: rawptr) -> rawptr { - if m != nil { - h := Map_Header{(^Raw_Map)(m), table} - index := __dynamic_map_find(h, key_hash, key_ptr).entry_index - if index != MAP_SENTINEL { - data := uintptr(__dynamic_map_get_entry(h, index)) - return rawptr(data + h.value_offset) - } +// Same as the above procedure but at runtime with the cell Map_Cell_Info value. +map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, info: ^Map_Cell_Info, index: uintptr) -> uintptr { + // Micro-optimize the case when the number of elements per cell is one or two + // to save on expensive integer division. + switch elements_per_cell := info.elements_per_cell; elements_per_cell { + case 1: + return base + (index * info.size_of_cell) + case 2: + cell_index := index >> 1 + data_index := index & 1 + return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) + case 4: + cell_index := index >> 2 + data_index := index & 3 + return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) + case 8: + cell_index := index >> 3 + data_index := index & 7 + return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) + case 16: + cell_index := index >> 4 + data_index := index & 15 + return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) + case 32: + cell_index := index >> 5 + data_index := index & 31 + return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) + case 64: + cell_index := index >> 6 + data_index := index & 63 + return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) + case: + cell_index := index / elements_per_cell + data_index := index % elements_per_cell + return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) } +} + +// Same as above procedure but with compile-time constant index. +map_cell_index_dynamic_const :: proc "contextless" (base: uintptr, #no_alias info: ^Map_Cell_Info, $INDEX: uintptr) -> uintptr { + elements_per_cell := uintptr(info.elements_per_cell) + size_of_cell := uintptr(info.size_of_cell) + size_of_type := uintptr(info.size_of_type) + cell_index := INDEX / elements_per_cell + data_index := INDEX % elements_per_cell + return base + (cell_index * size_of_cell) + (data_index * size_of_type) +} + +// len() for map +map_len :: #force_inline proc "contextless" (m: Raw_Map) -> int { + return int(m.len) +} + +// cap() for map +map_cap :: #force_inline proc "contextless" (m: Raw_Map) -> int { + // The data uintptr stores the capacity in the lower six bits which gives the + // a maximum value of 2^6-1, or 63. We store the integer log2 of capacity + // since our capacity is always a power of two. We only need 63 bits as Odin + // represents length and capacity as a signed integer. + return 0 if m.data == 0 else 1 << map_log2_cap(m) +} + +// Query the load factor of the map. This is not actually configurable, but +// some math is needed to compute it. Compute it as a fixed point percentage to +// avoid floating point operations. This division can be optimized out by +// multiplying by the multiplicative inverse of 100. +map_load_factor :: #force_inline proc "contextless" (log2_capacity: uintptr) -> uintptr { + return ((uintptr(1) << log2_capacity) * MAP_LOAD_FACTOR) / 100 +} + +map_resize_threshold :: #force_inline proc "contextless" (m: Raw_Map) -> int { + return int(map_load_factor(map_log2_cap(m))) +} + +// The data stores the log2 capacity in the lower six bits. This is primarily +// used in the implementation rather than map_cap since the check for data = 0 +// isn't necessary in the implementation. cap() on the otherhand needs to work +// when called on an empty map. +map_log2_cap :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr { + return m.data & (64 - 1) +} + +// Canonicalize the data by removing the tagged capacity stored in the lower six +// bits of the data uintptr. +map_data :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr { + return m.data & ~uintptr(64 - 1) +} + + + +Map_Hash :: uintptr + +// __get_map_key_hash :: #force_inline proc "contextless" (k: ^$K) -> uintptr { +// hasher := intrinsics.type_hasher_proc(K) +// return hasher(k, 0) +// } + +// __get_map_entry_key_ptr :: #force_inline proc "contextless" (h: Map_Header_Table, entry: ^Map_Entry_Header) -> rawptr { +// return rawptr(uintptr(entry) + h.key_offset) +// } + + +// Procedure to check if a slot is empty for a given hash. This is represented +// by the zero value to make the zero value useful. This is a procedure just +// for prose reasons. +map_hash_is_empty :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { + return hash == 0 +} + +map_hash_is_deleted :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { + // The MSB indicates a tombstone + return (hash >> ((size_of(Map_Hash) * 8) - 1)) != 0 +} + +// Computes the desired position in the array. This is just index % capacity, +// but a procedure as there's some math involved here to recover the capacity. +map_desired_position :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Hash) -> uintptr { + // We do not use map_cap since we know the capacity will not be zero here. + capacity := uintptr(1) << map_log2_cap(m) + return uintptr(hash & Map_Hash(capacity - 1)) +} + +map_probe_distance :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Hash, slot: uintptr) -> uintptr { + // We do not use map_cap since we know the capacity will not be zero here. + capacity := uintptr(1) << map_log2_cap(m) + return (slot + capacity - map_desired_position(m, hash)) & (capacity - 1) +} + +// When working with the type-erased structure at runtime we need information +// about the map to make working with it possible. This info structure stores +// that. +// +// The Odin compiler should generate this for __get_map_header. +// +// 80-bytes on 64-bit +// 40-bytes on 32-bit +Map_Info :: struct { + ks: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit + vs: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit + hash: proc "contextless" (key: rawptr, seed: Map_Hash) -> Map_Hash, // 8-bytes on 64-bit, 4-bytes on 32-bit + cmp: proc "contextless" (lhs, rhs: rawptr) -> bool, // 8-bytes on 64-bit, 4-bytes on 32-bit +} + + +// The Map_Info structure is basically a pseudo-table of information for a given K and V pair. +map_info :: #force_inline proc "contextless" ($K: typeid, $V: typeid) -> ^Map_Info where intrinsics.type_is_comparable(K) { + @static INFO := Map_Info { + Map_Cell_Info { + size_of(K), + align_of(K), + size_of(Map_Cell(K)), + len(Map_Cell(K){}.data), + }, + Map_Cell_Info { + size_of(V), + align_of(V), + size_of(Map_Cell(V)), + len(Map_Cell(V){}.data), + }, + proc "contextless" (ptr: rawptr, seed: uintptr) -> Map_Hash { return intrinsics.type_hasher_proc(K)(ptr, seed) } , + proc "contextless" (a, b: rawptr) -> bool { return intrinsics.type_equal_proc(K)(a, b) }, + } + return &INFO +} + + +map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (ks: uintptr, vs: uintptr, hs: [^]Map_Hash, sk: uintptr, sv: uintptr) { + @static INFO_HS := Map_Cell_Info { + size_of(Map_Hash), + align_of(Map_Hash), + size_of(Map_Cell(Map_Hash)), + len(Map_Cell(Map_Hash){}.data), + } + + capacity := uintptr(1) << map_log2_cap(m) + ks = map_data(m) + vs = map_cell_index_dynamic(ks, &info.ks, capacity) // Skip past ks to get start of vs + hs_ := map_cell_index_dynamic(vs, &info.vs, capacity) // Skip past vs to get start of hs + sk = map_cell_index_dynamic(hs_, &INFO_HS, capacity) // Skip past hs to get start of sk + // Need to skip past two elements in the scratch key space to get to the start + // of the scratch value space, of which there's only two elements as well. + sv = map_cell_index_dynamic_const(sk, &info.ks, 2) + + hs = ([^]Map_Hash)(hs_) + return +} + + +// The only procedure which needs access to the context is the one which allocates the map. +map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator) -> (result: Raw_Map, err: Allocator_Error) { + if log2_capacity == 0 { + // Empty map, but set the allocator. + return { 0, 0, allocator }, nil + } + + if log2_capacity >= 64 { + // Overflowed, would be caused by log2_capacity > 64 + return {}, .Out_Of_Memory + } + + capacity := uintptr(1) << log2_capacity + + @static INFO_HS := Map_Cell_Info { + size_of(Map_Hash), + align_of(Map_Hash), + size_of(Map_Cell(Map_Hash)), + len(Map_Cell(Map_Hash){}.data), + } + + round :: #force_inline proc "contextless" (value: uintptr) -> uintptr { + return (value + MAP_CACHE_LINE_SIZE - 1) & ~uintptr(MAP_CACHE_LINE_SIZE - 1) + } + + size := uintptr(0) + size = round(map_cell_index_dynamic(size, &info.ks, capacity)) + size = round(map_cell_index_dynamic(size, &info.vs, capacity)) + size = round(map_cell_index_dynamic(size, &INFO_HS, capacity)) + + data := mem_alloc(int(size), MAP_CACHE_LINE_SIZE, allocator) or_return + data_ptr := uintptr(raw_data(data)) + + result = { + // Tagged pointer representation for capacity. + data_ptr | log2_capacity, + 0, + allocator, + } + + map_clear_dynamic(&result, info) + + return +} + +// When the type information is known we should use map_insert_hash_static for +// better performance. This procedure has to stack allocate storage to store +// local keys during the Robin Hood hashing technique where elements are swapped +// in the backing arrays to reduce variance. This swapping can only be done with +// memcpy since there is no type information. +// +// This procedure returns the address of the just inserted value. +@(optimization_mode="size") +map_insert_hash_dynamic :: proc(m: Raw_Map, info: ^Map_Info, h: Map_Hash, k, v: uintptr) -> (result: uintptr) { + info_ks := &info.ks + info_vs := &info.vs + + // Storage to exchange when reducing variance. + k_storage := intrinsics.alloca(info_ks.size_of_type, MAP_CACHE_LINE_SIZE) + v_storage := intrinsics.alloca(info_vs.size_of_type, MAP_CACHE_LINE_SIZE) + intrinsics.mem_copy_non_overlapping(rawptr(k_storage), rawptr(k), info_ks.size_of_type) + intrinsics.mem_copy_non_overlapping(rawptr(v_storage), rawptr(v), info_vs.size_of_type) + h := h + + p := map_desired_position(m, h) + d := uintptr(0) + c := (uintptr(1) << map_log2_cap(m)) - 1 // Saturating arithmetic mask + + ks, vs, hs, _, _ := map_kvh_data_dynamic(m, info) + + for { + hp := &hs[p] + element_hash := hp^ + + if map_hash_is_empty(element_hash) { + k_dst := map_cell_index_dynamic(ks, info_ks, p) + v_dst := map_cell_index_dynamic(vs, info_vs, p) + intrinsics.mem_copy_non_overlapping(rawptr(k_dst), k_storage, info_ks.size_of_type) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), v_storage, info_vs.size_of_type) + hp^ = h + return result if result != 0 else v_dst + } + + if pd := map_probe_distance(m, element_hash, p); pd < d { + if map_hash_is_deleted(element_hash) { + k_dst := map_cell_index_dynamic(ks, info_ks, p) + v_dst := map_cell_index_dynamic(vs, info_vs, p) + intrinsics.mem_copy_non_overlapping(rawptr(k_dst), k_storage, info_ks.size_of_type) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), v_storage, info_vs.size_of_type) + hp^ = h + return result if result != 0 else v_dst + } + + if result == 0 { + result = map_cell_index_dynamic(vs, info_vs, p) + } + + swap :: #force_inline proc "contextless" (lhs, rhs, size: uintptr) { + tmp := intrinsics.alloca(size, MAP_CACHE_LINE_SIZE) + intrinsics.mem_copy_non_overlapping(&tmp[0], rawptr(lhs), size) + intrinsics.mem_copy_non_overlapping(rawptr(lhs), rawptr(rhs), size) + intrinsics.mem_copy_non_overlapping(rawptr(rhs), &tmp[0], size) + } + + // Exchange to reduce variance. + swap(uintptr(k_storage), map_cell_index_dynamic(ks, info_ks, p), info_ks.size_of_type) + swap(uintptr(v_storage), map_cell_index_dynamic(vs, info_vs, p), info_vs.size_of_type) + hp^, h = h, hp^ + + d = pd + } + + p = (p + 1) & c + d += 1 + } +} + +@(optimization_mode="speed") +map_add_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) { + info_ks := &info.ks + info_vs := &info.vs + + capacity := uintptr(1) << map_log2_cap(m) + p := map_desired_position(m, h) + d := uintptr(0) + c := capacity - 1 // Saturating arithmetic mask + + ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) + + // Avoid redundant loads of these values + size_of_k := info_ks.size_of_type + size_of_v := info_vs.size_of_type + + // Use sk and sv scratch storage space for dynamic k and v storage here. + // + // Simulate the following at runtime + // k = ik + // v = iv + // h = h + k := map_cell_index_dynamic_const(sk, info_ks, 0) + v := map_cell_index_dynamic_const(sv, info_vs, 0) + intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(ik), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(iv), size_of_v) + h := h + + // Temporary k and v dynamic storage for swap below + tk := map_cell_index_dynamic_const(sk, info_ks, 1) + tv := map_cell_index_dynamic_const(sv, info_vs, 1) + + for { + hp := &hs[p] + element_hash := hp^ + + if map_hash_is_empty(element_hash) { + k_dst := map_cell_index_dynamic(ks, info_ks, p) + v_dst := map_cell_index_dynamic(vs, info_vs, p) + intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) + hp^ = h + return + } + + if pd := map_probe_distance(m, element_hash, p); pd < d { + if map_hash_is_deleted(element_hash) { + k_dst := map_cell_index_dynamic(ks, info_ks, p) + v_dst := map_cell_index_dynamic(vs, info_vs, p) + intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) + hp^ = h + return + } + + kp := map_cell_index_dynamic(ks, info_vs, p) + vp := map_cell_index_dynamic(vs, info_ks, p) + + // Simulate the following at runtime with dynamic storage + // + // kp^, k = k, kp^ + // vp^, v = v, vp^ + // hp^, h = h, hp^ + intrinsics.mem_copy_non_overlapping(rawptr(tk), rawptr(kp), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(tv), rawptr(vp), size_of_v) + intrinsics.mem_copy_non_overlapping(rawptr(kp), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(vp), rawptr(v), size_of_v) + intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(tk), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(tv), size_of_v) + hp^, h = h, hp^ + + d = pd + } + + p = (p + 1) & c + d += 1 + } +} + +@(optimization_mode="size") +map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Allocator_Error { + allocator := m.allocator + + log2_capacity := map_log2_cap(m^) + + if m.data == 0 { + n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return + m.data = n.data + return nil + } + + resized := map_alloc_dynamic(info, log2_capacity + 1, allocator) or_return + + capacity := uintptr(1) << log2_capacity + + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) + + // Cache these loads to avoid hitting them in the for loop. + info_ks := &info.ks + info_vs := &info.vs + + n := map_len(m^) + for i := uintptr(0); i < capacity; i += 1 { + hash := hs[i] + if map_hash_is_empty(hash) do continue + if map_hash_is_deleted(hash) do continue + k := map_cell_index_dynamic(ks, info_ks, i) + v := map_cell_index_dynamic(vs, info_vs, i) + map_insert_hash_dynamic(resized, info, hash, k, v) + // Only need to do this comparison on each actually added pair, so do not + // fold it into the for loop comparator as a micro-optimization. + n -= 1 + if n == 0 do break + } + + mem_free(rawptr(ks), allocator) + + m.data = resized.data // Should copy the capacity too + return nil } -// USED INTERNALLY BY THE COMPILER -__dynamic_map_set :: proc "odin" (m: rawptr, table: Map_Header_Table, key_hash: uintptr, key_ptr: rawptr, value: rawptr, loc := #caller_location) -> ^Map_Entry_Header #no_bounds_check { - add_entry :: proc "odin" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr, loc := #caller_location) -> Map_Index { - prev := Map_Index(h.m.entries.len) - c := Map_Index(__dynamic_array_append_nothing(&h.m.entries, h.entry_size, h.entry_align, loc)) - if c != prev { - end := __dynamic_map_get_entry(h, c-1) - end.hash = key_hash - mem_copy(rawptr(uintptr(end) + h.key_offset), key_ptr, h.key_size) - end.next = MAP_SENTINEL - } - return prev + +@(optimization_mode="size") +map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr) -> Allocator_Error { + allocator := m.allocator + + log2_capacity := map_log2_cap(m^) + capacity := uintptr(1) << log2_capacity + + if capacity >= new_capacity { + return nil + } + // ceiling nearest power of two + log2_new_capacity := size_of(uintptr) - intrinsics.count_leading_zeros(new_capacity-1) + + if m.data == 0 { + n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return + m.data = n.data + return nil } - h := Map_Header{(^Raw_Map)(m), table} + resized := map_alloc_dynamic(info, log2_new_capacity, allocator) or_return - index := MAP_SENTINEL - if len(h.m.hashes) == 0 { - __dynamic_map_reserve(m, table, INITIAL_MAP_CAP, loc) - __dynamic_map_grow(h, loc) + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) + + // Cache these loads to avoid hitting them in the for loop. + info_ks := &info.ks + info_vs := &info.vs + + n := map_len(m^) + for i := uintptr(0); i < capacity; i += 1 { + hash := hs[i] + if map_hash_is_empty(hash) do continue + if map_hash_is_deleted(hash) do continue + k := map_cell_index_dynamic(ks, info_ks, i) + v := map_cell_index_dynamic(vs, info_vs, i) + map_insert_hash_dynamic(resized, info, hash, k, v) + // Only need to do this comparison on each actually added pair, so do not + // fold it into the for loop comparator as a micro-optimization. + n -= 1 + if n == 0 do break } - fr := __dynamic_map_find(h, key_hash, key_ptr) - if fr.entry_index != MAP_SENTINEL { - index = fr.entry_index - } else { - index = add_entry(h, key_hash, key_ptr, loc) - if fr.entry_prev != MAP_SENTINEL { - entry := __dynamic_map_get_entry(h, fr.entry_prev) - entry.next = index - } else if fr.hash_index != MAP_SENTINEL { - h.m.hashes[fr.hash_index] = index - } else { - return nil - } - } + mem_free(rawptr(ks), allocator) - e := __dynamic_map_get_entry(h, index) - e.hash = key_hash + m.data = resized.data // Should copy the capacity too - key := rawptr(uintptr(e) + h.key_offset) - val := rawptr(uintptr(e) + h.value_offset) - - mem_copy(key, key_ptr, h.key_size) - mem_copy(val, value, h.value_size) - - if __dynamic_map_full(h) { - __dynamic_map_grow(h, loc) - } - - return __dynamic_map_get_entry(h, index) + return nil } -// USED INTERNALLY BY THE COMPILER -__dynamic_map_reserve :: proc "odin" (m: rawptr, table: Map_Header_Table, cap: uint, loc := #caller_location) { - h := Map_Header{(^Raw_Map)(m), table} - c := context - if h.m.entries.allocator.procedure != nil { - c.allocator = h.m.entries.allocator +@(optimization_mode="size") +map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Allocator_Error { + allocator := m.allocator + + // Cannot shrink the capacity if the number of items in the map would exceed + // one minus the current log2 capacity's resize threshold. That is the shrunk + // map needs to be within the max load factor. + log2_capacity := map_log2_cap(m^) + if m.len >= map_load_factor(log2_capacity - 1) do return nil + + shrinked := map_alloc_dynamic(info, log2_capacity - 1, allocator) or_return + + capacity := uintptr(1) << log2_capacity + + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) + + info_ks := &info.ks + info_vs := &info.vs + + n := map_len(m^) + for i := uintptr(0); i < capacity; i += 1 { + hash := hs[i] + if map_hash_is_empty(hash) do continue + if map_hash_is_deleted(hash) do continue + + k := map_cell_index_dynamic(ks, info_ks, i) + v := map_cell_index_dynamic(vs, info_vs, i) + + map_insert_hash_dynamic(shrinked, info, hash, k, v) + + // Only need to do this comparison on each actually added pair, so do not + // fold it into the for loop comparator as a micro-optimization. + n -= 1 + if n == 0 do break } - context = c - cap := cap - cap = ceil_to_pow2(cap) + free(rawptr(ks), allocator) - __dynamic_array_reserve(&h.m.entries, h.entry_size, h.entry_align, int(cap), loc) + m.data = shrinked.data // Should copy the capacity too - if h.m.entries.len*2 < len(h.m.hashes) { - return - } - if __slice_resize(&h.m.hashes, int(cap*2), h.m.entries.allocator, loc) { - __dynamic_map_reset_entries(h, loc) + return nil +} + +// Single procedure for static and dynamic paths. +@(require_results) +map_free :: proc(m: Raw_Map, loc := #caller_location) -> Allocator_Error { + return mem_free(rawptr(map_data(m)), m.allocator, loc) +} + +@(optimization_mode="speed") +map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (index: uintptr, ok: bool) { + if map_len(m) == 0 do return 0, false + h := info.hash(rawptr(k), 0) + p := map_desired_position(m, h) + d := uintptr(0) + c := (uintptr(1) << map_log2_cap(m)) - 1 + ks, _, hs, _, _ := map_kvh_data_dynamic(m, info) + info_ks := &info.ks + for { + element_hash := hs[p] + if map_hash_is_empty(element_hash) { + return 0, false + } else if d > map_probe_distance(m, element_hash, p) { + return 0, false + } else if element_hash == h && info.cmp(rawptr(k), rawptr(map_cell_index_dynamic(ks, info_ks, p))) { + return p, true + } + p = (p + 1) & c + d += 1 } } + + + +@(optimization_mode="speed") +map_insert_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr) -> (value: uintptr, err: Allocator_Error) { + if map_len(m^) + 1 >= map_resize_threshold(m^) { + map_grow_dynamic(m, info) or_return + } + hashed := info.hash(rawptr(k), 0) + result := map_insert_hash_dynamic(m^, info, hashed, k, v) + m.len += 1 + return result, nil +} + +// Same as map_insert_dynamic but does not return address to the inserted element. +@(optimization_mode="speed") +map_add_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr) -> Allocator_Error { + if map_len(m^) + 1 >= map_resize_threshold(m^) { + map_grow_dynamic(m, info) or_return + } + map_add_hash_dynamic(m^, info, info.hash(rawptr(k), 0), k, v) + m.len += 1 + return nil +} + +map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> bool { + when size_of(Map_Hash) == 4 do MASK :: Map_Hash(0x8000_0000) + when size_of(Map_Hash) == 8 do MASK :: Map_Hash(0x8000_0000_0000_0000) + when size_of(Map_Hash) == 16 do MASK :: Map_Hash(0x8000_0000_0000_0000_0000_0000_0000_0000) + index := map_lookup_dynamic(m^, info, k) or_return + _, _, hs, _, _ := map_kvh_data_dynamic(m^, info) + hs[index] |= MASK + m.len -= 1 + return true +} + +map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) { + if m.data == 0 do return + _, _, hs, _, _ := map_kvh_data_dynamic(m^, info) + intrinsics.mem_zero(rawptr(hs), map_cap(m^) * size_of(Map_Hash)) + m.len = 0 +} + + +// TODO(bill): Change signature to not be a `rawptr` +__dynamic_map_get :: proc "contextless" (m: rawptr, #no_alias info: ^Map_Info, key: rawptr) -> rawptr { + rm := (^Raw_Map)(m)^ + index, ok := map_lookup_dynamic(rm, info, uintptr(key)) + if !ok { + return nil + } + _ = index + // TODO(bill) + return nil +} + +__dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { + // value, _ := map_insert_dynamic(m, info, uintptr(key), uintptr(value)) + // return rawptr(value) + // TODO(bill) + return nil +} + +__dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) { + map_reserve_dynamic(m, info, uintptr(new_capacity)) +} + + + + INITIAL_HASH_SEED :: 0xcbf29ce484222325 _fnv64a :: proc "contextless" (data: []byte, seed: u64 = INITIAL_HASH_SEED) -> u64 { @@ -154,7 +731,7 @@ _fnv64a :: proc "contextless" (data: []byte, seed: u64 = INITIAL_HASH_SEED) -> u for b in data { h = (h ~ u64(b)) * 0x100000001b3 } - return h + return h | u64(h == 0) } default_hash :: #force_inline proc "contextless" (data: []byte) -> uintptr { @@ -177,7 +754,7 @@ _default_hasher_const :: #force_inline proc "contextless" (data: rawptr, seed: u h = (h ~ b) * 0x100000001b3 p += 1 } - return uintptr(h) + return uintptr(h) | uintptr(h == 0) } default_hasher_n :: #force_inline proc "contextless" (data: rawptr, seed: uintptr, N: int) -> uintptr { @@ -188,7 +765,7 @@ default_hasher_n :: #force_inline proc "contextless" (data: rawptr, seed: uintpt h = (h ~ b) * 0x100000001b3 p += 1 } - return uintptr(h) + return uintptr(h) | uintptr(h == 0) } // NOTE(bill): There are loads of predefined ones to improve optimizations for small types @@ -216,7 +793,7 @@ default_hasher_string :: proc "contextless" (data: rawptr, seed: uintptr) -> uin for b in str { h = (h ~ u64(b)) * 0x100000001b3 } - return uintptr(h) + return uintptr(h) | uintptr(h == 0) } default_hasher_cstring :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { h := u64(seed) + 0xcbf29ce484222325 @@ -226,203 +803,5 @@ default_hasher_cstring :: proc "contextless" (data: rawptr, seed: uintptr) -> ui h = (h ~ u64(b)) * 0x100000001b3 ptr += 1 } - return uintptr(h) -} - - -__get_map_header :: proc "contextless" (m: ^$T/map[$K]$V) -> (header: Map_Header) { - header.m = (^Raw_Map)(m) - header.table = #force_inline __get_map_header_table(T) - return -} - -__get_map_header_runtime :: proc "contextless" (m: ^Raw_Map, ti: Type_Info_Map) -> (header: Map_Header) { - header.m = m - header.table = #force_inline __get_map_header_table_runtime(ti) - return -} - -__get_map_header_table :: proc "contextless" ($T: typeid/map[$K]$V) -> (header: Map_Header_Table) { - Entry :: struct { - hash: uintptr, - next: Map_Index, - key: K, - value: V, - } - - header.equal = intrinsics.type_equal_proc(K) - - header.entry_size = size_of(Entry) - header.entry_align = align_of(Entry) - - header.key_offset = offset_of(Entry, key) - header.key_size = size_of(K) - - header.value_offset = offset_of(Entry, value) - header.value_size = size_of(V) - - return -} - -__get_map_header_table_runtime :: proc "contextless" (ti: Type_Info_Map) -> (header: Map_Header) { - header.equal = ti.key_equal - - entries := ti.generated_struct.variant.(Type_Info_Struct).types[1] - entry := entries.variant.(Type_Info_Dynamic_Array).elem - e := entry.variant.(Type_Info_Struct) - - header.entry_size = entry.size - header.entry_align = entry.align - - header.key_offset = e.offsets[2] - header.key_size = e.types[2].size - - header.value_offset = e.offsets[3] - header.value_size = e.types[3].size - - return -} - - - -__slice_resize :: proc "odin" (array_: ^$T/[]$E, new_count: int, allocator: Allocator, loc := #caller_location) -> bool { - array := (^Raw_Slice)(array_) - - if new_count < array.len { - return true - } - - old_size := array.len*size_of(T) - new_size := new_count*size_of(T) - - new_data, err := mem_resize(array.data, old_size, new_size, align_of(T), allocator, loc) - if err != nil { - return false - } - if new_data != nil || size_of(E) == 0 { - array.data = raw_data(new_data) - array.len = new_count - return true - } - return false -} - -__dynamic_map_reset_entries :: proc "contextless" (h: Map_Header, loc := #caller_location) { - for i in 0.. (did_shrink: bool) { - c := context - if h.m.entries.allocator.procedure != nil { - c.allocator = h.m.entries.allocator - } - context = c - - return __dynamic_array_shrink(&h.m.entries, h.entry_size, h.entry_align, cap, loc) -} - - -@(private="file") -ceil_to_pow2 :: proc "contextless" (n: uint) -> uint { - if n <= 2 { - return n - } - n := n - n -= 1 - n |= n >> 1 - n |= n >> 2 - n |= n >> 4 - n |= n >> 8 - n |= n >> 16 - when size_of(int) == 8 { - n |= n >> 32 - } - n += 1 - return n -} - -__dynamic_map_grow :: proc "odin" (h: Map_Header, loc := #caller_location) { - new_count := max(uint(h.m.entries.cap) * 2, INITIAL_MAP_CAP) - // Rehash through Reserve - __dynamic_map_reserve(h.m, h.table, new_count, loc) -} - -__dynamic_map_full :: #force_inline proc "contextless" (h: Map_Header) -> bool { - return int(0.75 * f64(len(h.m.hashes))) <= h.m.entries.len -} - -__dynamic_map_find_from_entry :: proc "contextless" (h: Map_Header, e: ^Map_Entry_Header) -> Map_Find_Result #no_bounds_check { - key_ptr := __get_map_entry_key_ptr(h, e) - return __dynamic_map_find(h, e.hash, key_ptr) - -} - -__dynamic_map_find :: proc "contextless" (h: Map_Header, key_hash: uintptr, key_ptr: rawptr) -> Map_Find_Result #no_bounds_check { - fr := Map_Find_Result{MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL} - if n := uintptr(len(h.m.hashes)); n != 0 { - fr.hash_index = Map_Index(key_hash & (n-1)) - fr.entry_index = h.m.hashes[fr.hash_index] - for fr.entry_index != MAP_SENTINEL { - entry := __dynamic_map_get_entry(h, fr.entry_index) - entry_key_ptr := __get_map_entry_key_ptr(h, entry) - if entry.hash == key_hash && h.equal(entry_key_ptr, key_ptr) { - return fr - } - - fr.entry_prev = fr.entry_index - fr.entry_index = entry.next - } - } - return fr -} - -// Utility procedure used by other runtime procedures -__map_find :: proc "contextless" (h: Map_Header, key_ptr: ^$K) -> Map_Find_Result #no_bounds_check { - hash := __get_map_key_hash(key_ptr) - return #force_inline __dynamic_map_find(h, hash, key_ptr) -} - -__dynamic_map_get_entry :: #force_inline proc "contextless" (h: Map_Header, index: Map_Index) -> ^Map_Entry_Header { - return (^Map_Entry_Header)(uintptr(h.m.entries.data) + uintptr(index*Map_Index(h.entry_size))) -} - -__dynamic_map_erase :: proc "contextless" (h: Map_Header, fr: Map_Find_Result) #no_bounds_check { - if fr.entry_prev != MAP_SENTINEL { - prev := __dynamic_map_get_entry(h, fr.entry_prev) - curr := __dynamic_map_get_entry(h, fr.entry_index) - prev.next = curr.next - } else { - h.m.hashes[fr.hash_index] = __dynamic_map_get_entry(h, fr.entry_index).next - } - last_index := Map_Index(h.m.entries.len-1) - if fr.entry_index != last_index { - old := __dynamic_map_get_entry(h, fr.entry_index) - end := __dynamic_map_get_entry(h, last_index) - mem_copy(old, end, h.entry_size) - - last := __dynamic_map_find_from_entry(h, old) - if last.entry_prev != MAP_SENTINEL { - e := __dynamic_map_get_entry(h, last.entry_prev) - e.next = fr.entry_index - } else { - h.m.hashes[last.hash_index] = fr.entry_index - } - } - - h.m.entries.len -= 1 + return uintptr(h) | uintptr(h == 0) } diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 9e48fd8ad..043b98173 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -1364,7 +1364,6 @@ bool is_polymorphic_type_assignable(CheckerContext *c, Type *poly, Type *source, bool key = is_polymorphic_type_assignable(c, poly->Map.key, source->Map.key, true, modify_type); bool value = is_polymorphic_type_assignable(c, poly->Map.value, source->Map.value, true, modify_type); if (key || value) { - poly->Map.entry_type = nullptr; poly->Map.internal_type = nullptr; poly->Map.lookup_result_type = nullptr; init_map_internal_types(poly); diff --git a/src/check_type.cpp b/src/check_type.cpp index 2ffe04342..ea1c90a66 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2176,40 +2176,9 @@ Type *make_optional_ok_type(Type *value, bool typed) { return t; } -void init_map_entry_type(Type *type) { - GB_ASSERT(type->kind == Type_Map); - if (type->Map.entry_type != nullptr) return; - - // NOTE(bill): The preload types may have not been set yet - GB_ASSERT(t_map_hash != nullptr); - - /* - struct { - hash: uintptr, - next: int, - key: Key, - value: Value, - } - */ - Scope *s = create_scope(nullptr, builtin_pkg->scope); - - auto fields = slice_make(permanent_allocator(), 4); - fields[0] = alloc_entity_field(s, make_token_ident(str_lit("hash")), t_uintptr, false, 0, EntityState_Resolved); - fields[1] = alloc_entity_field(s, make_token_ident(str_lit("next")), t_int, false, 1, EntityState_Resolved); - fields[2] = alloc_entity_field(s, make_token_ident(str_lit("key")), type->Map.key, false, 2, EntityState_Resolved); - fields[3] = alloc_entity_field(s, make_token_ident(str_lit("value")), type->Map.value, false, 3, EntityState_Resolved); - - Type *entry_type = alloc_type_struct(); - entry_type->Struct.fields = fields; - entry_type->Struct.tags = gb_alloc_array(permanent_allocator(), String, fields.count); - - type_set_offsets(entry_type); - type->Map.entry_type = entry_type; -} - void init_map_internal_types(Type *type) { GB_ASSERT(type->kind == Type_Map); - init_map_entry_type(type); + GB_ASSERT(t_allocator != nullptr); if (type->Map.internal_type != nullptr) return; Type *key = type->Map.key; @@ -2221,19 +2190,17 @@ void init_map_internal_types(Type *type) { /* struct { - hashes: []int; - entries: [dynamic]EntryType; + data: uintptr, + size: uintptr, + allocator: runtime.Allocator, } */ Scope *s = create_scope(nullptr, builtin_pkg->scope); - Type *hashes_type = alloc_type_slice(t_int); - Type *entries_type = alloc_type_dynamic_array(type->Map.entry_type); - - - auto fields = slice_make(permanent_allocator(), 2); - fields[0] = alloc_entity_field(s, make_token_ident(str_lit("hashes")), hashes_type, false, 0, EntityState_Resolved); - fields[1] = alloc_entity_field(s, make_token_ident(str_lit("entries")), entries_type, false, 1, EntityState_Resolved); + auto fields = slice_make(permanent_allocator(), 3); + fields[0] = alloc_entity_field(s, make_token_ident(str_lit("data")), t_uintptr, false, 0, EntityState_Resolved); + fields[1] = alloc_entity_field(s, make_token_ident(str_lit("size")), t_uintptr, false, 1, EntityState_Resolved); + fields[2] = alloc_entity_field(s, make_token_ident(str_lit("allocator")), t_allocator, false, 2, EntityState_Resolved); generated_struct_type->Struct.fields = fields; type_set_offsets(generated_struct_type); diff --git a/src/checker.cpp b/src/checker.cpp index dd81e2a48..d5d2c6026 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -2842,12 +2842,12 @@ void init_core_source_code_location(Checker *c) { } void init_core_map_type(Checker *c) { - if (t_map_hash != nullptr) { + if (t_map_info != nullptr) { return; } - t_map_hash = find_core_type(c, str_lit("Map_Hash")); - t_map_header = find_core_type(c, str_lit("Map_Header")); - t_map_header_table = find_core_type(c, str_lit("Map_Header_Table")); + t_map_info = find_core_type(c, str_lit("Map_Info")); + t_map_cell_info = find_core_type(c, str_lit("Map_Cell_Info")); + init_mem_allocator(c); } void init_preload(Checker *c) { diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 1d2c00700..40b861d40 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -298,7 +298,7 @@ lbValue lb_simple_compare_hash(lbProcedure *p, Type *type, lbValue data, lbValue lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { type = core_type(type); - GB_ASSERT(is_type_valid_for_keys(type)); + GB_ASSERT_MSG(is_type_valid_for_keys(type), "%s", type_to_string(type)); Type *pt = alloc_type_pointer(type); @@ -500,51 +500,67 @@ lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, A return value; } -lbValue lb_gen_map_header_table_internal(lbProcedure *p, Type *map_type) { +// IMPORTANT NOTE(bill): This must match the definition in dynamic_map_internal.odin +enum : i64 { + MAP_CACHE_LINE_LOG2 = 6, + MAP_CACHE_LINE_SIZE = 1 << MAP_CACHE_LINE_LOG2 +}; +GB_STATIC_ASSERT(MAP_CACHE_LINE_SIZE >= 64); +void lb_map_cell_size_and_len(Type *type, i64 *size_, i64 *len_) { + i64 elem_sz = type_size_of(type); + + i64 len = 1; + if (0 < elem_sz && elem_sz < MAP_CACHE_LINE_SIZE) { + len = MAP_CACHE_LINE_SIZE / elem_sz; + } + i64 size = align_formula(elem_sz * len, MAP_CACHE_LINE_SIZE); + if (size_) *size_ = size; + if (len_) *len_ = len; +} + +LLVMValueRef lb_gen_map_cell_info(lbModule *m, Type *type) { + i64 size = 0, len = 0; + lb_map_cell_size_and_len(type, &size, &len); + + LLVMValueRef const_values[4] = {}; + const_values[0] = lb_const_int(m, t_uintptr, type_size_of(type)).value; + const_values[1] = lb_const_int(m, t_uintptr, type_align_of(type)).value; + const_values[2] = lb_const_int(m, t_uintptr, size).value; + const_values[3] = lb_const_int(m, t_uintptr, len).value; + return llvm_const_named_struct(m, t_map_cell_info, const_values, gb_count_of(const_values)); + +} +lbValue lb_gen_map_info_ptr(lbProcedure *p, Type *map_type) { lbModule *m = p->module; map_type = base_type(map_type); GB_ASSERT(map_type->kind == Type_Map); - lbAddr *found = map_get(&m->map_header_table_map, map_type); + lbAddr *found = map_get(&m->map_info_map, map_type); if (found) { - return lb_addr_load(p, *found); + return lb_addr_get_ptr(p, *found); } - GB_ASSERT(map_type->Map.entry_type->kind == Type_Struct); - i64 entry_size = type_size_of (map_type->Map.entry_type); - i64 entry_align = type_align_of (map_type->Map.entry_type); + GB_ASSERT(t_map_info != nullptr); + GB_ASSERT(t_map_cell_info != nullptr); - i64 key_offset = type_offset_of(map_type->Map.entry_type, 2); - i64 key_size = type_size_of (map_type->Map.key); + LLVMValueRef key_cell_info = lb_gen_map_cell_info(m, map_type->Map.key); + LLVMValueRef value_cell_info = lb_gen_map_cell_info(m, map_type->Map.value); - i64 value_offset = type_offset_of(map_type->Map.entry_type, 3); - i64 value_size = type_size_of (map_type->Map.value); + LLVMValueRef const_values[4] = {}; + const_values[0] = key_cell_info; + const_values[1] = value_cell_info; + const_values[2] = lb_get_hasher_proc_for_type(m, map_type->Map.key).value; + const_values[3] = lb_get_equal_proc_for_type(m, map_type->Map.key).value; - Type *key_type = map_type->Map.key; - Type *val_type = map_type->Map.value; - gb_unused(val_type); + LLVMValueRef llvm_res = llvm_const_named_struct(m, t_map_info, const_values, gb_count_of(const_values)); + lbValue res = {llvm_res, t_map_info}; - Type *st = base_type(t_map_header_table); - GB_ASSERT(st->Struct.fields.count == 7); - - LLVMValueRef const_values[7] = {}; - const_values[0] = lb_get_equal_proc_for_type(m, key_type) .value; - const_values[1] = lb_const_int(m, t_int, entry_size) .value; - const_values[2] = lb_const_int(m, t_int, entry_align) .value; - const_values[3] = lb_const_int(m, t_uintptr, key_offset) .value; - const_values[4] = lb_const_int(m, t_int, key_size) .value; - const_values[5] = lb_const_int(m, t_uintptr, value_offset).value; - const_values[6] = lb_const_int(m, t_int, value_size) .value; - - LLVMValueRef llvm_res = llvm_const_named_struct(m, t_map_header_table, const_values, gb_count_of(const_values)); - lbValue res = {llvm_res, t_map_header_table}; - - lbAddr addr = lb_add_global_generated(m, t_map_header_table, res, nullptr); + lbAddr addr = lb_add_global_generated(m, t_map_info, res, nullptr); lb_make_global_private_const(addr); - map_set(&m->map_header_table_map, map_type, addr); - return lb_addr_load(p, addr); + map_set(&m->map_info_map, map_type, addr); + return lb_addr_get_ptr(p, addr); } lbValue lb_const_hash(lbModule *m, lbValue key, Type *key_type) { @@ -616,12 +632,13 @@ lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key) { Type *map_type = base_type(type_deref(map_ptr.type)); - lbValue key_ptr = {}; - auto args = array_make(permanent_allocator(), 4); + lbValue key_ptr = lb_address_from_load_or_generate_local(p, key); + key_ptr = lb_emit_conv(p, key_ptr, t_rawptr); + + auto args = array_make(permanent_allocator(), 3); args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_header_table_internal(p, map_type); - args[2] = lb_gen_map_key_hash(p, key, map_type->Map.key, &key_ptr); - args[3] = key_ptr; + args[1] = lb_gen_map_info_ptr(p, map_type); + args[2] = key_ptr; lbValue ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); @@ -633,20 +650,19 @@ void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, map_type = base_type(map_type); GB_ASSERT(map_type->kind == Type_Map); - lbValue key_ptr = {}; - lbValue key_hash = lb_gen_map_key_hash(p, map_key, map_type->Map.key, &key_ptr); + lbValue key_ptr = lb_address_from_load_or_generate_local(p, map_key); + key_ptr = lb_emit_conv(p, key_ptr, t_rawptr); lbValue v = lb_emit_conv(p, map_value, map_type->Map.value); lbAddr value_addr = lb_add_local_generated(p, v.type, false); lb_addr_store(p, value_addr, v); - auto args = array_make(permanent_allocator(), 6); + auto args = array_make(permanent_allocator(), 5); args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_header_table_internal(p, map_type); - args[2] = key_hash; - args[3] = key_ptr; - args[4] = lb_emit_conv(p, value_addr.addr, t_rawptr); - args[5] = lb_emit_source_code_location_as_global(p, node); + args[1] = lb_gen_map_info_ptr(p, map_type); + args[2] = key_ptr; + args[3] = lb_emit_conv(p, value_addr.addr, t_rawptr); + args[4] = lb_emit_source_code_location_as_global(p, node); lb_emit_runtime_call(p, "__dynamic_map_set", args); } @@ -660,8 +676,8 @@ void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const auto args = array_make(permanent_allocator(), 4); args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_header_table_internal(p, type_deref(map_ptr.type)); - args[2] = lb_const_int(p->module, t_int, capacity); + args[1] = lb_gen_map_info_ptr(p, type_deref(map_ptr.type)); + args[2] = lb_const_int(p->module, t_uint, capacity); args[3] = lb_emit_source_code_location_as_global(p, proc_name, pos); lb_emit_runtime_call(p, "__dynamic_map_reserve", args); } diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index a8ff1571c..b797f28f9 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -160,7 +160,7 @@ struct lbModule { StringMap objc_classes; StringMap objc_selectors; - PtrMap map_header_table_map; + PtrMap map_info_map; }; struct lbGenerator { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 87f8afa05..69b1fce20 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -75,7 +75,7 @@ void lb_init_module(lbModule *m, Checker *c) { string_map_init(&m->objc_classes, a); string_map_init(&m->objc_selectors, a); - map_init(&m->map_header_table_map, a, 0); + map_init(&m->map_info_map, a, 0); } @@ -1939,27 +1939,13 @@ LLVMTypeRef lb_type_internal(lbModule *m, Type *type) { defer (m->internal_type_level += 1); unsigned field_count = cast(unsigned)(internal_type->Struct.fields.count); - GB_ASSERT(field_count == 2); - LLVMTypeRef *fields = gb_alloc_array(temporary_allocator(), LLVMTypeRef, field_count); + GB_ASSERT(field_count == 3); - LLVMTypeRef entries_fields[] = { - lb_type(m, t_rawptr), // data - lb_type(m, t_int), // len - lb_type(m, t_int), // cap + LLVMTypeRef fields[3] = { + lb_type(m, t_uintptr), // data + lb_type(m, t_uintptr), // len lb_type(m, t_allocator), // allocator }; - - fields[0] = lb_type(m, internal_type->Struct.fields[0]->type); - fields[1] = LLVMStructTypeInContext(ctx, entries_fields, gb_count_of(entries_fields), false); - - { // Add this to simplify things - lbStructFieldRemapping entries_field_remapping = {}; - slice_init(&entries_field_remapping, permanent_allocator(), gb_count_of(entries_fields)); - for_array(i, entries_field_remapping) { - entries_field_remapping[i] = cast(i32)i; - } - map_set(&m->struct_field_remapping, cast(void *)fields[1], entries_field_remapping); - } return LLVMStructTypeInContext(ctx, fields, field_count, false); } diff --git a/src/types.cpp b/src/types.cpp index b9f2b375f..220d1a6ab 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -226,7 +226,6 @@ struct TypeProc { TYPE_KIND(Map, struct { \ Type *key; \ Type *value; \ - Type *entry_type; \ Type *internal_type; \ Type *lookup_result_type; \ }) \ @@ -685,9 +684,8 @@ gb_global Type *t_allocator_error = nullptr; gb_global Type *t_source_code_location = nullptr; gb_global Type *t_source_code_location_ptr = nullptr; -gb_global Type *t_map_hash = nullptr; -gb_global Type *t_map_header = nullptr; -gb_global Type *t_map_header_table = nullptr; +gb_global Type *t_map_info = nullptr; +gb_global Type *t_map_cell_info = nullptr; gb_global Type *t_equal_proc = nullptr; @@ -3330,8 +3328,6 @@ Selection lookup_field_with_selection(Type *type_, String field_name, bool is_ty } } } else if (type->kind == Type_DynamicArray) { - // IMPORTANT TODO(bill): Should these members be available to should I only allow them with - // `Raw_Dynamic_Array` type? GB_ASSERT(t_allocator != nullptr); String allocator_str = str_lit("allocator"); gb_local_persist Entity *entity__allocator = alloc_entity_field(nullptr, make_token_ident(allocator_str), t_allocator, false, 3); @@ -3342,15 +3338,12 @@ Selection lookup_field_with_selection(Type *type_, String field_name, bool is_ty return sel; } } else if (type->kind == Type_Map) { - // IMPORTANT TODO(bill): Should these members be available to should I only allow them with - // `Raw_Map` type? GB_ASSERT(t_allocator != nullptr); String allocator_str = str_lit("allocator"); - gb_local_persist Entity *entity__allocator = alloc_entity_field(nullptr, make_token_ident(allocator_str), t_allocator, false, 3); + gb_local_persist Entity *entity__allocator = alloc_entity_field(nullptr, make_token_ident(allocator_str), t_allocator, false, 2); if (field_name == allocator_str) { - selection_add_index(&sel, 1); - selection_add_index(&sel, 3); + selection_add_index(&sel, 2); sel.entity = entity__allocator; return sel; } @@ -3795,11 +3788,12 @@ i64 type_size_of_internal(Type *t, TypePath *path) { case Type_Map: /* struct { - hashes: []int, // 2 words - entries: [dynamic]Entry_Type, // 5 words + data: uintptr, // 1 word + size: uintptr, // 1 word + allocator: runtime.Allocator, // 2 words } */ - return (2 + (3 + 2))*build_context.word_size; + return (1 + 1 + 2)*build_context.word_size; case Type_Tuple: { i64 count, align, size; From e914a8710d6de3fbde8f03316080751e738ffd11 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 7 Nov 2022 23:17:37 +0000 Subject: [PATCH 02/62] Basic get and set support for new `map` --- core/runtime/dynamic_map_internal.odin | 90 ++++++++++++++++++-------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 07c1fbe7e..3fd86f38e 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -290,6 +290,12 @@ map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Inf return } +map_kvh_data_values_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (vs: uintptr) { + capacity := uintptr(1) << map_log2_cap(m) + return map_cell_index_dynamic(map_data(m), &info.ks, capacity) // Skip past ks to get start of vs +} + + // The only procedure which needs access to the context is the one which allocates the map. map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator) -> (result: Raw_Map, err: Allocator_Error) { @@ -490,12 +496,14 @@ map_add_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, @(optimization_mode="size") map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Allocator_Error { allocator := m.allocator + if allocator.procedure == nil { + allocator = context.allocator + } log2_capacity := map_log2_cap(m^) if m.data == 0 { - n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return - m.data = n.data + m^ = map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return return nil } @@ -512,15 +520,21 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al n := map_len(m^) for i := uintptr(0); i < capacity; i += 1 { hash := hs[i] - if map_hash_is_empty(hash) do continue - if map_hash_is_deleted(hash) do continue + if map_hash_is_empty(hash) { + continue + } + if map_hash_is_deleted(hash) { + continue + } k := map_cell_index_dynamic(ks, info_ks, i) v := map_cell_index_dynamic(vs, info_vs, i) map_insert_hash_dynamic(resized, info, hash, k, v) // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. n -= 1 - if n == 0 do break + if n == 0 { + break + } } mem_free(rawptr(ks), allocator) @@ -534,6 +548,9 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al @(optimization_mode="size") map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr) -> Allocator_Error { allocator := m.allocator + if allocator.procedure == nil { + allocator = context.allocator + } log2_capacity := map_log2_cap(m^) capacity := uintptr(1) << log2_capacity @@ -545,8 +562,7 @@ map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, ne log2_new_capacity := size_of(uintptr) - intrinsics.count_leading_zeros(new_capacity-1) if m.data == 0 { - n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return - m.data = n.data + m^ = map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return return nil } @@ -562,15 +578,21 @@ map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, ne n := map_len(m^) for i := uintptr(0); i < capacity; i += 1 { hash := hs[i] - if map_hash_is_empty(hash) do continue - if map_hash_is_deleted(hash) do continue + if map_hash_is_empty(hash) { + continue + } + if map_hash_is_deleted(hash) { + continue + } k := map_cell_index_dynamic(ks, info_ks, i) v := map_cell_index_dynamic(vs, info_vs, i) map_insert_hash_dynamic(resized, info, hash, k, v) // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. n -= 1 - if n == 0 do break + if n == 0 { + break + } } mem_free(rawptr(ks), allocator) @@ -584,12 +606,18 @@ map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, ne @(optimization_mode="size") map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Allocator_Error { allocator := m.allocator + if allocator.procedure == nil { + // TODO(bill): is this correct behaviour? + allocator = context.allocator + } // Cannot shrink the capacity if the number of items in the map would exceed // one minus the current log2 capacity's resize threshold. That is the shrunk // map needs to be within the max load factor. log2_capacity := map_log2_cap(m^) - if m.len >= map_load_factor(log2_capacity - 1) do return nil + if m.len >= map_load_factor(log2_capacity - 1) { + return nil + } shrinked := map_alloc_dynamic(info, log2_capacity - 1, allocator) or_return @@ -603,8 +631,12 @@ map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> n := map_len(m^) for i := uintptr(0); i < capacity; i += 1 { hash := hs[i] - if map_hash_is_empty(hash) do continue - if map_hash_is_deleted(hash) do continue + if map_hash_is_empty(hash) { + continue + } + if map_hash_is_deleted(hash) { + continue + } k := map_cell_index_dynamic(ks, info_ks, i) v := map_cell_index_dynamic(vs, info_vs, i) @@ -614,7 +646,9 @@ map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. n -= 1 - if n == 0 do break + if n == 0 { + break + } } free(rawptr(ks), allocator) @@ -632,7 +666,9 @@ map_free :: proc(m: Raw_Map, loc := #caller_location) -> Allocator_Error { @(optimization_mode="speed") map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (index: uintptr, ok: bool) { - if map_len(m) == 0 do return 0, false + if map_len(m) == 0 { + return 0, false + } h := info.hash(rawptr(k), 0) p := map_desired_position(m, h) d := uintptr(0) @@ -663,9 +699,9 @@ map_insert_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, map_grow_dynamic(m, info) or_return } hashed := info.hash(rawptr(k), 0) - result := map_insert_hash_dynamic(m^, info, hashed, k, v) + value = map_insert_hash_dynamic(m^, info, hashed, k, v) m.len += 1 - return result, nil + return } // Same as map_insert_dynamic but does not return address to the inserted element. @@ -680,9 +716,8 @@ map_add_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: } map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> bool { - when size_of(Map_Hash) == 4 do MASK :: Map_Hash(0x8000_0000) - when size_of(Map_Hash) == 8 do MASK :: Map_Hash(0x8000_0000_0000_0000) - when size_of(Map_Hash) == 16 do MASK :: Map_Hash(0x8000_0000_0000_0000_0000_0000_0000_0000) + MASK :: 1 << (size_of(Map_Hash)*8 - 1) + index := map_lookup_dynamic(m^, info, k) or_return _, _, hs, _, _ := map_kvh_data_dynamic(m^, info) hs[index] |= MASK @@ -691,7 +726,9 @@ map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #n } map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) { - if m.data == 0 do return + if m.data == 0 { + return + } _, _, hs, _, _ := map_kvh_data_dynamic(m^, info) intrinsics.mem_zero(rawptr(hs), map_cap(m^) * size_of(Map_Hash)) m.len = 0 @@ -705,16 +742,13 @@ __dynamic_map_get :: proc "contextless" (m: rawptr, #no_alias info: ^Map_Info, k if !ok { return nil } - _ = index - // TODO(bill) - return nil + vs := map_kvh_data_values_dynamic(rm, info) + return rawptr(map_cell_index_dynamic(vs, &info.vs, index)) } __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { - // value, _ := map_insert_dynamic(m, info, uintptr(key), uintptr(value)) - // return rawptr(value) - // TODO(bill) - return nil + value, _ := map_insert_dynamic(m, info, uintptr(key), uintptr(value)) + return rawptr(value) } __dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) { From bce62b98d4608d627b3a98b4d969fc7a7e5fd9c7 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 7 Nov 2022 23:32:59 +0000 Subject: [PATCH 03/62] Basic fmt printing for `map` --- core/fmt/fmt.odin | 44 ++++++++------------------ core/runtime/core.odin | 8 ++--- core/runtime/dynamic_map_internal.odin | 13 ++++---- src/llvm_backend.cpp | 14 ++++---- src/llvm_backend.hpp | 1 + src/llvm_backend_type.cpp | 8 ++--- 6 files changed, 32 insertions(+), 56 deletions(-) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index ce79aab9f..dbccab67b 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -2069,45 +2069,29 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { m := (^mem.Raw_Map)(v.data) if m != nil { - if info.generated_struct == nil { + if info.map_info == nil { return } - /* - entries := &m.entries - gs := runtime.type_info_base(info.generated_struct).variant.(runtime.Type_Info_Struct) - ed := runtime.type_info_base(gs.types[1]).variant.(runtime.Type_Info_Dynamic_Array) - entry_type := ed.elem.variant.(runtime.Type_Info_Struct) - entry_size := ed.elem_size - /* - NOTE: The layout of a `map` is as follows: - - map[Key]Value - - ## Internal Layout - struct { - hashes: []int, - entries: [dynamic]struct{ - hash: uintptr, - next: int, - key: Key, - value: Value, - }, + map_cap := uintptr(runtime.map_cap(m^)) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(m^, info.map_info) + j := 0 + for bucket_index in 0.. 0 { io.write_string(fi.writer, ", ", &fi.n) } - data := uintptr(entries.data) + uintptr(i*entry_size) + if j > 0 { + io.write_string(fi.writer, ", ", &fi.n) + } + j += 1 + + key := ks + bucket_index*uintptr(info.key.size) + value := vs + bucket_index*uintptr(info.value.size) - key := data + entry_type.offsets[2] // key: Key fmt_arg(&Info{writer = fi.writer}, any{rawptr(key), info.key.id}, 'v') - io.write_string(fi.writer, "=", &fi.n) - - value := data + entry_type.offsets[3] // value: Value fmt_arg(fi, any{rawptr(value), info.value.id}, 'v') } - */ } case runtime.Type_Info_Struct: diff --git a/core/runtime/core.odin b/core/runtime/core.odin index ce3aa239b..69e5128a9 100644 --- a/core/runtime/core.odin +++ b/core/runtime/core.odin @@ -143,11 +143,9 @@ Type_Info_Enum :: struct { values: []Type_Info_Enum_Value, } Type_Info_Map :: struct { - key: ^Type_Info, - value: ^Type_Info, - generated_struct: ^Type_Info, - key_equal: Equal_Proc, - key_hasher: Hasher_Proc, + key: ^Type_Info, + value: ^Type_Info, + map_info: ^Map_Info, } Type_Info_Bit_Set :: struct { elem: ^Type_Info, diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 3fd86f38e..5e1c67e1c 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -242,8 +242,8 @@ map_probe_distance :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Ha Map_Info :: struct { ks: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit vs: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit - hash: proc "contextless" (key: rawptr, seed: Map_Hash) -> Map_Hash, // 8-bytes on 64-bit, 4-bytes on 32-bit - cmp: proc "contextless" (lhs, rhs: rawptr) -> bool, // 8-bytes on 64-bit, 4-bytes on 32-bit + key_hasher: proc "contextless" (key: rawptr, seed: Map_Hash) -> Map_Hash, // 8-bytes on 64-bit, 4-bytes on 32-bit + key_equal: proc "contextless" (lhs, rhs: rawptr) -> bool, // 8-bytes on 64-bit, 4-bytes on 32-bit } @@ -669,7 +669,7 @@ map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, if map_len(m) == 0 { return 0, false } - h := info.hash(rawptr(k), 0) + h := info.key_hasher(rawptr(k), 0) p := map_desired_position(m, h) d := uintptr(0) c := (uintptr(1) << map_log2_cap(m)) - 1 @@ -681,7 +681,7 @@ map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, return 0, false } else if d > map_probe_distance(m, element_hash, p) { return 0, false - } else if element_hash == h && info.cmp(rawptr(k), rawptr(map_cell_index_dynamic(ks, info_ks, p))) { + } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info_ks, p))) { return p, true } p = (p + 1) & c @@ -698,7 +698,7 @@ map_insert_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, if map_len(m^) + 1 >= map_resize_threshold(m^) { map_grow_dynamic(m, info) or_return } - hashed := info.hash(rawptr(k), 0) + hashed := info.key_hasher(rawptr(k), 0) value = map_insert_hash_dynamic(m^, info, hashed, k, v) m.len += 1 return @@ -710,7 +710,7 @@ map_add_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: if map_len(m^) + 1 >= map_resize_threshold(m^) { map_grow_dynamic(m, info) or_return } - map_add_hash_dynamic(m^, info, info.hash(rawptr(k), 0), k, v) + map_add_hash_dynamic(m^, info, info.key_hasher(rawptr(k), 0), k, v) m.len += 1 return nil } @@ -735,7 +735,6 @@ map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #n } -// TODO(bill): Change signature to not be a `rawptr` __dynamic_map_get :: proc "contextless" (m: rawptr, #no_alias info: ^Map_Info, key: rawptr) -> rawptr { rm := (^Raw_Map)(m)^ index, ok := map_lookup_dynamic(rm, info, uintptr(key)) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 40b861d40..7c1e53b7f 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -530,15 +530,13 @@ LLVMValueRef lb_gen_map_cell_info(lbModule *m, Type *type) { return llvm_const_named_struct(m, t_map_cell_info, const_values, gb_count_of(const_values)); } -lbValue lb_gen_map_info_ptr(lbProcedure *p, Type *map_type) { - lbModule *m = p->module; - +lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type) { map_type = base_type(map_type); GB_ASSERT(map_type->kind == Type_Map); lbAddr *found = map_get(&m->map_info_map, map_type); if (found) { - return lb_addr_get_ptr(p, *found); + return found->addr; } GB_ASSERT(t_map_info != nullptr); @@ -560,7 +558,7 @@ lbValue lb_gen_map_info_ptr(lbProcedure *p, Type *map_type) { lb_make_global_private_const(addr); map_set(&m->map_info_map, map_type, addr); - return lb_addr_get_ptr(p, addr); + return addr.addr; } lbValue lb_const_hash(lbModule *m, lbValue key, Type *key_type) { @@ -637,7 +635,7 @@ lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, auto args = array_make(permanent_allocator(), 3); args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_info_ptr(p, map_type); + args[1] = lb_gen_map_info_ptr(p->module, map_type); args[2] = key_ptr; lbValue ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); @@ -659,7 +657,7 @@ void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, auto args = array_make(permanent_allocator(), 5); args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_info_ptr(p, map_type); + args[1] = lb_gen_map_info_ptr(p->module, map_type); args[2] = key_ptr; args[3] = lb_emit_conv(p, value_addr.addr, t_rawptr); args[4] = lb_emit_source_code_location_as_global(p, node); @@ -676,7 +674,7 @@ void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const auto args = array_make(permanent_allocator(), 4); args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = lb_gen_map_info_ptr(p, type_deref(map_ptr.type)); + args[1] = lb_gen_map_info_ptr(p->module, type_deref(map_ptr.type)); args[2] = lb_const_int(p->module, t_uint, capacity); args[3] = lb_emit_source_code_location_as_global(p, proc_name, pos); lb_emit_runtime_call(p, "__dynamic_map_reserve", args); diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index b797f28f9..f4d4cce21 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -447,6 +447,7 @@ String lb_get_const_string(lbModule *m, lbValue value); lbValue lb_generate_local_array(lbProcedure *p, Type *elem_type, i64 count, bool zero_init=true); lbValue lb_generate_global_array(lbModule *m, Type *elem_type, i64 count, String prefix, i64 id); lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue *key_ptr_); +lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type); lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key); void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, Type *map_type, lbValue const &map_key, lbValue const &map_value, Ast *node); diff --git a/src/llvm_backend_type.cpp b/src/llvm_backend_type.cpp index 4abf674c5..26f89f985 100644 --- a/src/llvm_backend_type.cpp +++ b/src/llvm_backend_type.cpp @@ -788,15 +788,11 @@ void lb_setup_type_info_data(lbProcedure *p) { // NOTE(bill): Setup type_info da case Type_Map: { tag = lb_const_ptr_cast(m, variant_ptr, t_type_info_map_ptr); init_map_internal_types(t); - - lbValue gst = lb_type_info(m, t->Map.internal_type); - LLVMValueRef vals[5] = { + LLVMValueRef vals[3] = { lb_type_info(m, t->Map.key).value, lb_type_info(m, t->Map.value).value, - gst.value, - lb_get_equal_proc_for_type(m, t->Map.key).value, - lb_get_hasher_proc_for_type(m, t->Map.key).value + lb_gen_map_info_ptr(p->module, t).value }; lbValue res = {}; From 2c3febd6203b3b55f6e4e98eaf30a2821489f97f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 7 Nov 2022 23:35:44 +0000 Subject: [PATCH 04/62] Correct `fmt` printing to be robust --- core/fmt/fmt.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index dbccab67b..b4118a95b 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -2085,8 +2085,8 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { } j += 1 - key := ks + bucket_index*uintptr(info.key.size) - value := vs + bucket_index*uintptr(info.value.size) + key := runtime.map_cell_index_dynamic(ks, &info.map_info.ks, bucket_index) + value := runtime.map_cell_index_dynamic(vs, &info.map_info.vs, bucket_index) fmt_arg(&Info{writer = fi.writer}, any{rawptr(key), info.key.id}, 'v') io.write_string(fi.writer, "=", &fi.n) From da774e3fd22848bdedf5aadb9d897c0c4e9d9b7a Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 00:38:31 +0000 Subject: [PATCH 05/62] General modifications --- core/fmt/fmt.odin | 3 +- core/runtime/dynamic_map_internal.odin | 48 ++++++----- src/check_type.cpp | 19 +++++ src/llvm_backend.cpp | 19 +---- src/llvm_backend.hpp | 2 - src/llvm_backend_stmt.cpp | 107 ++++++++++++++++++++++--- src/llvm_backend_utility.cpp | 46 ++++++----- 7 files changed, 166 insertions(+), 78 deletions(-) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index b4118a95b..384f43aa5 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -2075,9 +2075,10 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { map_cap := uintptr(runtime.map_cap(m^)) ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(m^, info.map_info) j := 0 + fmt_arg(fi, map_cap, 'v') for bucket_index in 0.. 0 { diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 5e1c67e1c..f173f095a 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -88,7 +88,7 @@ MAP_CACHE_LINE_SIZE :: 1 << MAP_CACHE_LINE_LOG2 // In the optimal case, len(Map_Cell(T){}.data) = 1 so the cell array can be treated // as a regular array of T, which is the case for hashes. Map_Cell :: struct($T: typeid) #align MAP_CACHE_LINE_SIZE { - data: [MAP_CACHE_LINE_SIZE / size_of(T) when size_of(T) < MAP_CACHE_LINE_SIZE else 1]T, + data: [MAP_CACHE_LINE_SIZE / size_of(T) when 0 < size_of(T) && size_of(T) < MAP_CACHE_LINE_SIZE else 1]T, } // So we can operate on a cell data structure at runtime without any type @@ -107,36 +107,38 @@ Map_Cell_Info :: struct { map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, info: ^Map_Cell_Info, index: uintptr) -> uintptr { // Micro-optimize the case when the number of elements per cell is one or two // to save on expensive integer division. + + cell_index, data_index: uintptr switch elements_per_cell := info.elements_per_cell; elements_per_cell { case 1: return base + (index * info.size_of_cell) case 2: - cell_index := index >> 1 - data_index := index & 1 + cell_index = index >> 1 + data_index = index & 1 return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) case 4: - cell_index := index >> 2 - data_index := index & 3 + cell_index = index >> 2 + data_index = index & 3 return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) case 8: - cell_index := index >> 3 - data_index := index & 7 + cell_index = index >> 3 + data_index = index & 7 return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) case 16: - cell_index := index >> 4 - data_index := index & 15 + cell_index = index >> 4 + data_index = index & 15 return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) case 32: - cell_index := index >> 5 - data_index := index & 31 + cell_index = index >> 5 + data_index = index & 31 return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) case 64: - cell_index := index >> 6 - data_index := index & 63 + cell_index = index >> 6 + data_index = index & 63 return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) case: - cell_index := index / elements_per_cell - data_index := index % elements_per_cell + cell_index = index / elements_per_cell + data_index = index % elements_per_cell return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) } } @@ -309,7 +311,7 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := return {}, .Out_Of_Memory } - capacity := uintptr(1) << log2_capacity + capacity := uintptr(1) << max(log2_capacity, MAP_MIN_LOG2_CAPACITY) @static INFO_HS := Map_Cell_Info { size_of(Map_Hash), @@ -552,6 +554,9 @@ map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, ne allocator = context.allocator } + new_capacity := new_capacity + new_capacity = max(new_capacity, uintptr(1)< rawptr { +__dynamic_map_get :: proc "contextless" (m: rawptr, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { rm := (^Raw_Map)(m)^ - index, ok := map_lookup_dynamic(rm, info, uintptr(key)) - if !ok { - return nil + if index, ok := map_lookup_dynamic(rm, info, uintptr(key)); ok { + vs := map_kvh_data_values_dynamic(rm, info) + ptr = rawptr(map_cell_index_dynamic(vs, &info.vs, index)) } - vs := map_kvh_data_values_dynamic(rm, info) - return rawptr(map_cell_index_dynamic(vs, &info.vs, index)) + return } __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { diff --git a/src/check_type.cpp b/src/check_type.cpp index ea1c90a66..39344fb2c 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2176,6 +2176,25 @@ Type *make_optional_ok_type(Type *value, bool typed) { return t; } + +// IMPORTANT NOTE(bill): This must match the definition in dynamic_map_internal.odin +enum : i64 { + MAP_CACHE_LINE_LOG2 = 6, + MAP_CACHE_LINE_SIZE = 1 << MAP_CACHE_LINE_LOG2 +}; +GB_STATIC_ASSERT(MAP_CACHE_LINE_SIZE >= 64); +void map_cell_size_and_len(Type *type, i64 *size_, i64 *len_) { + i64 elem_sz = type_size_of(type); + + i64 len = 1; + if (0 < elem_sz && elem_sz < MAP_CACHE_LINE_SIZE) { + len = MAP_CACHE_LINE_SIZE / elem_sz; + } + i64 size = align_formula(elem_sz * len, MAP_CACHE_LINE_SIZE); + if (size_) *size_ = size; + if (len_) *len_ = len; +} + void init_map_internal_types(Type *type) { GB_ASSERT(type->kind == Type_Map); GB_ASSERT(t_allocator != nullptr); diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 7c1e53b7f..fa9727106 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -500,27 +500,10 @@ lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, A return value; } -// IMPORTANT NOTE(bill): This must match the definition in dynamic_map_internal.odin -enum : i64 { - MAP_CACHE_LINE_LOG2 = 6, - MAP_CACHE_LINE_SIZE = 1 << MAP_CACHE_LINE_LOG2 -}; -GB_STATIC_ASSERT(MAP_CACHE_LINE_SIZE >= 64); -void lb_map_cell_size_and_len(Type *type, i64 *size_, i64 *len_) { - i64 elem_sz = type_size_of(type); - - i64 len = 1; - if (0 < elem_sz && elem_sz < MAP_CACHE_LINE_SIZE) { - len = MAP_CACHE_LINE_SIZE / elem_sz; - } - i64 size = align_formula(elem_sz * len, MAP_CACHE_LINE_SIZE); - if (size_) *size_ = size; - if (len_) *len_ = len; -} LLVMValueRef lb_gen_map_cell_info(lbModule *m, Type *type) { i64 size = 0, len = 0; - lb_map_cell_size_and_len(type, &size, &len); + map_cell_size_and_len(type, &size, &len); LLVMValueRef const_values[4] = {}; const_values[0] = lb_const_int(m, t_uintptr, type_size_of(type)).value; diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index f4d4cce21..2b9014d94 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -422,8 +422,6 @@ lbValue lb_dynamic_array_elem(lbProcedure *p, lbValue da); lbValue lb_dynamic_array_len(lbProcedure *p, lbValue da); lbValue lb_dynamic_array_cap(lbProcedure *p, lbValue da); lbValue lb_dynamic_array_allocator(lbProcedure *p, lbValue da); -lbValue lb_map_entries(lbProcedure *p, lbValue value); -lbValue lb_map_entries_ptr(lbProcedure *p, lbValue value); lbValue lb_map_len(lbProcedure *p, lbValue value); lbValue lb_map_cap(lbProcedure *p, lbValue value); lbValue lb_soa_struct_len(lbProcedure *p, lbValue value); diff --git a/src/llvm_backend_stmt.cpp b/src/llvm_backend_stmt.cpp index bd622d411..55b0c4d90 100644 --- a/src/llvm_backend_stmt.cpp +++ b/src/llvm_backend_stmt.cpp @@ -354,16 +354,6 @@ void lb_build_range_indexed(lbProcedure *p, lbValue expr, Type *val_type, lbValu } break; } - case Type_Map: { - lbValue entries = lb_map_entries_ptr(p, expr); - lbValue elem = lb_emit_struct_ep(p, entries, 0); - elem = lb_emit_load(p, elem); - lbValue entry = lb_emit_ptr_offset(p, elem, idx); - idx = lb_emit_load(p, lb_emit_struct_ep(p, entry, 2)); - val = lb_emit_load(p, lb_emit_struct_ep(p, entry, 3)); - - break; - } case Type_Struct: { GB_ASSERT(is_type_soa_struct(expr_type)); break; @@ -380,6 +370,99 @@ void lb_build_range_indexed(lbProcedure *p, lbValue expr, Type *val_type, lbValu if (done_) *done_ = done; } +lbValue lb_map_cell_index_static(lbProcedure *p, Type *type, lbValue cells_ptr, lbValue index) { + i64 size, N; + i64 sz = type_size_of(type); + map_cell_size_and_len(type, &size, &N); + + index = lb_emit_conv(p, index, t_uint); + + if (size == N*sz) { + lbValue elems_ptr = lb_emit_conv(p, cells_ptr, alloc_type_pointer(type)); + return lb_emit_ptr_offset(p, elems_ptr, index); + } + + // TOOD(bill): N power of two optimization to use >> and & + + lbValue N_const = lb_const_int(p->module, index.type, N); + lbValue cell_index = lb_emit_arith(p, Token_Quo, index, N_const, index.type); + lbValue data_index = lb_emit_arith(p, Token_Mod, index, N_const, index.type); + lbValue cell = lb_emit_ptr_offset(p, cells_ptr, cell_index); + lbValue elems_ptr = lb_emit_conv(p, cell, alloc_type_pointer(type)); + return lb_emit_ptr_offset(p, elems_ptr, data_index); +} + +void lb_map_kvh_data_static(lbProcedure *p, lbValue map_value, lbValue *ks_, lbValue *vs_, lbValue *hs_) { + lbValue capacity = lb_map_cap(p, map_value); + lbValue ks = lb_map_data_uintptr(p, map_value); + lbValue vs = {}; + lbValue hs = {}; + if (ks_) *ks_ = ks; + if (vs_) *vs_ = vs; + if (hs_) *hs_ = hs; +} + +void lb_build_range_map(lbProcedure *p, lbValue expr, Type *val_type, + lbValue *val_, lbValue *key_, lbBlock **loop_, lbBlock **done_) { + lbModule *m = p->module; + + Type *type = base_type(type_deref(expr.type)); + GB_ASSERT(type->kind == Type_Map); + + lbValue idx = {}; + lbBlock *loop = nullptr; + lbBlock *done = nullptr; + lbBlock *body = nullptr; + lbBlock *hash_check = nullptr; + + + lbAddr index = lb_add_local_generated(p, t_int, false); + lb_addr_store(p, index, lb_const_int(m, t_int, cast(u64)-1)); + + loop = lb_create_block(p, "for.index.loop"); + lb_emit_jump(p, loop); + lb_start_block(p, loop); + + lbValue incr = lb_emit_arith(p, Token_Add, lb_addr_load(p, index), lb_const_int(m, t_int, 1), t_int); + lb_addr_store(p, index, incr); + + hash_check = lb_create_block(p, "for.index.hash_check"); + body = lb_create_block(p, "for.index.body"); + done = lb_create_block(p, "for.index.done"); + + lbValue map_value = lb_emit_load(p, expr); + lbValue capacity = lb_map_cap(p, map_value); + lbValue cond = lb_emit_comp(p, Token_Lt, incr, capacity); + lb_emit_if(p, cond, hash_check, done); + lb_start_block(p, hash_check); + + idx = lb_addr_load(p, index); + + lbValue ks = lb_map_data_uintptr(p, map_value); + lbValue vs = lb_map_cell_index_static(p, type->Map.key, ks, capacity); + lbValue hs = lb_map_cell_index_static(p, type->Map.value, vs, capacity); + + lbValue hash = lb_emit_load(p, lb_emit_ptr_offset(p, hs, idx)); + + lbValue hash_cond = lb_emit_comp(p, Token_CmpEq, hash, lb_const_int(m, t_uintptr, 0)); + lb_emit_if(p, hash_cond, body, loop); + lb_start_block(p, body); + + // lbValue entries = lb_map_entries_ptr(p, expr); + // lbValue elem = lb_emit_struct_ep(p, entries, 0); + // elem = lb_emit_load(p, elem); + // lbValue entry = lb_emit_ptr_offset(p, elem, idx); + lbValue key = lb_const_nil(m, type->Map.key); + lbValue val = lb_const_nil(m, type->Map.value); + + + if (val_) *val_ = val; + if (key_) *key_ = key; + if (loop_) *loop_ = loop; + if (done_) *done_ = done; +} + + void lb_build_range_string(lbProcedure *p, lbValue expr, Type *val_type, lbValue *val_, lbValue *idx_, lbBlock **loop_, lbBlock **done_) { @@ -749,9 +832,7 @@ void lb_build_range_stmt(lbProcedure *p, AstRangeStmt *rs, Scope *scope) { if (is_type_pointer(type_deref(map.type))) { map = lb_emit_load(p, map); } - lbValue entries_ptr = lb_map_entries_ptr(p, map); - lbValue count_ptr = lb_emit_struct_ep(p, entries_ptr, 1); - lb_build_range_indexed(p, map, val1_type, count_ptr, &val, &key, &loop, &done); + lb_build_range_map(p, map, val1_type, &val, &key, &loop, &done); break; } case Type_Array: { diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index a54171b51..dce74126e 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -998,6 +998,7 @@ lbValue lb_emit_struct_ep(lbProcedure *p, lbValue s, i32 index) { switch (index) { case 0: result_type = get_struct_field_type(gst, 0); break; case 1: result_type = get_struct_field_type(gst, 1); break; + case 2: result_type = get_struct_field_type(gst, 2); break; } } else if (is_type_array(t)) { return lb_emit_array_epi(p, s, index); @@ -1134,6 +1135,7 @@ lbValue lb_emit_struct_ev(lbProcedure *p, lbValue s, i32 index) { switch (index) { case 0: result_type = get_struct_field_type(gst, 0); break; case 1: result_type = get_struct_field_type(gst, 1); break; + case 2: result_type = get_struct_field_type(gst, 2); break; } } break; @@ -1439,34 +1441,34 @@ lbValue lb_dynamic_array_allocator(lbProcedure *p, lbValue da) { return lb_emit_struct_ev(p, da, 3); } -lbValue lb_map_entries(lbProcedure *p, lbValue value) { - Type *t = base_type(value.type); - GB_ASSERT_MSG(t->kind == Type_Map, "%s", type_to_string(t)); - init_map_internal_types(t); - i32 index = 1; - lbValue entries = lb_emit_struct_ev(p, value, index); - return entries; -} - -lbValue lb_map_entries_ptr(lbProcedure *p, lbValue value) { - Type *t = base_type(type_deref(value.type)); - GB_ASSERT_MSG(t->kind == Type_Map, "%s", type_to_string(t)); - init_map_internal_types(t); - i32 index = 1; - lbValue entries = lb_emit_struct_ep(p, value, index); - return entries; -} - lbValue lb_map_len(lbProcedure *p, lbValue value) { - lbValue entries = lb_map_entries(p, value); - return lb_dynamic_array_len(p, entries); + GB_ASSERT(is_type_map(value.type)); + lbValue len = lb_emit_struct_ev(p, value, 1); + return lb_emit_conv(p, len, t_int); } lbValue lb_map_cap(lbProcedure *p, lbValue value) { - lbValue entries = lb_map_entries(p, value); - return lb_dynamic_array_cap(p, entries); + GB_ASSERT(is_type_map(value.type)); + lbValue zero = lb_const_int(p->module, t_uintptr, 0); + lbValue one = lb_const_int(p->module, t_uintptr, 1); + + lbValue mask = lb_const_int(p->module, t_uintptr, MAP_CACHE_LINE_SIZE-1); + + lbValue data = lb_emit_struct_ev(p, value, 0); + lbValue log2_cap = lb_emit_arith(p, Token_And, data, mask, t_uintptr); + lbValue cap = lb_emit_arith(p, Token_Shl, one, log2_cap, t_uintptr); + lbValue cmp = lb_emit_comp(p, Token_CmpEq, data, zero); + return lb_emit_conv(p, lb_emit_select(p, cmp, zero, cap), t_int); } +lbValue lb_map_data_uintptr(lbProcedure *p, lbValue value) { + GB_ASSERT(is_type_map(value.type)); + lbValue data = lb_emit_struct_ev(p, value, 0); + lbValue mask = lb_const_int(p->module, t_uintptr, MAP_CACHE_LINE_SIZE-1); + return lb_emit_arith(p, Token_And, data, mask, t_uintptr); +} + + lbValue lb_soa_struct_len(lbProcedure *p, lbValue value) { Type *t = base_type(value.type); bool is_ptr = false; From 50e10ceb3b94d7816239b1ac5d840da83c75b3d1 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 01:20:08 +0000 Subject: [PATCH 06/62] Correct hashing for `map` types --- core/fmt/fmt.odin | 5 +- core/runtime/dynamic_map_internal.odin | 179 +++++++++++++------------ 2 files changed, 95 insertions(+), 89 deletions(-) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 384f43aa5..76a1ee20f 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -2075,10 +2075,9 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { map_cap := uintptr(runtime.map_cap(m^)) ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(m^, info.map_info) j := 0 - fmt_arg(fi, map_cap, 'v') for bucket_index in 0.. 0 { diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index f173f095a..26aed2378 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -105,41 +105,22 @@ Map_Cell_Info :: struct { // Same as the above procedure but at runtime with the cell Map_Cell_Info value. map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, info: ^Map_Cell_Info, index: uintptr) -> uintptr { - // Micro-optimize the case when the number of elements per cell is one or two - // to save on expensive integer division. - - cell_index, data_index: uintptr - switch elements_per_cell := info.elements_per_cell; elements_per_cell { + // Micro-optimize the common cases to save on integer division. + elements_per_cell := uintptr(info.elements_per_cell) + size_of_cell := uintptr(info.size_of_cell) + switch elements_per_cell { case 1: - return base + (index * info.size_of_cell) + return base + (index * size_of_cell) case 2: - cell_index = index >> 1 - data_index = index & 1 - return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) - case 4: - cell_index = index >> 2 - data_index = index & 3 - return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) - case 8: - cell_index = index >> 3 - data_index = index & 7 - return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) - case 16: - cell_index = index >> 4 - data_index = index & 15 - return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) - case 32: - cell_index = index >> 5 - data_index = index & 31 - return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) - case 64: - cell_index = index >> 6 - data_index = index & 63 - return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) + cell_index := index >> 1 + data_index := index & 1 + size_of_type := uintptr(info.size_of_type) + return base + (cell_index * size_of_cell) + (data_index * size_of_type) case: - cell_index = index / elements_per_cell - data_index = index % elements_per_cell - return base + (cell_index * info.size_of_cell) + (data_index * info.size_of_type) + cell_index := index / elements_per_cell + data_index := index % elements_per_cell + size_of_type := uintptr(info.size_of_type) + return base + (cell_index * size_of_cell) + (data_index * size_of_type) } } @@ -190,23 +171,12 @@ map_log2_cap :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr { // Canonicalize the data by removing the tagged capacity stored in the lower six // bits of the data uintptr. map_data :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr { - return m.data & ~uintptr(64 - 1) + return m.data &~ uintptr(64 - 1) } - Map_Hash :: uintptr -// __get_map_key_hash :: #force_inline proc "contextless" (k: ^$K) -> uintptr { -// hasher := intrinsics.type_hasher_proc(K) -// return hasher(k, 0) -// } - -// __get_map_entry_key_ptr :: #force_inline proc "contextless" (h: Map_Header_Table, entry: ^Map_Entry_Header) -> rawptr { -// return rawptr(uintptr(entry) + h.key_offset) -// } - - // Procedure to check if a slot is empty for a given hash. This is represented // by the zero value to make the zero value useful. This is a procedure just // for prose reasons. @@ -218,6 +188,11 @@ map_hash_is_deleted :: #force_inline proc "contextless" (hash: Map_Hash) -> bool // The MSB indicates a tombstone return (hash >> ((size_of(Map_Hash) * 8) - 1)) != 0 } +map_hash_is_valid :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { + // The MSB indicates a tombstone + return hash != 0 && (hash >> ((size_of(Map_Hash) * 8) - 1)) == 0 +} + // Computes the desired position in the array. This is just index % capacity, // but a procedure as there's some math involved here to recover the capacity. @@ -242,10 +217,10 @@ map_probe_distance :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Ha // 80-bytes on 64-bit // 40-bytes on 32-bit Map_Info :: struct { - ks: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit - vs: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit - key_hasher: proc "contextless" (key: rawptr, seed: Map_Hash) -> Map_Hash, // 8-bytes on 64-bit, 4-bytes on 32-bit - key_equal: proc "contextless" (lhs, rhs: rawptr) -> bool, // 8-bytes on 64-bit, 4-bytes on 32-bit + ks: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit + vs: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit + key_hasher: proc "contextless" (key: rawptr, seed: Map_Hash) -> Map_Hash, // 8-bytes on 64-bit, 4-bytes on 32-bit + key_equal: proc "contextless" (lhs, rhs: rawptr) -> bool, // 8-bytes on 64-bit, 4-bytes on 32-bit } @@ -264,8 +239,12 @@ map_info :: #force_inline proc "contextless" ($K: typeid, $V: typeid) -> ^Map_In size_of(Map_Cell(V)), len(Map_Cell(V){}.data), }, - proc "contextless" (ptr: rawptr, seed: uintptr) -> Map_Hash { return intrinsics.type_hasher_proc(K)(ptr, seed) } , - proc "contextless" (a, b: rawptr) -> bool { return intrinsics.type_equal_proc(K)(a, b) }, + proc "contextless" (ptr: rawptr, seed: uintptr) -> Map_Hash { + return intrinsics.type_hasher_proc(K)(ptr, seed) + } , + proc "contextless" (a, b: rawptr) -> bool { + return intrinsics.type_equal_proc(K)(a, b) + }, } return &INFO } @@ -280,10 +259,10 @@ map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Inf } capacity := uintptr(1) << map_log2_cap(m) - ks = map_data(m) - vs = map_cell_index_dynamic(ks, &info.ks, capacity) // Skip past ks to get start of vs - hs_ := map_cell_index_dynamic(vs, &info.vs, capacity) // Skip past vs to get start of hs - sk = map_cell_index_dynamic(hs_, &INFO_HS, capacity) // Skip past hs to get start of sk + ks = map_data(m) + vs = map_cell_index_dynamic(ks, &info.ks, capacity) // Skip past ks to get start of vs + hs_ := map_cell_index_dynamic(vs, &info.vs, capacity) // Skip past vs to get start of hs + sk = map_cell_index_dynamic(hs_, &INFO_HS, capacity) // Skip past hs to get start of sk // Need to skip past two elements in the scratch key space to get to the start // of the scratch value space, of which there's only two elements as well. sv = map_cell_index_dynamic_const(sk, &info.ks, 2) @@ -321,16 +300,19 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := } round :: #force_inline proc "contextless" (value: uintptr) -> uintptr { - return (value + MAP_CACHE_LINE_SIZE - 1) & ~uintptr(MAP_CACHE_LINE_SIZE - 1) + return (value + MAP_CACHE_LINE_SIZE - 1) &~ uintptr(MAP_CACHE_LINE_SIZE - 1) } size := uintptr(0) size = round(map_cell_index_dynamic(size, &info.ks, capacity)) size = round(map_cell_index_dynamic(size, &info.vs, capacity)) size = round(map_cell_index_dynamic(size, &INFO_HS, capacity)) + size = round(map_cell_index_dynamic(size, &info.ks, 2)) // Two additional ks for scratch storage + size = round(map_cell_index_dynamic(size, &info.vs, 2)) // Two additional vs for scratch storage data := mem_alloc(int(size), MAP_CACHE_LINE_SIZE, allocator) or_return data_ptr := uintptr(raw_data(data)) + assert(data_ptr & 63 == 0) result = { // Tagged pointer representation for capacity. @@ -351,23 +333,36 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := // memcpy since there is no type information. // // This procedure returns the address of the just inserted value. -@(optimization_mode="size") -map_insert_hash_dynamic :: proc(m: Raw_Map, info: ^Map_Info, h: Map_Hash, k, v: uintptr) -> (result: uintptr) { +@(optimization_mode="speed") +map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { info_ks := &info.ks info_vs := &info.vs - // Storage to exchange when reducing variance. - k_storage := intrinsics.alloca(info_ks.size_of_type, MAP_CACHE_LINE_SIZE) - v_storage := intrinsics.alloca(info_vs.size_of_type, MAP_CACHE_LINE_SIZE) - intrinsics.mem_copy_non_overlapping(rawptr(k_storage), rawptr(k), info_ks.size_of_type) - intrinsics.mem_copy_non_overlapping(rawptr(v_storage), rawptr(v), info_vs.size_of_type) - h := h - p := map_desired_position(m, h) d := uintptr(0) c := (uintptr(1) << map_log2_cap(m)) - 1 // Saturating arithmetic mask - ks, vs, hs, _, _ := map_kvh_data_dynamic(m, info) + ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) + + // Avoid redundant loads of these values + size_of_k := info_ks.size_of_type + size_of_v := info_vs.size_of_type + + // Use sk and sv scratch storage space for dynamic k and v storage here. + // + // Simulate the following at runtime + // k = ik + // v = iv + // h = h + k := map_cell_index_dynamic_const(sk, info_ks, 0) + v := map_cell_index_dynamic_const(sv, info_vs, 0) + intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(ik), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(iv), size_of_v) + h := h + + // Temporary k and v dynamic storage for swap below + tk := map_cell_index_dynamic_const(sk, info_ks, 1) + tv := map_cell_index_dynamic_const(sv, info_vs, 1) for { hp := &hs[p] @@ -376,8 +371,8 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, info: ^Map_Info, h: Map_Hash, k, v: if map_hash_is_empty(element_hash) { k_dst := map_cell_index_dynamic(ks, info_ks, p) v_dst := map_cell_index_dynamic(vs, info_vs, p) - intrinsics.mem_copy_non_overlapping(rawptr(k_dst), k_storage, info_ks.size_of_type) - intrinsics.mem_copy_non_overlapping(rawptr(v_dst), v_storage, info_vs.size_of_type) + intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h return result if result != 0 else v_dst } @@ -386,8 +381,8 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, info: ^Map_Info, h: Map_Hash, k, v: if map_hash_is_deleted(element_hash) { k_dst := map_cell_index_dynamic(ks, info_ks, p) v_dst := map_cell_index_dynamic(vs, info_vs, p) - intrinsics.mem_copy_non_overlapping(rawptr(k_dst), k_storage, info_ks.size_of_type) - intrinsics.mem_copy_non_overlapping(rawptr(v_dst), v_storage, info_vs.size_of_type) + intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h return result if result != 0 else v_dst } @@ -396,16 +391,20 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, info: ^Map_Info, h: Map_Hash, k, v: result = map_cell_index_dynamic(vs, info_vs, p) } - swap :: #force_inline proc "contextless" (lhs, rhs, size: uintptr) { - tmp := intrinsics.alloca(size, MAP_CACHE_LINE_SIZE) - intrinsics.mem_copy_non_overlapping(&tmp[0], rawptr(lhs), size) - intrinsics.mem_copy_non_overlapping(rawptr(lhs), rawptr(rhs), size) - intrinsics.mem_copy_non_overlapping(rawptr(rhs), &tmp[0], size) - } + kp := map_cell_index_dynamic(ks, info_vs, p) + vp := map_cell_index_dynamic(vs, info_ks, p) - // Exchange to reduce variance. - swap(uintptr(k_storage), map_cell_index_dynamic(ks, info_ks, p), info_ks.size_of_type) - swap(uintptr(v_storage), map_cell_index_dynamic(vs, info_vs, p), info_vs.size_of_type) + // Simulate the following at runtime with dynamic storage + // + // kp^, k = k, kp^ + // vp^, v = v, vp^ + // hp^, h = h, hp^ + intrinsics.mem_copy_non_overlapping(rawptr(tk), rawptr(kp), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(tv), rawptr(vp), size_of_v) + intrinsics.mem_copy_non_overlapping(rawptr(kp), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(vp), rawptr(v), size_of_v) + intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(tk), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(tv), size_of_v) hp^, h = h, hp^ d = pd @@ -505,13 +504,14 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al log2_capacity := map_log2_cap(m^) if m.data == 0 { - m^ = map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return + n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return + m.data = n.data return nil } resized := map_alloc_dynamic(info, log2_capacity + 1, allocator) or_return - capacity := uintptr(1) << log2_capacity + old_capacity := uintptr(1) << log2_capacity ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) @@ -520,7 +520,7 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al info_vs := &info.vs n := map_len(m^) - for i := uintptr(0); i < capacity; i += 1 { + for i := uintptr(0); i < old_capacity; i += 1 { hash := hs[i] if map_hash_is_empty(hash) { continue @@ -535,7 +535,7 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al // fold it into the for loop comparator as a micro-optimization. n -= 1 if n == 0 { - break + // break } } @@ -602,7 +602,7 @@ map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, ne mem_free(rawptr(ks), allocator) - m.data = resized.data // Should copy the capacity too + m^ = resized // Should copy the capacity too return nil } @@ -656,7 +656,7 @@ map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> } } - free(rawptr(ks), allocator) + mem_free(rawptr(ks), allocator) m.data = shrinked.data // Should copy the capacity too @@ -750,8 +750,8 @@ __dynamic_map_get :: proc "contextless" (m: rawptr, #no_alias info: ^Map_Info, k } __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { - value, _ := map_insert_dynamic(m, info, uintptr(key), uintptr(value)) - return rawptr(value) + value, err := map_insert_dynamic(m, info, uintptr(key), uintptr(value)) + return rawptr(value) if err == nil else nil } __dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) { @@ -763,11 +763,14 @@ __dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Ma INITIAL_HASH_SEED :: 0xcbf29ce484222325 +HASH_MASK :: 1 << (8*size_of(uintptr) - 1) -1 + _fnv64a :: proc "contextless" (data: []byte, seed: u64 = INITIAL_HASH_SEED) -> u64 { h: u64 = seed for b in data { h = (h ~ u64(b)) * 0x100000001b3 } + h &= HASH_MASK return h | u64(h == 0) } @@ -791,6 +794,7 @@ _default_hasher_const :: #force_inline proc "contextless" (data: rawptr, seed: u h = (h ~ b) * 0x100000001b3 p += 1 } + h &= HASH_MASK return uintptr(h) | uintptr(h == 0) } @@ -802,6 +806,7 @@ default_hasher_n :: #force_inline proc "contextless" (data: rawptr, seed: uintpt h = (h ~ b) * 0x100000001b3 p += 1 } + h &= HASH_MASK return uintptr(h) | uintptr(h == 0) } @@ -830,6 +835,7 @@ default_hasher_string :: proc "contextless" (data: rawptr, seed: uintptr) -> uin for b in str { h = (h ~ u64(b)) * 0x100000001b3 } + h &= HASH_MASK return uintptr(h) | uintptr(h == 0) } default_hasher_cstring :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { @@ -840,5 +846,6 @@ default_hasher_cstring :: proc "contextless" (data: rawptr, seed: uintptr) -> ui h = (h ~ u64(b)) * 0x100000001b3 ptr += 1 } + h &= HASH_MASK return uintptr(h) | uintptr(h == 0) } From e3e225d21b613d789ebad0273b198b5509800bf0 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 11:04:37 +0000 Subject: [PATCH 07/62] Support `for in` loops for `map` --- src/llvm_backend_stmt.cpp | 30 +++++++++++++++++++++++------- src/llvm_backend_utility.cpp | 8 +++++++- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/llvm_backend_stmt.cpp b/src/llvm_backend_stmt.cpp index 55b0c4d90..3e4846f02 100644 --- a/src/llvm_backend_stmt.cpp +++ b/src/llvm_backend_stmt.cpp @@ -402,6 +402,22 @@ void lb_map_kvh_data_static(lbProcedure *p, lbValue map_value, lbValue *ks_, lbV if (hs_) *hs_ = hs; } +lbValue lb_map_hash_is_valid(lbProcedure *p, lbValue hash) { + // N :: size_of(uintptr)*8 - 1 + // (hash != 0) & (hash>>N == 0) + + u64 top_bit_index = cast(u64)(type_size_of(t_uintptr)*8 - 1); + lbValue shift_amount = lb_const_int(p->module, t_uintptr, top_bit_index); + lbValue zero = lb_const_int(p->module, t_uintptr, 0); + + lbValue not_empty = lb_emit_comp(p, Token_NotEq, hash, zero); + + lbValue not_deleted = lb_emit_arith(p, Token_Shr, hash, shift_amount, t_uintptr); + not_deleted = lb_emit_comp(p, Token_CmpEq, not_deleted, zero); + + return lb_emit_arith(p, Token_And, not_deleted, not_empty, t_uintptr); +} + void lb_build_range_map(lbProcedure *p, lbValue expr, Type *val_type, lbValue *val_, lbValue *key_, lbBlock **loop_, lbBlock **done_) { lbModule *m = p->module; @@ -442,19 +458,19 @@ void lb_build_range_map(lbProcedure *p, lbValue expr, Type *val_type, lbValue vs = lb_map_cell_index_static(p, type->Map.key, ks, capacity); lbValue hs = lb_map_cell_index_static(p, type->Map.value, vs, capacity); + // NOTE(bill): no need to use lb_map_cell_index_static for that hashes + // since it will always be packed without padding into the cells lbValue hash = lb_emit_load(p, lb_emit_ptr_offset(p, hs, idx)); - lbValue hash_cond = lb_emit_comp(p, Token_CmpEq, hash, lb_const_int(m, t_uintptr, 0)); + lbValue hash_cond = lb_map_hash_is_valid(p, hash); lb_emit_if(p, hash_cond, body, loop); lb_start_block(p, body); - // lbValue entries = lb_map_entries_ptr(p, expr); - // lbValue elem = lb_emit_struct_ep(p, entries, 0); - // elem = lb_emit_load(p, elem); - // lbValue entry = lb_emit_ptr_offset(p, elem, idx); - lbValue key = lb_const_nil(m, type->Map.key); - lbValue val = lb_const_nil(m, type->Map.value); + lbValue key_ptr = lb_map_cell_index_static(p, type->Map.key, ks, idx); + lbValue val_ptr = lb_map_cell_index_static(p, type->Map.value, vs, idx); + lbValue key = lb_emit_load(p, key_ptr); + lbValue val = lb_emit_load(p, val_ptr); if (val_) *val_ = val; if (key_) *key_ = key; diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index dce74126e..cbe690155 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -1464,7 +1464,13 @@ lbValue lb_map_cap(lbProcedure *p, lbValue value) { lbValue lb_map_data_uintptr(lbProcedure *p, lbValue value) { GB_ASSERT(is_type_map(value.type)); lbValue data = lb_emit_struct_ev(p, value, 0); - lbValue mask = lb_const_int(p->module, t_uintptr, MAP_CACHE_LINE_SIZE-1); + u64 mask_value = 0; + if (build_context.word_size == 4) { + mask_value = 0xfffffffful & ~(MAP_CACHE_LINE_SIZE-1); + } else { + mask_value = 0xffffffffffffffffull & ~(MAP_CACHE_LINE_SIZE-1); + } + lbValue mask = lb_const_int(p->module, t_uintptr, mask_value); return lb_emit_arith(p, Token_And, data, mask, t_uintptr); } From 810a1eee41cc8e047759c8934af70d6e68113082 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 11:13:46 +0000 Subject: [PATCH 08/62] Remove the need for `type->Map.internal_type` and replace with the definition of `runtime.Raw_Map` --- src/check_expr.cpp | 1 - src/check_type.cpp | 24 ++---------------------- src/checker.cpp | 11 +++++++---- src/llvm_backend_debug.cpp | 3 ++- src/llvm_backend_general.cpp | 20 ++------------------ src/llvm_backend_utility.cpp | 17 +++++++---------- src/types.cpp | 2 +- 7 files changed, 21 insertions(+), 57 deletions(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 043b98173..c2753e979 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -1364,7 +1364,6 @@ bool is_polymorphic_type_assignable(CheckerContext *c, Type *poly, Type *source, bool key = is_polymorphic_type_assignable(c, poly->Map.key, source->Map.key, true, modify_type); bool value = is_polymorphic_type_assignable(c, poly->Map.value, source->Map.value, true, modify_type); if (key || value) { - poly->Map.internal_type = nullptr; poly->Map.lookup_result_type = nullptr; init_map_internal_types(poly); return true; diff --git a/src/check_type.cpp b/src/check_type.cpp index 39344fb2c..5d7f8d7b5 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2198,34 +2198,14 @@ void map_cell_size_and_len(Type *type, i64 *size_, i64 *len_) { void init_map_internal_types(Type *type) { GB_ASSERT(type->kind == Type_Map); GB_ASSERT(t_allocator != nullptr); - if (type->Map.internal_type != nullptr) return; + if (type->Map.lookup_result_type != nullptr) return; Type *key = type->Map.key; Type *value = type->Map.value; GB_ASSERT(key != nullptr); GB_ASSERT(value != nullptr); - Type *generated_struct_type = alloc_type_struct(); - - /* - struct { - data: uintptr, - size: uintptr, - allocator: runtime.Allocator, - } - */ - Scope *s = create_scope(nullptr, builtin_pkg->scope); - - auto fields = slice_make(permanent_allocator(), 3); - fields[0] = alloc_entity_field(s, make_token_ident(str_lit("data")), t_uintptr, false, 0, EntityState_Resolved); - fields[1] = alloc_entity_field(s, make_token_ident(str_lit("size")), t_uintptr, false, 1, EntityState_Resolved); - fields[2] = alloc_entity_field(s, make_token_ident(str_lit("allocator")), t_allocator, false, 2, EntityState_Resolved); - - generated_struct_type->Struct.fields = fields; - type_set_offsets(generated_struct_type); - - type->Map.internal_type = generated_struct_type; - type->Map.lookup_result_type = make_optional_ok_type(value); + type->Map.lookup_result_type = make_optional_ok_type(value); } void add_map_key_type_dependencies(CheckerContext *ctx, Type *key) { diff --git a/src/checker.cpp b/src/checker.cpp index d5d2c6026..fa3ef245b 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -1933,7 +1933,8 @@ void add_type_info_type_internal(CheckerContext *c, Type *t) { init_map_internal_types(bt); add_type_info_type_internal(c, bt->Map.key); add_type_info_type_internal(c, bt->Map.value); - add_type_info_type_internal(c, bt->Map.internal_type); + add_type_info_type_internal(c, t_uintptr); // hash value + add_type_info_type_internal(c, t_allocator); break; case Type_Tuple: @@ -2155,7 +2156,8 @@ void add_min_dep_type_info(Checker *c, Type *t) { init_map_internal_types(bt); add_min_dep_type_info(c, bt->Map.key); add_min_dep_type_info(c, bt->Map.value); - add_min_dep_type_info(c, bt->Map.internal_type); + add_min_dep_type_info(c, t_uintptr); // hash value + add_min_dep_type_info(c, t_allocator); break; case Type_Tuple: @@ -2845,9 +2847,10 @@ void init_core_map_type(Checker *c) { if (t_map_info != nullptr) { return; } - t_map_info = find_core_type(c, str_lit("Map_Info")); - t_map_cell_info = find_core_type(c, str_lit("Map_Cell_Info")); init_mem_allocator(c); + t_map_info = find_core_type(c, str_lit("Map_Info")); + t_map_cell_info = find_core_type(c, str_lit("Map_Cell_Info")); + t_raw_map = find_core_type(c, str_lit("Raw_Map")); } void init_preload(Checker *c) { diff --git a/src/llvm_backend_debug.cpp b/src/llvm_backend_debug.cpp index 8a98b7f39..e69424929 100644 --- a/src/llvm_backend_debug.cpp +++ b/src/llvm_backend_debug.cpp @@ -671,7 +671,8 @@ void lb_debug_complete_types(lbModule *m) { break; case Type_Map: - bt = bt->Map.internal_type; + GB_ASSERT(t_raw_map != nullptr); + bt = base_type(t_raw_map); /*fallthrough*/ case Type_Struct: if (file == nullptr) { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 69b1fce20..ffc7a1496 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -1931,24 +1931,8 @@ LLVMTypeRef lb_type_internal(lbModule *m, Type *type) { case Type_Map: init_map_internal_types(type); - { - Type *internal_type = type->Map.internal_type; - GB_ASSERT(internal_type->kind == Type_Struct); - - m->internal_type_level -= 1; - defer (m->internal_type_level += 1); - - unsigned field_count = cast(unsigned)(internal_type->Struct.fields.count); - GB_ASSERT(field_count == 3); - - LLVMTypeRef fields[3] = { - lb_type(m, t_uintptr), // data - lb_type(m, t_uintptr), // len - lb_type(m, t_allocator), // allocator - }; - - return LLVMStructTypeInContext(ctx, fields, field_count, false); - } + GB_ASSERT(t_raw_map != nullptr); + return lb_type_internal(m, t_raw_map); case Type_Struct: { diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index cbe690155..f4d17c7a2 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -990,15 +990,13 @@ lbValue lb_emit_struct_ep(lbProcedure *p, lbValue s, i32 index) { } } else if (is_type_map(t)) { init_map_internal_types(t); - Type *itp = alloc_type_pointer(t->Map.internal_type); + Type *itp = alloc_type_pointer(t_raw_map); s = lb_emit_transmute(p, s, itp); - Type *gst = t->Map.internal_type; - GB_ASSERT(gst->kind == Type_Struct); switch (index) { - case 0: result_type = get_struct_field_type(gst, 0); break; - case 1: result_type = get_struct_field_type(gst, 1); break; - case 2: result_type = get_struct_field_type(gst, 2); break; + case 0: result_type = get_struct_field_type(t_raw_map, 0); break; + case 1: result_type = get_struct_field_type(t_raw_map, 1); break; + case 2: result_type = get_struct_field_type(t_raw_map, 2); break; } } else if (is_type_array(t)) { return lb_emit_array_epi(p, s, index); @@ -1131,11 +1129,10 @@ lbValue lb_emit_struct_ev(lbProcedure *p, lbValue s, i32 index) { case Type_Map: { init_map_internal_types(t); - Type *gst = t->Map.internal_type; switch (index) { - case 0: result_type = get_struct_field_type(gst, 0); break; - case 1: result_type = get_struct_field_type(gst, 1); break; - case 2: result_type = get_struct_field_type(gst, 2); break; + case 0: result_type = get_struct_field_type(t_raw_map, 0); break; + case 1: result_type = get_struct_field_type(t_raw_map, 1); break; + case 2: result_type = get_struct_field_type(t_raw_map, 2); break; } } break; diff --git a/src/types.cpp b/src/types.cpp index 220d1a6ab..9b2fd30d4 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -226,7 +226,6 @@ struct TypeProc { TYPE_KIND(Map, struct { \ Type *key; \ Type *value; \ - Type *internal_type; \ Type *lookup_result_type; \ }) \ TYPE_KIND(Struct, TypeStruct) \ @@ -686,6 +685,7 @@ gb_global Type *t_source_code_location_ptr = nullptr; gb_global Type *t_map_info = nullptr; gb_global Type *t_map_cell_info = nullptr; +gb_global Type *t_raw_map = nullptr; gb_global Type *t_equal_proc = nullptr; From 45f0c812afc91038eaf2ed674c1b2d6792c6c349 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 11:21:45 +0000 Subject: [PATCH 09/62] Correct `reflect.map_entry_info_slice` --- core/reflect/map.odin | 42 +++++++++++--------------- core/runtime/dynamic_map_internal.odin | 3 +- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/core/reflect/map.odin b/core/reflect/map.odin index c56173074..e83a35baa 100644 --- a/core/reflect/map.odin +++ b/core/reflect/map.odin @@ -1,9 +1,7 @@ package reflect import "core:runtime" -import "core:mem" _ :: runtime -_ :: mem Map_Entry_Info :: struct($Key, $Value: typeid) { hash: uintptr, @@ -11,32 +9,28 @@ Map_Entry_Info :: struct($Key, $Value: typeid) { value: Value, } -map_entry_info_slice :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (entries: []Map_Entry_Info(K, V)) #no_bounds_check { +map_entry_info_slice :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (entries: []Map_Entry_Info(K, V), err: runtime.Allocator_Error) #no_bounds_check { m := m - rm := (^mem.Raw_Map)(&m) + rm := (^runtime.Raw_Map)(&m) info := type_info_base(type_info_of(M)).variant.(Type_Info_Map) - gs := type_info_base(info.generated_struct).variant.(Type_Info_Struct) - ed := type_info_base(gs.types[1]).variant.(Type_Info_Dynamic_Array) - entry_type := ed.elem.variant.(Type_Info_Struct) - key_offset := entry_type.offsets[2] - value_offset := entry_type.offsets[3] - entry_size := uintptr(ed.elem_size) + if info.map_info != nil { + entries = make(type_of(entries), len(m), allocator) or_return - entries = make(type_of(entries), rm.entries.len) + map_cap := uintptr(cap(m)) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(rm^, info.map_info) + entry_index := 0 + for bucket_index in 0.. bool } map_hash_is_valid :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { // The MSB indicates a tombstone - return hash != 0 && (hash >> ((size_of(Map_Hash) * 8) - 1)) == 0 + return (hash != 0) & ((hash >> ((size_of(Map_Hash) * 8) - 1)) == 0) } @@ -249,7 +249,6 @@ map_info :: #force_inline proc "contextless" ($K: typeid, $V: typeid) -> ^Map_In return &INFO } - map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (ks: uintptr, vs: uintptr, hs: [^]Map_Hash, sk: uintptr, sv: uintptr) { @static INFO_HS := Map_Cell_Info { size_of(Map_Hash), From ea263b8cc55cdfbec38dcd6bf64e9cc3fee22fb0 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 11:29:09 +0000 Subject: [PATCH 10/62] Add `runtime.map_exists_dynamic` --- core/runtime/dynamic_map_internal.odin | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 465c5bfb7..6f6a17f11 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -692,7 +692,30 @@ map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, d += 1 } } - +@(optimization_mode="speed") +map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (ok: bool) { + if map_len(m) == 0 { + return false + } + h := info.key_hasher(rawptr(k), 0) + p := map_desired_position(m, h) + d := uintptr(0) + c := (uintptr(1) << map_log2_cap(m)) - 1 + ks, _, hs, _, _ := map_kvh_data_dynamic(m, info) + info_ks := &info.ks + for { + element_hash := hs[p] + if map_hash_is_empty(element_hash) { + return false + } else if d > map_probe_distance(m, element_hash, p) { + return false + } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info_ks, p))) { + return true + } + p = (p + 1) & c + d += 1 + } +} From d77269dee2abe3104df0c36bdffe157a079bec7c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 11:42:42 +0000 Subject: [PATCH 11/62] Disallow zero sized map keys --- src/types.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types.cpp b/src/types.cpp index 9b2fd30d4..c92d8a78f 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -1921,7 +1921,7 @@ bool is_type_valid_for_keys(Type *t) { if (is_type_untyped(t)) { return false; } - return is_type_comparable(t); + return type_size_of(t) > 0 && is_type_comparable(t); } bool is_type_valid_bit_set_elem(Type *t) { From 6dd4d1a9244cb7ba1c819e457d7aef493d5eb482 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 11:50:55 +0000 Subject: [PATCH 12/62] Correct reflection usage of maps --- core/encoding/json/marshal.odin | 25 +++++++--------- core/encoding/json/unmarshal.odin | 9 ++---- core/reflect/map.odin | 36 ----------------------- core/slice/map.odin | 48 ++++++++++++++----------------- 4 files changed, 36 insertions(+), 82 deletions(-) delete mode 100644 core/reflect/map.odin diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 10fb4c348..79a92a73e 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -257,21 +257,18 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: opt_write_start(w, opt, '{') or_return if m != nil { - if info.generated_struct == nil { + if info.map_info == nil { return .Unsupported_Type } - entries := &m.entries - gs := runtime.type_info_base(info.generated_struct).variant.(runtime.Type_Info_Struct) - ed := runtime.type_info_base(gs.types[1]).variant.(runtime.Type_Info_Dynamic_Array) - entry_type := ed.elem.variant.(runtime.Type_Info_Struct) - entry_size := ed.elem_size + map_cap := uintptr(runtime.map_cap(m^)) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(m^, info.map_info) + for bucket_index in 0.. (err: name: string #partial switch info in ti.variant { - case runtime.Type_Info_String: + case runtime.Type_Info_String: switch s in a { case string: name = s case cstring: name = string(s) } opt_write_key(w, opt, name) or_return - + case: return .Unsupported_Type } } diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index 19288a294..8edf1a016 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -399,12 +399,10 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm return UNSUPPORTED_TYPE } raw_map := (^mem.Raw_Map)(v.data) - if raw_map.entries.allocator.procedure == nil { - raw_map.entries.allocator = p.allocator + if raw_map.allocator.procedure == nil { + raw_map.allocator = p.allocator } - header := runtime.__get_map_header_table_runtime(t) - elem_backing := bytes_make(t.value.size, t.value.align, p.allocator) or_return defer delete(elem_backing, p.allocator) @@ -421,7 +419,6 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm return err } - key_hash := runtime.default_hasher_string(&key, 0) key_ptr := rawptr(&key) key_cstr: cstring @@ -430,7 +427,7 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm key_ptr = &key_cstr } - set_ptr := runtime.__dynamic_map_set(raw_map, header, key_hash, key_ptr, map_backing_value.data) + set_ptr := runtime.__dynamic_map_set(raw_map, t.map_info, key_ptr, map_backing_value.data) if set_ptr == nil { delete(key, p.allocator) } diff --git a/core/reflect/map.odin b/core/reflect/map.odin deleted file mode 100644 index e83a35baa..000000000 --- a/core/reflect/map.odin +++ /dev/null @@ -1,36 +0,0 @@ -package reflect - -import "core:runtime" -_ :: runtime - -Map_Entry_Info :: struct($Key, $Value: typeid) { - hash: uintptr, - key: Key, - value: Value, -} - -map_entry_info_slice :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (entries: []Map_Entry_Info(K, V), err: runtime.Allocator_Error) #no_bounds_check { - m := m - rm := (^runtime.Raw_Map)(&m) - - info := type_info_base(type_info_of(M)).variant.(Type_Info_Map) - if info.map_info != nil { - entries = make(type_of(entries), len(m), allocator) or_return - - map_cap := uintptr(cap(m)) - ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(rm^, info.map_info) - entry_index := 0 - for bucket_index in 0.. (keys: []K) { - keys = make(type_of(keys), len(m), allocator) +map_keys :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (keys: []K, err: runtime.Allocator_Error) { + keys = make(type_of(keys), len(m), allocator) or_return i := 0 for key in m { keys[i] = key @@ -15,8 +15,8 @@ map_keys :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (keys: []K) } return } -map_values :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (values: []V) { - values = make(type_of(values), len(m), allocator) +map_values :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (values: []V, err: runtime.Allocator_Error) { + values = make(type_of(values), len(m), allocator) or_return i := 0 for _, value in m { values[i] = value @@ -37,8 +37,8 @@ Map_Entry_Info :: struct($Key, $Value: typeid) { } -map_entries :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (entries: []Map_Entry(K, V)) { - entries = make(type_of(entries), len(m), allocator) +map_entries :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (entries: []Map_Entry(K, V), err: runtime.Allocator) { + entries = make(type_of(entries), len(m), allocator) or_return i := 0 for key, value in m { entries[i].key = key @@ -52,28 +52,24 @@ map_entry_infos :: proc(m: $M/map[$K]$V, allocator := context.allocator) -> (ent m := m rm := (^runtime.Raw_Map)(&m) - info := runtime.type_info_base(type_info_of(M)).variant.(runtime.Type_Info_Map) - gs := runtime.type_info_base(info.generated_struct).variant.(runtime.Type_Info_Struct) - ed := runtime.type_info_base(gs.types[1]).variant.(runtime.Type_Info_Dynamic_Array) - entry_type := ed.elem.variant.(runtime.Type_Info_Struct) - key_offset := entry_type.offsets[2] - value_offset := entry_type.offsets[3] - entry_size := uintptr(ed.elem_size) + info := type_info_base(type_info_of(M)).variant.(Type_Info_Map) + if info.map_info != nil { + entries = make(type_of(entries), len(m), allocator) or_return - entries = make(type_of(entries), rm.entries.len) + map_cap := uintptr(cap(m)) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(rm^, info.map_info) + entry_index := 0 + for bucket_index in 0.. Date: Tue, 8 Nov 2022 12:18:36 +0000 Subject: [PATCH 13/62] Make `Map_Info` store pointers to cell info rather than inline --- core/fmt/fmt.odin | 4 +- core/runtime/core_builtin.odin | 7 +-- core/runtime/dynamic_map_internal.odin | 85 ++++++++++---------------- src/check_builtin.cpp | 23 +++++++ src/checker.cpp | 3 + src/checker_builtin_procs.hpp | 2 + src/llvm_backend.cpp | 14 ++++- src/llvm_backend.hpp | 3 +- src/llvm_backend_general.cpp | 1 + src/llvm_backend_proc.cpp | 3 + src/types.cpp | 2 + 11 files changed, 86 insertions(+), 61 deletions(-) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 76a1ee20f..9dd38eb29 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -2085,8 +2085,8 @@ fmt_value :: proc(fi: ^Info, v: any, verb: rune) { } j += 1 - key := runtime.map_cell_index_dynamic(ks, &info.map_info.ks, bucket_index) - value := runtime.map_cell_index_dynamic(vs, &info.map_info.vs, bucket_index) + key := runtime.map_cell_index_dynamic(ks, info.map_info.ks, bucket_index) + value := runtime.map_cell_index_dynamic(vs, info.map_info.vs, bucket_index) fmt_arg(&Info{writer = fi.writer}, any{rawptr(key), info.key.id}, 'v') io.write_string(fi.writer, "=", &fi.n) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 80a9f2944..5093fbbd7 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -272,13 +272,13 @@ clear_map :: proc "contextless" (m: ^$T/map[$K]$V) { if m == nil { return } - map_clear_dynamic((^Raw_Map)(m), map_info(K, V)) + map_clear_dynamic((^Raw_Map)(m), map_info(T)) } @builtin reserve_map :: proc(m: ^$T/map[$K]$V, capacity: int, loc := #caller_location) { if m != nil { - __dynamic_map_reserve((^Raw_Map)(m), map_info(K, V), uint(capacity), loc) + __dynamic_map_reserve((^Raw_Map)(m), map_info(T), uint(capacity), loc) } } @@ -306,8 +306,7 @@ shrink_map :: proc(m: ^$T/map[$K]$V, new_cap := -1, loc := #caller_location) -> delete_key :: proc(m: ^$T/map[$K]$V, key: K) -> (deleted_key: K, deleted_value: V) { if m != nil { key := key - info := map_info(K, V) - _ = map_erase_dynamic((^Raw_Map)(m), info, uintptr(&key)) + _ = map_erase_dynamic((^Raw_Map)(m), map_info(T), uintptr(&key)) // TODO(bill) old key and value } return diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 6f6a17f11..79e1a4f30 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -184,13 +184,15 @@ map_hash_is_empty :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { return hash == 0 } -map_hash_is_deleted :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { +map_hash_is_deleted :: #force_no_inline proc "contextless" (hash: Map_Hash) -> bool { // The MSB indicates a tombstone - return (hash >> ((size_of(Map_Hash) * 8) - 1)) != 0 + N :: size_of(Map_Hash)*8 - 1 + return hash >> N != 0 } map_hash_is_valid :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { // The MSB indicates a tombstone - return (hash != 0) & ((hash >> ((size_of(Map_Hash) * 8) - 1)) == 0) + N :: size_of(Map_Hash)*8 - 1 + return (hash != 0) & (hash >> N == 0) } @@ -212,42 +214,19 @@ map_probe_distance :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Ha // about the map to make working with it possible. This info structure stores // that. // -// The Odin compiler should generate this for __get_map_header. -// -// 80-bytes on 64-bit -// 40-bytes on 32-bit +// 32-bytes on 64-bit +// 16-bytes on 32-bit Map_Info :: struct { - ks: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit - vs: Map_Cell_Info, // 32-bytes on 64-bit, 16-bytes on 32-bit + ks: ^Map_Cell_Info, // 8-bytes on 64-bit, 4-bytes on 32-bit + vs: ^Map_Cell_Info, // 8-bytes on 64-bit, 4-bytes on 32-bit key_hasher: proc "contextless" (key: rawptr, seed: Map_Hash) -> Map_Hash, // 8-bytes on 64-bit, 4-bytes on 32-bit key_equal: proc "contextless" (lhs, rhs: rawptr) -> bool, // 8-bytes on 64-bit, 4-bytes on 32-bit } // The Map_Info structure is basically a pseudo-table of information for a given K and V pair. -map_info :: #force_inline proc "contextless" ($K: typeid, $V: typeid) -> ^Map_Info where intrinsics.type_is_comparable(K) { - @static INFO := Map_Info { - Map_Cell_Info { - size_of(K), - align_of(K), - size_of(Map_Cell(K)), - len(Map_Cell(K){}.data), - }, - Map_Cell_Info { - size_of(V), - align_of(V), - size_of(Map_Cell(V)), - len(Map_Cell(V){}.data), - }, - proc "contextless" (ptr: rawptr, seed: uintptr) -> Map_Hash { - return intrinsics.type_hasher_proc(K)(ptr, seed) - } , - proc "contextless" (a, b: rawptr) -> bool { - return intrinsics.type_equal_proc(K)(a, b) - }, - } - return &INFO -} +// map_info :: proc "contextless" ($T: typeid/map[$K]$V) -> ^Map_Info {...} +map_info :: intrinsics.type_map_info map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (ks: uintptr, vs: uintptr, hs: [^]Map_Hash, sk: uintptr, sv: uintptr) { @static INFO_HS := Map_Cell_Info { @@ -259,12 +238,12 @@ map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Inf capacity := uintptr(1) << map_log2_cap(m) ks = map_data(m) - vs = map_cell_index_dynamic(ks, &info.ks, capacity) // Skip past ks to get start of vs - hs_ := map_cell_index_dynamic(vs, &info.vs, capacity) // Skip past vs to get start of hs + vs = map_cell_index_dynamic(ks, info.ks, capacity) // Skip past ks to get start of vs + hs_ := map_cell_index_dynamic(vs, info.vs, capacity) // Skip past vs to get start of hs sk = map_cell_index_dynamic(hs_, &INFO_HS, capacity) // Skip past hs to get start of sk // Need to skip past two elements in the scratch key space to get to the start // of the scratch value space, of which there's only two elements as well. - sv = map_cell_index_dynamic_const(sk, &info.ks, 2) + sv = map_cell_index_dynamic_const(sk, info.ks, 2) hs = ([^]Map_Hash)(hs_) return @@ -272,7 +251,7 @@ map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Inf map_kvh_data_values_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (vs: uintptr) { capacity := uintptr(1) << map_log2_cap(m) - return map_cell_index_dynamic(map_data(m), &info.ks, capacity) // Skip past ks to get start of vs + return map_cell_index_dynamic(map_data(m), info.ks, capacity) // Skip past ks to get start of vs } @@ -303,11 +282,11 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := } size := uintptr(0) - size = round(map_cell_index_dynamic(size, &info.ks, capacity)) - size = round(map_cell_index_dynamic(size, &info.vs, capacity)) + size = round(map_cell_index_dynamic(size, info.ks, capacity)) + size = round(map_cell_index_dynamic(size, info.vs, capacity)) size = round(map_cell_index_dynamic(size, &INFO_HS, capacity)) - size = round(map_cell_index_dynamic(size, &info.ks, 2)) // Two additional ks for scratch storage - size = round(map_cell_index_dynamic(size, &info.vs, 2)) // Two additional vs for scratch storage + size = round(map_cell_index_dynamic(size, info.ks, 2)) // Two additional ks for scratch storage + size = round(map_cell_index_dynamic(size, info.vs, 2)) // Two additional vs for scratch storage data := mem_alloc(int(size), MAP_CACHE_LINE_SIZE, allocator) or_return data_ptr := uintptr(raw_data(data)) @@ -334,8 +313,8 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := // This procedure returns the address of the just inserted value. @(optimization_mode="speed") map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { - info_ks := &info.ks - info_vs := &info.vs + info_ks := info.ks + info_vs := info.vs p := map_desired_position(m, h) d := uintptr(0) @@ -416,8 +395,8 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Ha @(optimization_mode="speed") map_add_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) { - info_ks := &info.ks - info_vs := &info.vs + info_ks := info.ks + info_vs := info.vs capacity := uintptr(1) << map_log2_cap(m) p := map_desired_position(m, h) @@ -515,8 +494,8 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) // Cache these loads to avoid hitting them in the for loop. - info_ks := &info.ks - info_vs := &info.vs + info_ks := info.ks + info_vs := info.vs n := map_len(m^) for i := uintptr(0); i < old_capacity; i += 1 { @@ -576,8 +555,8 @@ map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, ne ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) // Cache these loads to avoid hitting them in the for loop. - info_ks := &info.ks - info_vs := &info.vs + info_ks := info.ks + info_vs := info.vs n := map_len(m^) for i := uintptr(0); i < capacity; i += 1 { @@ -629,8 +608,8 @@ map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) - info_ks := &info.ks - info_vs := &info.vs + info_ks := info.ks + info_vs := info.vs n := map_len(m^) for i := uintptr(0); i < capacity; i += 1 { @@ -678,7 +657,7 @@ map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, d := uintptr(0) c := (uintptr(1) << map_log2_cap(m)) - 1 ks, _, hs, _, _ := map_kvh_data_dynamic(m, info) - info_ks := &info.ks + info_ks := info.ks for { element_hash := hs[p] if map_hash_is_empty(element_hash) { @@ -702,7 +681,7 @@ map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, d := uintptr(0) c := (uintptr(1) << map_log2_cap(m)) - 1 ks, _, hs, _, _ := map_kvh_data_dynamic(m, info) - info_ks := &info.ks + info_ks := info.ks for { element_hash := hs[p] if map_hash_is_empty(element_hash) { @@ -766,7 +745,7 @@ __dynamic_map_get :: proc "contextless" (m: rawptr, #no_alias info: ^Map_Info, k rm := (^Raw_Map)(m)^ if index, ok := map_lookup_dynamic(rm, info, uintptr(key)); ok { vs := map_kvh_data_values_dynamic(rm, info) - ptr = rawptr(map_cell_index_dynamic(vs, &info.vs, index)) + ptr = rawptr(map_cell_index_dynamic(vs, info.vs, index)) } return } diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index b8bf1bcc7..457100d6e 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -5363,6 +5363,29 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 break; } + case BuiltinProc_type_map_info: + { + Operand op = {}; + Type *bt = check_type(c, ce->args[0]); + Type *type = base_type(bt); + if (type == nullptr || type == t_invalid) { + error(ce->args[0], "Expected a type for '%.*s'", LIT(builtin_name)); + return false; + } + if (!is_type_map(type)) { + gbString t = type_to_string(type); + error(ce->args[0], "Expected a map type for '%.*s', got %s", LIT(builtin_name), t); + gb_string_free(t); + return false; + } + + add_map_key_type_dependencies(c, type); + + operand->mode = Addressing_Value; + operand->type = t_map_info_ptr; + break; + } + case BuiltinProc_constant_utf16_cstring: { String value = {}; diff --git a/src/checker.cpp b/src/checker.cpp index fa3ef245b..5b9e83bda 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -2851,6 +2851,9 @@ void init_core_map_type(Checker *c) { t_map_info = find_core_type(c, str_lit("Map_Info")); t_map_cell_info = find_core_type(c, str_lit("Map_Cell_Info")); t_raw_map = find_core_type(c, str_lit("Raw_Map")); + + t_map_info_ptr = alloc_type_pointer(t_map_info); + t_map_cell_info_ptr = alloc_type_pointer(t_map_cell_info); } void init_preload(Checker *c) { diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index c59ae7867..72639e071 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -277,6 +277,7 @@ BuiltinProc__type_simple_boolean_end, BuiltinProc_type_equal_proc, BuiltinProc_type_hasher_proc, + BuiltinProc_type_map_info, BuiltinProc__type_end, @@ -572,6 +573,7 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("type_equal_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("type_hasher_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_map_info"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index fa9727106..960ef84f6 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -502,6 +502,11 @@ lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, A LLVMValueRef lb_gen_map_cell_info(lbModule *m, Type *type) { + lbAddr *found = map_get(&m->map_cell_info_map, type); + if (found) { + return found->addr.value; + } + i64 size = 0, len = 0; map_cell_size_and_len(type, &size, &len); @@ -510,8 +515,15 @@ LLVMValueRef lb_gen_map_cell_info(lbModule *m, Type *type) { const_values[1] = lb_const_int(m, t_uintptr, type_align_of(type)).value; const_values[2] = lb_const_int(m, t_uintptr, size).value; const_values[3] = lb_const_int(m, t_uintptr, len).value; - return llvm_const_named_struct(m, t_map_cell_info, const_values, gb_count_of(const_values)); + LLVMValueRef llvm_res = llvm_const_named_struct(m, t_map_cell_info, const_values, gb_count_of(const_values)); + lbValue res = {llvm_res, t_map_cell_info}; + lbAddr addr = lb_add_global_generated(m, t_map_cell_info, res, nullptr); + lb_make_global_private_const(addr); + + map_set(&m->map_cell_info_map, type, addr); + + return addr.addr.value; } lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type) { map_type = base_type(map_type); diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index 2b9014d94..065bf4943 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -160,7 +160,8 @@ struct lbModule { StringMap objc_classes; StringMap objc_selectors; - PtrMap map_info_map; + PtrMap map_cell_info_map; // address of runtime.Map_Info + PtrMap map_info_map; // address of runtime.Map_Cell_Info }; struct lbGenerator { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index ffc7a1496..b7654614e 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -76,6 +76,7 @@ void lb_init_module(lbModule *m, Checker *c) { string_map_init(&m->objc_selectors, a); map_init(&m->map_info_map, a, 0); + map_init(&m->map_cell_info_map, a, 0); } diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index a9de7b231..3881790e0 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -2324,6 +2324,9 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, case BuiltinProc_type_hasher_proc: return lb_get_hasher_proc_for_type(p->module, ce->args[0]->tav.type); + case BuiltinProc_type_map_info: + return lb_gen_map_info_ptr(p->module, ce->args[0]->tav.type); + case BuiltinProc_fixed_point_mul: case BuiltinProc_fixed_point_div: case BuiltinProc_fixed_point_mul_sat: diff --git a/src/types.cpp b/src/types.cpp index c92d8a78f..ca15531dd 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -685,6 +685,8 @@ gb_global Type *t_source_code_location_ptr = nullptr; gb_global Type *t_map_info = nullptr; gb_global Type *t_map_cell_info = nullptr; +gb_global Type *t_map_info_ptr = nullptr; +gb_global Type *t_map_cell_info_ptr = nullptr; gb_global Type *t_raw_map = nullptr; From a74093784cea3637b445657541cb7fff2f374f50 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 12:24:00 +0000 Subject: [PATCH 14/62] Add `intrinsics.map_cell_info` and `intrinsics.map_info` --- core/intrinsics/intrinsics.odin | 3 +++ core/runtime/dynamic_map_internal.odin | 23 ++++++++--------------- src/check_builtin.cpp | 14 ++++++++++++++ src/checker_builtin_procs.hpp | 8 +++++--- src/llvm_backend.cpp | 10 +++++----- src/llvm_backend.hpp | 1 + src/llvm_backend_proc.cpp | 4 ++++ 7 files changed, 40 insertions(+), 23 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 69f77cea2..38542d2fc 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -188,6 +188,9 @@ type_field_index_of :: proc($T: typeid, $name: string) -> uintptr --- type_equal_proc :: proc($T: typeid) -> (equal: proc "contextless" (rawptr, rawptr) -> bool) where type_is_comparable(T) --- type_hasher_proc :: proc($T: typeid) -> (hasher: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr) where type_is_comparable(T) --- +type_map_info :: proc($T: typeid/map[$K]$V) -> ^runtime.Map_Info --- +type_map_cell_info :: proc($T: typeid) -> ^runtime.Map_Cell_Info --- + type_convert_variants_to_pointers :: proc($T: typeid) -> typeid where type_is_union(T) --- constant_utf16_cstring :: proc($literal: string) -> [^]u16 --- diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 79e1a4f30..983bfde35 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -103,6 +103,9 @@ Map_Cell_Info :: struct { elements_per_cell: uintptr, // 8-bytes on 64-bit, 4-bytes on 32-bits } +// map_cell_info :: proc "contextless" ($T: typeid) -> ^Map_Cell_Info {...} +map_cell_info :: intrinsics.type_map_cell_info + // Same as the above procedure but at runtime with the cell Map_Cell_Info value. map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, info: ^Map_Cell_Info, index: uintptr) -> uintptr { // Micro-optimize the common cases to save on integer division. @@ -229,18 +232,13 @@ Map_Info :: struct { map_info :: intrinsics.type_map_info map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (ks: uintptr, vs: uintptr, hs: [^]Map_Hash, sk: uintptr, sv: uintptr) { - @static INFO_HS := Map_Cell_Info { - size_of(Map_Hash), - align_of(Map_Hash), - size_of(Map_Cell(Map_Hash)), - len(Map_Cell(Map_Hash){}.data), - } + INFO_HS := intrinsics.type_map_cell_info(Map_Hash) capacity := uintptr(1) << map_log2_cap(m) ks = map_data(m) vs = map_cell_index_dynamic(ks, info.ks, capacity) // Skip past ks to get start of vs hs_ := map_cell_index_dynamic(vs, info.vs, capacity) // Skip past vs to get start of hs - sk = map_cell_index_dynamic(hs_, &INFO_HS, capacity) // Skip past hs to get start of sk + sk = map_cell_index_dynamic(hs_, INFO_HS, capacity) // Skip past hs to get start of sk // Need to skip past two elements in the scratch key space to get to the start // of the scratch value space, of which there's only two elements as well. sv = map_cell_index_dynamic_const(sk, info.ks, 2) @@ -270,21 +268,16 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := capacity := uintptr(1) << max(log2_capacity, MAP_MIN_LOG2_CAPACITY) - @static INFO_HS := Map_Cell_Info { - size_of(Map_Hash), - align_of(Map_Hash), - size_of(Map_Cell(Map_Hash)), - len(Map_Cell(Map_Hash){}.data), - } - round :: #force_inline proc "contextless" (value: uintptr) -> uintptr { return (value + MAP_CACHE_LINE_SIZE - 1) &~ uintptr(MAP_CACHE_LINE_SIZE - 1) } + INFO_HS := intrinsics.type_map_cell_info(Map_Hash) + size := uintptr(0) size = round(map_cell_index_dynamic(size, info.ks, capacity)) size = round(map_cell_index_dynamic(size, info.vs, capacity)) - size = round(map_cell_index_dynamic(size, &INFO_HS, capacity)) + size = round(map_cell_index_dynamic(size, INFO_HS, capacity)) size = round(map_cell_index_dynamic(size, info.ks, 2)) // Two additional ks for scratch storage size = round(map_cell_index_dynamic(size, info.vs, 2)) // Two additional vs for scratch storage diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 457100d6e..890f7a39b 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -5385,6 +5385,20 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 operand->type = t_map_info_ptr; break; } + case BuiltinProc_type_map_cell_info: + { + Operand op = {}; + Type *bt = check_type(c, ce->args[0]); + Type *type = base_type(bt); + if (type == nullptr || type == t_invalid) { + error(ce->args[0], "Expected a type for '%.*s'", LIT(builtin_name)); + return false; + } + + operand->mode = Addressing_Value; + operand->type = t_map_cell_info_ptr; + break; + } case BuiltinProc_constant_utf16_cstring: { diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 72639e071..e03e94ab4 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -278,6 +278,7 @@ BuiltinProc__type_simple_boolean_end, BuiltinProc_type_equal_proc, BuiltinProc_type_hasher_proc, BuiltinProc_type_map_info, + BuiltinProc_type_map_cell_info, BuiltinProc__type_end, @@ -571,9 +572,10 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("type_field_index_of"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("type_equal_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("type_hasher_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("type_map_info"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_equal_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_hasher_proc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_map_info"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_map_cell_info"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 960ef84f6..bce1fa1d1 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -501,10 +501,10 @@ lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, A } -LLVMValueRef lb_gen_map_cell_info(lbModule *m, Type *type) { +lbValue lb_gen_map_cell_info_ptr(lbModule *m, Type *type) { lbAddr *found = map_get(&m->map_cell_info_map, type); if (found) { - return found->addr.value; + return found->addr; } i64 size = 0, len = 0; @@ -523,7 +523,7 @@ LLVMValueRef lb_gen_map_cell_info(lbModule *m, Type *type) { map_set(&m->map_cell_info_map, type, addr); - return addr.addr.value; + return addr.addr; } lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type) { map_type = base_type(map_type); @@ -537,8 +537,8 @@ lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type) { GB_ASSERT(t_map_info != nullptr); GB_ASSERT(t_map_cell_info != nullptr); - LLVMValueRef key_cell_info = lb_gen_map_cell_info(m, map_type->Map.key); - LLVMValueRef value_cell_info = lb_gen_map_cell_info(m, map_type->Map.value); + LLVMValueRef key_cell_info = lb_gen_map_cell_info_ptr(m, map_type->Map.key).value; + LLVMValueRef value_cell_info = lb_gen_map_cell_info_ptr(m, map_type->Map.value).value; LLVMValueRef const_values[4] = {}; const_values[0] = key_cell_info; diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index 065bf4943..6c7c2e392 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -446,6 +446,7 @@ String lb_get_const_string(lbModule *m, lbValue value); lbValue lb_generate_local_array(lbProcedure *p, Type *elem_type, i64 count, bool zero_init=true); lbValue lb_generate_global_array(lbModule *m, Type *elem_type, i64 count, String prefix, i64 id); lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue *key_ptr_); +lbValue lb_gen_map_cell_info_ptr(lbModule *m, Type *type); lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type); lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key); diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 3881790e0..6d7d7eecb 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -2327,6 +2327,10 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, case BuiltinProc_type_map_info: return lb_gen_map_info_ptr(p->module, ce->args[0]->tav.type); + case BuiltinProc_type_map_cell_info: + return lb_gen_map_cell_info_ptr(p->module, ce->args[0]->tav.type); + + case BuiltinProc_fixed_point_mul: case BuiltinProc_fixed_point_div: case BuiltinProc_fixed_point_mul_sat: From 2fc3da3fde70e4428d23e5f58b93482148c8d2ae Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 12:29:20 +0000 Subject: [PATCH 15/62] Change `Raw_Map.len` to `int` from `uintptr` --- core/runtime/core.odin | 2 +- core/runtime/dynamic_map_internal.odin | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/runtime/core.odin b/core/runtime/core.odin index 69e5128a9..108609f78 100644 --- a/core/runtime/core.odin +++ b/core/runtime/core.odin @@ -416,7 +416,7 @@ Raw_Map :: struct { // Map_Hash directly, though for consistency sake it's written as if it were // an array of Map_Cell(Map_Hash). data: uintptr, // 8-bytes on 64-bits, 4-bytes on 32-bits - len: uintptr, // 8-bytes on 64-bits, 4-bytes on 32-bits + len: int, // 8-bytes on 64-bits, 4-bytes on 32-bits allocator: Allocator, // 16-bytes on 64-bits, 8-bytes on 32-bits } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 983bfde35..4218912c9 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -139,7 +139,7 @@ map_cell_index_dynamic_const :: proc "contextless" (base: uintptr, #no_alias inf // len() for map map_len :: #force_inline proc "contextless" (m: Raw_Map) -> int { - return int(m.len) + return m.len } // cap() for map @@ -591,7 +591,7 @@ map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> // one minus the current log2 capacity's resize threshold. That is the shrunk // map needs to be within the max load factor. log2_capacity := map_log2_cap(m^) - if m.len >= map_load_factor(log2_capacity - 1) { + if uintptr(m.len) >= map_load_factor(log2_capacity - 1) { return nil } From 046dd5503211c617a88d7de7d089dd5b74e63500 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 13:02:32 +0000 Subject: [PATCH 16/62] Change `__dynamic_map_get` signature --- core/runtime/dynamic_map_internal.odin | 173 +++++++++++++------------ src/checker.cpp | 1 + src/llvm_backend.cpp | 19 ++- src/llvm_backend_expr.cpp | 4 +- src/llvm_backend_general.cpp | 4 +- src/llvm_backend_utility.cpp | 23 ++-- src/types.cpp | 3 +- 7 files changed, 113 insertions(+), 114 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 4218912c9..7e453b4b8 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -217,6 +217,9 @@ map_probe_distance :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Ha // about the map to make working with it possible. This info structure stores // that. // +// `Map_Info` and `Map_Cell_Info` are read only data structures and cannot be +// modified after creation +// // 32-bytes on 64-bit // 16-bytes on 32-bit Map_Info :: struct { @@ -255,7 +258,7 @@ map_kvh_data_values_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^ // The only procedure which needs access to the context is the one which allocates the map. -map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator) -> (result: Raw_Map, err: Allocator_Error) { +map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator, loc := #caller_location) -> (result: Raw_Map, err: Allocator_Error) { if log2_capacity == 0 { // Empty map, but set the allocator. return { 0, 0, allocator }, nil @@ -268,8 +271,9 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := capacity := uintptr(1) << max(log2_capacity, MAP_MIN_LOG2_CAPACITY) + CACHE_MASK :: MAP_CACHE_LINE_SIZE - 1 round :: #force_inline proc "contextless" (value: uintptr) -> uintptr { - return (value + MAP_CACHE_LINE_SIZE - 1) &~ uintptr(MAP_CACHE_LINE_SIZE - 1) + return (value + CACHE_MASK) &~ CACHE_MASK } INFO_HS := intrinsics.type_map_cell_info(Map_Hash) @@ -281,19 +285,20 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := size = round(map_cell_index_dynamic(size, info.ks, 2)) // Two additional ks for scratch storage size = round(map_cell_index_dynamic(size, info.vs, 2)) // Two additional vs for scratch storage - data := mem_alloc(int(size), MAP_CACHE_LINE_SIZE, allocator) or_return + data := mem_alloc_non_zeroed(int(size), MAP_CACHE_LINE_SIZE, allocator, loc) or_return data_ptr := uintptr(raw_data(data)) - assert(data_ptr & 63 == 0) + if intrinsics.expect(data_ptr & CACHE_MASK != 0, false) { + panic("allocation not aligned to a cache line", loc) + } else { + result = { + // Tagged pointer representation for capacity. + data_ptr | log2_capacity, + 0, + allocator, + } - result = { - // Tagged pointer representation for capacity. - data_ptr | log2_capacity, - 0, - allocator, + map_clear_dynamic(&result, info) } - - map_clear_dynamic(&result, info) - return } @@ -305,10 +310,7 @@ map_alloc_dynamic :: proc(info: ^Map_Info, log2_capacity: uintptr, allocator := // // This procedure returns the address of the just inserted value. @(optimization_mode="speed") -map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { - info_ks := info.ks - info_vs := info.vs - +map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { p := map_desired_position(m, h) d := uintptr(0) c := (uintptr(1) << map_log2_cap(m)) - 1 // Saturating arithmetic mask @@ -316,8 +318,8 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Ha ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) // Avoid redundant loads of these values - size_of_k := info_ks.size_of_type - size_of_v := info_vs.size_of_type + size_of_k := info.ks.size_of_type + size_of_v := info.vs.size_of_type // Use sk and sv scratch storage space for dynamic k and v storage here. // @@ -325,23 +327,23 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Ha // k = ik // v = iv // h = h - k := map_cell_index_dynamic_const(sk, info_ks, 0) - v := map_cell_index_dynamic_const(sv, info_vs, 0) + k := map_cell_index_dynamic_const(sk, info.ks, 0) + v := map_cell_index_dynamic_const(sv, info.vs, 0) intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(ik), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(iv), size_of_v) h := h // Temporary k and v dynamic storage for swap below - tk := map_cell_index_dynamic_const(sk, info_ks, 1) - tv := map_cell_index_dynamic_const(sv, info_vs, 1) + tk := map_cell_index_dynamic_const(sk, info.ks, 1) + tv := map_cell_index_dynamic_const(sv, info.vs, 1) for { hp := &hs[p] element_hash := hp^ if map_hash_is_empty(element_hash) { - k_dst := map_cell_index_dynamic(ks, info_ks, p) - v_dst := map_cell_index_dynamic(vs, info_vs, p) + k_dst := map_cell_index_dynamic(ks, info.ks, p) + v_dst := map_cell_index_dynamic(vs, info.vs, p) intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h @@ -350,8 +352,8 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Ha if pd := map_probe_distance(m, element_hash, p); pd < d { if map_hash_is_deleted(element_hash) { - k_dst := map_cell_index_dynamic(ks, info_ks, p) - v_dst := map_cell_index_dynamic(vs, info_vs, p) + k_dst := map_cell_index_dynamic(ks, info.ks, p) + v_dst := map_cell_index_dynamic(vs, info.vs, p) intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h @@ -359,11 +361,11 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Ha } if result == 0 { - result = map_cell_index_dynamic(vs, info_vs, p) + result = map_cell_index_dynamic(vs, info.vs, p) } - kp := map_cell_index_dynamic(ks, info_vs, p) - vp := map_cell_index_dynamic(vs, info_ks, p) + kp := map_cell_index_dynamic(ks, info.vs, p) + vp := map_cell_index_dynamic(vs, info.ks, p) // Simulate the following at runtime with dynamic storage // @@ -387,10 +389,7 @@ map_insert_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Ha } @(optimization_mode="speed") -map_add_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) { - info_ks := info.ks - info_vs := info.vs - +map_add_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) { capacity := uintptr(1) << map_log2_cap(m) p := map_desired_position(m, h) d := uintptr(0) @@ -399,8 +398,8 @@ map_add_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) // Avoid redundant loads of these values - size_of_k := info_ks.size_of_type - size_of_v := info_vs.size_of_type + size_of_k := info.ks.size_of_type + size_of_v := info.vs.size_of_type // Use sk and sv scratch storage space for dynamic k and v storage here. // @@ -408,23 +407,23 @@ map_add_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, // k = ik // v = iv // h = h - k := map_cell_index_dynamic_const(sk, info_ks, 0) - v := map_cell_index_dynamic_const(sv, info_vs, 0) + k := map_cell_index_dynamic_const(sk, info.ks, 0) + v := map_cell_index_dynamic_const(sv, info.vs, 0) intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(ik), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(iv), size_of_v) h := h // Temporary k and v dynamic storage for swap below - tk := map_cell_index_dynamic_const(sk, info_ks, 1) - tv := map_cell_index_dynamic_const(sv, info_vs, 1) + tk := map_cell_index_dynamic_const(sk, info.ks, 1) + tv := map_cell_index_dynamic_const(sv, info.vs, 1) for { hp := &hs[p] element_hash := hp^ if map_hash_is_empty(element_hash) { - k_dst := map_cell_index_dynamic(ks, info_ks, p) - v_dst := map_cell_index_dynamic(vs, info_vs, p) + k_dst := map_cell_index_dynamic(ks, info.ks, p) + v_dst := map_cell_index_dynamic(vs, info.vs, p) intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h @@ -433,16 +432,16 @@ map_add_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, if pd := map_probe_distance(m, element_hash, p); pd < d { if map_hash_is_deleted(element_hash) { - k_dst := map_cell_index_dynamic(ks, info_ks, p) - v_dst := map_cell_index_dynamic(vs, info_vs, p) + k_dst := map_cell_index_dynamic(ks, info.ks, p) + v_dst := map_cell_index_dynamic(vs, info.vs, p) intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h return } - kp := map_cell_index_dynamic(ks, info_vs, p) - vp := map_cell_index_dynamic(vs, info_ks, p) + kp := map_cell_index_dynamic(ks, info.vs, p) + vp := map_cell_index_dynamic(vs, info.ks, p) // Simulate the following at runtime with dynamic storage // @@ -466,7 +465,7 @@ map_add_hash_dynamic :: proc(m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, } @(optimization_mode="size") -map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Allocator_Error { +map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { allocator := m.allocator if allocator.procedure == nil { allocator = context.allocator @@ -475,23 +474,20 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al log2_capacity := map_log2_cap(m^) if m.data == 0 { - n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return + n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator, loc) or_return m.data = n.data return nil } - resized := map_alloc_dynamic(info, log2_capacity + 1, allocator) or_return + resized := map_alloc_dynamic(info, log2_capacity + 1, allocator, loc) or_return old_capacity := uintptr(1) << log2_capacity ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) // Cache these loads to avoid hitting them in the for loop. - info_ks := info.ks - info_vs := info.vs - n := map_len(m^) - for i := uintptr(0); i < old_capacity; i += 1 { + for i in 0.. Al if map_hash_is_deleted(hash) { continue } - k := map_cell_index_dynamic(ks, info_ks, i) - v := map_cell_index_dynamic(vs, info_vs, i) + k := map_cell_index_dynamic(ks, info.ks, i) + v := map_cell_index_dynamic(vs, info.vs, i) map_insert_hash_dynamic(resized, info, hash, k, v) // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. @@ -510,7 +506,7 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al } } - mem_free(rawptr(ks), allocator) + mem_free(rawptr(ks), allocator, loc) m.data = resized.data // Should copy the capacity too @@ -519,7 +515,7 @@ map_grow_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Al @(optimization_mode="size") -map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr) -> Allocator_Error { +map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr) -> Allocator_Error { allocator := m.allocator if allocator.procedure == nil { allocator = context.allocator @@ -544,15 +540,11 @@ map_reserve_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, ne resized := map_alloc_dynamic(info, log2_new_capacity, allocator) or_return - ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) // Cache these loads to avoid hitting them in the for loop. - info_ks := info.ks - info_vs := info.vs - n := map_len(m^) - for i := uintptr(0); i < capacity; i += 1 { + for i in 0.. Allocator_Error { +map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Allocator_Error { allocator := m.allocator if allocator.procedure == nil { // TODO(bill): is this correct behaviour? @@ -601,11 +593,8 @@ map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) - info_ks := info.ks - info_vs := info.vs - n := map_len(m^) - for i := uintptr(0); i < capacity; i += 1 { + for i in 0.. continue } - k := map_cell_index_dynamic(ks, info_ks, i) - v := map_cell_index_dynamic(vs, info_vs, i) + k := map_cell_index_dynamic(ks, info.ks, i) + v := map_cell_index_dynamic(vs, info.vs, i) map_insert_hash_dynamic(shrinked, info, hash, k, v) @@ -636,7 +625,7 @@ map_shrink_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> // Single procedure for static and dynamic paths. @(require_results) -map_free :: proc(m: Raw_Map, loc := #caller_location) -> Allocator_Error { +map_free :: proc "odin" (m: Raw_Map, loc := #caller_location) -> Allocator_Error { return mem_free(rawptr(map_data(m)), m.allocator, loc) } @@ -650,14 +639,13 @@ map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, d := uintptr(0) c := (uintptr(1) << map_log2_cap(m)) - 1 ks, _, hs, _, _ := map_kvh_data_dynamic(m, info) - info_ks := info.ks for { element_hash := hs[p] if map_hash_is_empty(element_hash) { return 0, false } else if d > map_probe_distance(m, element_hash, p) { return 0, false - } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info_ks, p))) { + } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, p))) { return p, true } p = (p + 1) & c @@ -674,14 +662,13 @@ map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, d := uintptr(0) c := (uintptr(1) << map_log2_cap(m)) - 1 ks, _, hs, _, _ := map_kvh_data_dynamic(m, info) - info_ks := info.ks for { element_hash := hs[p] if map_hash_is_empty(element_hash) { return false } else if d > map_probe_distance(m, element_hash, p) { return false - } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info_ks, p))) { + } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, p))) { return true } p = (p + 1) & c @@ -693,9 +680,9 @@ map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, @(optimization_mode="speed") -map_insert_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr) -> (value: uintptr, err: Allocator_Error) { +map_insert_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr, loc := #caller_location) -> (value: uintptr, err: Allocator_Error) { if map_len(m^) + 1 >= map_resize_threshold(m^) { - map_grow_dynamic(m, info) or_return + map_grow_dynamic(m, info, loc) or_return } hashed := info.key_hasher(rawptr(k), 0) value = map_insert_hash_dynamic(m^, info, hashed, k, v) @@ -705,9 +692,9 @@ map_insert_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, // Same as map_insert_dynamic but does not return address to the inserted element. @(optimization_mode="speed") -map_add_dynamic :: proc(#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr) -> Allocator_Error { +map_add_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr, loc := #caller_location) -> Allocator_Error { if map_len(m^) + 1 >= map_resize_threshold(m^) { - map_grow_dynamic(m, info) or_return + map_grow_dynamic(m, info, loc) or_return } map_add_hash_dynamic(m^, info, info.key_hasher(rawptr(k), 0), k, v) m.len += 1 @@ -734,13 +721,28 @@ map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #n } -__dynamic_map_get :: proc "contextless" (m: rawptr, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { - rm := (^Raw_Map)(m)^ - if index, ok := map_lookup_dynamic(rm, info, uintptr(key)); ok { - vs := map_kvh_data_values_dynamic(rm, info) - ptr = rawptr(map_cell_index_dynamic(vs, info.vs, index)) +__dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { + if m.len == 0 { + return nil + } + + h := info.key_hasher(key, 0) + p := map_desired_position(m, h) + d := uintptr(0) + c := (uintptr(1) << map_log2_cap(m)) - 1 + ks, vs, hs, _, _ := map_kvh_data_dynamic(m, info) + for { + element_hash := hs[p] + if map_hash_is_empty(element_hash) { + return nil + } else if d > map_probe_distance(m, element_hash, p) { + return nil + } else if element_hash == h && info.key_equal(key, rawptr(map_cell_index_dynamic(ks, info.ks, p))) { + return rawptr(map_cell_index_dynamic(vs, info.vs, p)) + } + p = (p + 1) & c + d += 1 } - return } __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { @@ -748,6 +750,7 @@ __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_In return rawptr(value) if err == nil else nil } +@(private) __dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) { map_reserve_dynamic(m, info, uintptr(new_capacity)) } diff --git a/src/checker.cpp b/src/checker.cpp index 5b9e83bda..75a6da6fa 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -2854,6 +2854,7 @@ void init_core_map_type(Checker *c) { t_map_info_ptr = alloc_type_pointer(t_map_info); t_map_cell_info_ptr = alloc_type_pointer(t_map_cell_info); + t_raw_map_ptr = alloc_type_pointer(t_raw_map); } void init_preload(Checker *c) { diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index bce1fa1d1..629daf1c9 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -622,14 +622,15 @@ lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue return hashed_key; } -lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key) { - Type *map_type = base_type(type_deref(map_ptr.type)); +lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map, lbValue const &key) { + Type *map_type = base_type(map.type); + GB_ASSERT(map_type->kind == Type_Map); lbValue key_ptr = lb_address_from_load_or_generate_local(p, key); key_ptr = lb_emit_conv(p, key_ptr, t_rawptr); auto args = array_make(permanent_allocator(), 3); - args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[0] = lb_emit_transmute(p, map, t_raw_map); args[1] = lb_gen_map_info_ptr(p->module, map_type); args[2] = key_ptr; @@ -644,17 +645,15 @@ void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, GB_ASSERT(map_type->kind == Type_Map); lbValue key_ptr = lb_address_from_load_or_generate_local(p, map_key); - key_ptr = lb_emit_conv(p, key_ptr, t_rawptr); - lbValue v = lb_emit_conv(p, map_value, map_type->Map.value); - lbAddr value_addr = lb_add_local_generated(p, v.type, false); - lb_addr_store(p, value_addr, v); + lbValue v = lb_emit_conv(p, map_value, map_type->Map.value); + lbValue value_ptr = lb_address_from_load_or_generate_local(p, v); auto args = array_make(permanent_allocator(), 5); - args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[0] = lb_emit_conv(p, map_ptr, t_raw_map_ptr); args[1] = lb_gen_map_info_ptr(p->module, map_type); - args[2] = key_ptr; - args[3] = lb_emit_conv(p, value_addr.addr, t_rawptr); + args[2] = lb_emit_conv(p, key_ptr, t_rawptr); + args[3] = lb_emit_conv(p, value_ptr, t_rawptr); args[4] = lb_emit_source_code_location_as_global(p, node); lb_emit_runtime_call(p, "__dynamic_map_set", args); } diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 05a9fdfbf..7e9aa3a78 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -1423,9 +1423,9 @@ lbValue lb_build_binary_expr(lbProcedure *p, Ast *expr) { switch (rt->kind) { case Type_Map: { - lbValue map_ptr = lb_address_from_load_or_generate_local(p, right); + lbValue map = right; lbValue key = left; - lbValue ptr = lb_internal_dynamic_map_get_ptr(p, map_ptr, key); + lbValue ptr = lb_internal_dynamic_map_get_ptr(p, map, key); if (be->op.kind == Token_in) { return lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_NotEq, ptr), t_bool); } else { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index b7654614e..e1a926255 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -417,7 +417,7 @@ lbValue lb_addr_get_ptr(lbProcedure *p, lbAddr const &addr) { switch (addr.kind) { case lbAddr_Map: - return lb_internal_dynamic_map_get_ptr(p, addr.addr, addr.map.key); + return lb_internal_dynamic_map_get_ptr(p, lb_emit_load(p, addr.addr), addr.map.key); case lbAddr_RelativePointer: { Type *rel_ptr = base_type(lb_addr_type(addr)); @@ -1074,7 +1074,7 @@ lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { GB_ASSERT(map_type->kind == Type_Map); lbAddr v = lb_add_local_generated(p, map_type->Map.lookup_result_type, true); - lbValue ptr = lb_internal_dynamic_map_get_ptr(p, addr.addr, addr.map.key); + lbValue ptr = lb_internal_dynamic_map_get_ptr(p, lb_emit_load(p, addr.addr), addr.map.key); lbValue ok = lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_NotEq, ptr), t_bool); lb_emit_store(p, lb_emit_struct_ep(p, v.addr, 1), ok); diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index f4d17c7a2..a3493f864 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -203,26 +203,19 @@ lbValue lb_emit_transmute(lbProcedure *p, lbValue value, Type *t) { if (is_type_uintptr(src) && is_type_internally_pointer_like(dst)) { res.value = LLVMBuildIntToPtr(p->builder, value.value, lb_type(m, t), ""); return res; - } - if (is_type_internally_pointer_like(src) && is_type_uintptr(dst)) { + } else if (is_type_internally_pointer_like(src) && is_type_uintptr(dst)) { res.value = LLVMBuildPtrToInt(p->builder, value.value, lb_type(m, t), ""); return res; - } - - if (is_type_integer(src) && is_type_internally_pointer_like(dst)) { + } else if (is_type_integer(src) && is_type_internally_pointer_like(dst)) { res.value = LLVMBuildIntToPtr(p->builder, value.value, lb_type(m, t), ""); return res; } else if (is_type_internally_pointer_like(src) && is_type_integer(dst)) { res.value = LLVMBuildPtrToInt(p->builder, value.value, lb_type(m, t), ""); return res; - } - - if (is_type_internally_pointer_like(src) && is_type_internally_pointer_like(dst)) { + } else if (is_type_internally_pointer_like(src) && is_type_internally_pointer_like(dst)) { res.value = LLVMBuildPointerCast(p->builder, value.value, lb_type(p->module, t), ""); return res; - } - - if (is_type_simd_vector(src) && is_type_simd_vector(dst)) { + } else if (is_type_simd_vector(src) && is_type_simd_vector(dst)) { res.value = LLVMBuildBitCast(p->builder, value.value, lb_type(p->module, t), ""); return res; } else if (is_type_array_like(src) && is_type_simd_vector(dst)) { @@ -239,9 +232,11 @@ lbValue lb_emit_transmute(lbProcedure *p, lbValue value, Type *t) { ap = lb_emit_conv(p, ap, alloc_type_pointer(value.type)); lb_emit_store(p, ap, value); return lb_addr_load(p, addr); - } - - if (lb_is_type_aggregate(src) || lb_is_type_aggregate(dst)) { + } else if (is_type_map(src) && are_types_identical(t_raw_map, t)) { + res.value = value.value; + res.type = t; + return res; + } else if (lb_is_type_aggregate(src) || lb_is_type_aggregate(dst)) { lbValue s = lb_address_from_load_or_generate_local(p, value); lbValue d = lb_emit_transmute(p, s, alloc_type_pointer(t)); return lb_emit_load(p, d); diff --git a/src/types.cpp b/src/types.cpp index ca15531dd..ab82e87b8 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -685,9 +685,10 @@ gb_global Type *t_source_code_location_ptr = nullptr; gb_global Type *t_map_info = nullptr; gb_global Type *t_map_cell_info = nullptr; +gb_global Type *t_raw_map = nullptr; gb_global Type *t_map_info_ptr = nullptr; gb_global Type *t_map_cell_info_ptr = nullptr; -gb_global Type *t_raw_map = nullptr; +gb_global Type *t_raw_map_ptr = nullptr; gb_global Type *t_equal_proc = nullptr; From a71daee545f5425aae971c0e00d7064fe53d64c7 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 14:58:05 +0000 Subject: [PATCH 17/62] Allow for `-use-static-map-calls` which generates a get procedure per `map`; add `runtime.map_get` --- core/runtime/dynamic_map_internal.odin | 104 +++++++++++++-- src/build_settings.cpp | 2 + src/check_expr.cpp | 15 ++- src/checker.cpp | 4 + src/llvm_backend.cpp | 172 +++++++++++++++++++++++-- src/llvm_backend.hpp | 1 + src/llvm_backend_general.cpp | 1 + src/llvm_backend_stmt.cpp | 26 ++-- src/llvm_backend_utility.cpp | 4 +- src/main.cpp | 6 + src/types.cpp | 1 + 11 files changed, 302 insertions(+), 34 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 7e453b4b8..b9b10dd40 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -137,6 +137,47 @@ map_cell_index_dynamic_const :: proc "contextless" (base: uintptr, #no_alias inf return base + (cell_index * size_of_cell) + (data_index * size_of_type) } +// We always round the capacity to a power of two so this becomes [16]Foo, which +// works out to [4]Cell(Foo). +// +// The following compile-time procedure indexes such a [N]Cell(T) structure as +// if it were a flat array accounting for the internal padding introduced by the +// Cell structure. +map_cell_index_static :: #force_inline proc "contextless" (cells: [^]Map_Cell($T), index: uintptr) -> ^T #no_bounds_check { + N :: size_of(Map_Cell(T){}.data) / size_of(T) when size_of(T) > 0 else 1 + + #assert(N <= MAP_CACHE_LINE_SIZE) + + // No padding case, can treat as a regular array of []T. + when size_of(Map_Cell(T)) == size_of([N]T) { + return &([^]T)(cells)[index] + } + + // Likely case, N is a power of two because T is a power of two. + when (N & (N - 1)) == 0 { + // Compute the integer log 2 of N, this is the shift amount to index the + // correct cell. Odin's intrinsics.count_leading_zeros does not produce a + // constant, hence this approach. We only need to check up to N = 64. + SHIFT :: 1 when N < 2 else + 2 when N < 4 else + 3 when N < 8 else + 4 when N < 16 else + 5 when N < 32 else 6 + #assert(SHIFT <= MAP_CACHE_LINE_LOG2) + // Unique case, no need to index data here since only one element. + when N == 1 { + return &cells[index >> SHIFT].data[0] + } else { + return &cells[index >> SHIFT].data[index & (N - 1)] + } + } + + // Least likely (and worst case), we pay for a division operation but we + // assume the compiler does not actually generate a division. N will be in the + // range [1, CACHE_LINE_SIZE) and not a power of two. + return &cells[index / N].data[index % N] +} + // len() for map map_len :: #force_inline proc "contextless" (m: Raw_Map) -> int { return m.len @@ -721,27 +762,72 @@ map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #n } +map_kvh_data_static :: #force_inline proc "contextless" (m: $T/map[$K]$V) -> ([^]Map_Cell(K), [^]Map_Cell(V), [^]Map_Hash) { + H :: Map_Hash + capacity := uintptr(cap(m)) + ks := ([^]Map_Cell(K))(map_data(transmute(Raw_Map)m)) + vs := ([^]Map_Cell(V))(map_cell_index_static(ks, capacity)) + hs := ([^]Map_Cell(H))(map_cell_index_static(vs, capacity)) + return ks, vs, ([^]Map_Hash)(hs) +} + + +map_get :: proc "contextless" (m: $T/map[$K]$V, key: K) -> (stored_key: K, stored_value: V, ok: bool) { + rm := transmute(Raw_Map)m + if rm.len == 0 { + return + } + info := intrinsics.type_map_info(T) + key := key + + h := info.key_hasher(&key, 0) + pos := map_desired_position(rm, h) + distance := uintptr(0) + mask := (uintptr(1) << map_log2_cap(rm)) - 1 + ks, vs, hs := map_kvh_data_static(m) + for { + element_hash := hs[pos] + if map_hash_is_empty(element_hash) { + return + } else if distance > map_probe_distance(rm, element_hash, pos) { + return + } else if element_hash == h { + element_key := map_cell_index_static(ks, pos) + if info.key_equal(&key, rawptr(element_key)) { + element_value := map_cell_index_static(vs, pos) + stored_key = (^K)(element_key)^ + stored_value = (^V)(element_value)^ + ok = true + return + } + + } + pos = (pos + 1) & mask + distance += 1 + } +} + __dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { if m.len == 0 { return nil } h := info.key_hasher(key, 0) - p := map_desired_position(m, h) - d := uintptr(0) - c := (uintptr(1) << map_log2_cap(m)) - 1 + pos := map_desired_position(m, h) + distance := uintptr(0) + mask := (uintptr(1) << map_log2_cap(m)) - 1 ks, vs, hs, _, _ := map_kvh_data_dynamic(m, info) for { - element_hash := hs[p] + element_hash := hs[pos] if map_hash_is_empty(element_hash) { return nil - } else if d > map_probe_distance(m, element_hash, p) { + } else if distance > map_probe_distance(m, element_hash, pos) { return nil - } else if element_hash == h && info.key_equal(key, rawptr(map_cell_index_dynamic(ks, info.ks, p))) { - return rawptr(map_cell_index_dynamic(vs, info.vs, p)) + } else if element_hash == h && info.key_equal(key, rawptr(map_cell_index_dynamic(ks, info.ks, pos))) { + return rawptr(map_cell_index_dynamic(vs, info.vs, pos)) } - p = (p + 1) & c - d += 1 + pos = (pos + 1) & mask + distance += 1 } } diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 8067d1d01..1cd2899c4 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -307,6 +307,8 @@ struct BuildContext { bool disallow_rtti; + bool use_static_map_calls; + RelocMode reloc_mode; bool disable_red_zone; diff --git a/src/check_expr.cpp b/src/check_expr.cpp index c2753e979..045b22ca2 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -3244,7 +3244,12 @@ void check_binary_expr(CheckerContext *c, Operand *x, Ast *node, Type *type_hint check_assignment(c, x, yt->Map.key, str_lit("map 'not_in'")); } - add_package_dependency(c, "runtime", "__dynamic_map_get"); + if (build_context.use_static_map_calls) { + add_package_dependency(c, "runtime", "map_desired_position"); + add_package_dependency(c, "runtime", "map_probe_distance"); + } else { + add_package_dependency(c, "runtime", "__dynamic_map_get"); + } } else if (is_type_bit_set(rhs_type)) { Type *yt = base_type(rhs_type); @@ -8992,8 +8997,14 @@ ExprKind check_index_expr(CheckerContext *c, Operand *o, Ast *node, Type *type_h o->type = t->Map.value; o->expr = node; - add_package_dependency(c, "runtime", "__dynamic_map_get"); + add_package_dependency(c, "runtime", "__dynamic_map_set"); + if (build_context.use_static_map_calls) { + add_package_dependency(c, "runtime", "map_desired_position"); + add_package_dependency(c, "runtime", "map_probe_distance"); + } else { + add_package_dependency(c, "runtime", "__dynamic_map_get"); + } return Expr_Expr; } diff --git a/src/checker.cpp b/src/checker.cpp index 75a6da6fa..d48b37b26 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -926,6 +926,10 @@ void init_universal(void) { Type *hasher_args[2] = {t_rawptr, t_uintptr}; t_hasher_proc = alloc_type_proc_from_types(hasher_args, 2, t_uintptr, false, ProcCC_Contextless); + + Type *map_get_args[2] = {/*map*/t_rawptr, /*key*/t_rawptr}; + t_map_get_proc = alloc_type_proc_from_types(map_get_args, 2, t_rawptr, false, ProcCC_Contextless); + } // Constants diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 629daf1c9..2b95c5b2f 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -157,8 +157,8 @@ lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type) { static u32 proc_index = 0; - char buf[16] = {}; - isize n = gb_snprintf(buf, 16, "__$equal%u", ++proc_index); + char buf[32] = {}; + isize n = gb_snprintf(buf, 32, "__$equal%u", ++proc_index); char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); String proc_name = make_string_c(str); @@ -280,8 +280,8 @@ lbValue lb_simple_compare_hash(lbProcedure *p, Type *type, lbValue data, lbValue i64 sz = type_size_of(type); if (1 <= sz && sz <= 16) { - char name[20] = {}; - gb_snprintf(name, 20, "default_hasher%d", cast(i32)sz); + char name[32] = {}; + gb_snprintf(name, 32, "default_hasher%d", cast(i32)sz); auto args = array_make(permanent_allocator(), 2); args[0] = data; @@ -310,8 +310,8 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { static u32 proc_index = 0; - char buf[16] = {}; - isize n = gb_snprintf(buf, 16, "__$hasher%u", ++proc_index); + char buf[32] = {}; + isize n = gb_snprintf(buf, 32, "__$hasher%u", ++proc_index); char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); String proc_name = make_string_c(str); @@ -454,6 +454,141 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { } +lbValue lb_get_map_get_proc_for_type(lbModule *m, Type *type) { + GB_ASSERT(build_context.use_static_map_calls); + type = base_type(type); + GB_ASSERT(type->kind == Type_Map); + + + lbProcedure **found = map_get(&m->map_get_procs, type); + if (found) { + GB_ASSERT(*found != nullptr); + return {(*found)->value, (*found)->type}; + } + static u32 proc_index = 0; + + char buf[32] = {}; + isize n = gb_snprintf(buf, 32, "__$map_get%u", ++proc_index); + char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); + String proc_name = make_string_c(str); + + lbProcedure *p = lb_create_dummy_procedure(m, proc_name, t_map_get_proc); + map_set(&m->map_get_procs, type, p); + lb_begin_procedure_body(p); + defer (lb_end_procedure_body(p)); + + LLVMValueRef x = LLVMGetParam(p->value, 0); + LLVMValueRef y = LLVMGetParam(p->value, 1); + lbValue map_ptr = {x, t_rawptr}; + lbValue key_ptr = {y, t_rawptr}; + + LLVMAttributeRef nonnull_attr = lb_create_enum_attribute(m->ctx, "nonnull"); + LLVMAttributeRef noalias_attr = lb_create_enum_attribute(m->ctx, "noalias"); + LLVMAddAttributeAtIndex(p->value, 1+0, nonnull_attr); + LLVMAddAttributeAtIndex(p->value, 1+0, noalias_attr); + LLVMAddAttributeAtIndex(p->value, 1+1, nonnull_attr); + LLVMAddAttributeAtIndex(p->value, 1+1, noalias_attr); + + map_ptr = lb_emit_conv(p, map_ptr, t_raw_map_ptr); + lbValue map = lb_emit_load(p, map_ptr); + + key_ptr = lb_emit_conv(p, key_ptr, alloc_type_pointer(type->Map.key)); + lbValue key = lb_emit_load(p, key_ptr); + + lbValue h = lb_gen_map_key_hash(p, key, type->Map.key, nullptr); + lbAddr pos = lb_add_local_generated(p, t_uintptr, false); + lbAddr distance = lb_add_local_generated(p, t_uintptr, true); + lbValue capacity = lb_map_cap(p, map); + lbValue mask = lb_emit_conv(p, lb_emit_arith(p, Token_Sub, capacity, lb_const_int(m, t_int, 1), t_int), t_uintptr); + + { + auto args = array_make(heap_allocator(), 2); + args[0] = map; + args[1] = h; + lb_addr_store(p, pos, lb_emit_runtime_call(p, "map_desired_position", args)); + } + lbValue zero_uintptr = lb_const_int(m, t_uintptr, 0); + lbValue one_uintptr = lb_const_int(m, t_uintptr, 1); + + lbValue ks = lb_map_data_uintptr(p, map); + lbValue vs = lb_map_cell_index_static(p, type->Map.key, ks, capacity); + lbValue hs = lb_map_cell_index_static(p, type->Map.value, vs, capacity); + + ks = lb_emit_conv(p, ks, alloc_type_pointer(type->Map.key)); + vs = lb_emit_conv(p, vs, alloc_type_pointer(type->Map.value)); + hs = lb_emit_conv(p, hs, alloc_type_pointer(t_uintptr)); + + // lbValue res = + // LLVMBuildRet(p->builder, res.value); + + lbBlock *loop = lb_create_block(p, "loop"); + lbBlock *probe_block = lb_create_block(p, "probe"); + lbBlock *increment_block = lb_create_block(p, "increment"); + lbBlock *hash_compare_block = lb_create_block(p, "hash_compare"); + lbBlock *key_compare_block = lb_create_block(p, "key_compare"); + lbBlock *value_block = lb_create_block(p, "value"); + lbBlock *nil_block = lb_create_block(p, "nil"); + + lb_emit_jump(p, loop); + lb_start_block(p, loop); + + lbValue element_hash = lb_emit_load(p, lb_emit_ptr_offset(p, hs, lb_addr_load(p, pos))); + { + // if element_hash == 0 { return nil } + lb_emit_if(p, lb_emit_comp(p, Token_CmpEq, element_hash, zero_uintptr), nil_block, probe_block); + } + + lb_start_block(p, probe_block); + { + auto args = array_make(heap_allocator(), 3); + args[0] = map; + args[1] = element_hash; + args[2] = lb_addr_load(p, pos); + lbValue probe_distance = lb_emit_runtime_call(p, "map_probe_distance", args); + lbValue cond = lb_emit_comp(p, Token_Gt, lb_addr_load(p, distance), probe_distance); + lb_emit_if(p, cond, nil_block, hash_compare_block); + } + + lb_start_block(p, hash_compare_block); + { + lb_emit_if(p, lb_emit_comp(p, Token_CmpEq, element_hash, h), key_compare_block, increment_block); + } + + lb_start_block(p, key_compare_block); + { + lbValue element_key = lb_map_cell_index_static(p, type->Map.key, ks, lb_addr_load(p, pos)); + element_key = lb_emit_conv(p, element_key, ks.type); + lbValue cond = lb_emit_comp(p, Token_CmpEq, lb_emit_load(p, element_key), key); + lb_emit_if(p, cond, value_block, increment_block); + } + + lb_start_block(p, value_block); + { + lbValue element_value = lb_map_cell_index_static(p, type->Map.value, vs, lb_addr_load(p, pos)); + element_value = lb_emit_conv(p, element_value, t_rawptr); + LLVMBuildRet(p->builder, element_value.value); + } + + lb_start_block(p, increment_block); + { + lbValue pp = lb_addr_load(p, pos); + pp = lb_emit_arith(p, Token_Add, pp, one_uintptr, t_uintptr); + pp = lb_emit_arith(p, Token_And, pp, mask, t_uintptr); + lb_addr_store(p, pos, pp); + lb_emit_increment(p, distance.addr); + } + lb_emit_jump(p, loop); + + lb_start_block(p, nil_block); + { + lbValue res = lb_const_nil(m, t_rawptr); + LLVMBuildRet(p->builder, res.value); + } + + + return {p->value, p->type}; +} + lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, Ast *expr, lbProcedure *parent) { lbProcedure **found = map_get(&m->gen->anonymous_proc_lits, expr); if (found) { @@ -626,16 +761,27 @@ lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map, lbVa Type *map_type = base_type(map.type); GB_ASSERT(map_type->kind == Type_Map); + lbValue ptr = {}; + lbValue key_ptr = lb_address_from_load_or_generate_local(p, key); key_ptr = lb_emit_conv(p, key_ptr, t_rawptr); - auto args = array_make(permanent_allocator(), 3); - args[0] = lb_emit_transmute(p, map, t_raw_map); - args[1] = lb_gen_map_info_ptr(p->module, map_type); - args[2] = key_ptr; + if (build_context.use_static_map_calls) { + lbValue map_get_proc = lb_get_map_get_proc_for_type(p->module, map_type); - lbValue ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); + auto args = array_make(permanent_allocator(), 2); + args[0] = lb_address_from_load_or_generate_local(p, map); + args[1] = key_ptr; + ptr = lb_emit_call(p, map_get_proc, args); + } else { + auto args = array_make(permanent_allocator(), 3); + args[0] = lb_emit_transmute(p, map, t_raw_map); + args[1] = lb_gen_map_info_ptr(p->module, map_type); + args[2] = key_ptr; + + ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); + } return lb_emit_conv(p, ptr, alloc_type_pointer(map_type->Map.value)); } @@ -1206,6 +1352,10 @@ WORKER_TASK_PROC(lb_llvm_function_pass_worker_proc) { lbProcedure *p = m->hasher_procs.entries[i].value; lb_run_function_pass_manager(default_function_pass_manager, p); } + for_array(i, m->map_get_procs.entries) { + lbProcedure *p = m->map_get_procs.entries[i].value; + lb_run_function_pass_manager(default_function_pass_manager, p); + } return 0; } diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index 6c7c2e392..f9fe6cff0 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -144,6 +144,7 @@ struct lbModule { PtrMap equal_procs; PtrMap hasher_procs; + PtrMap map_get_procs; u32 nested_type_name_guid; diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index e1a926255..859542fb5 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -67,6 +67,7 @@ void lb_init_module(lbModule *m, Checker *c) { map_init(&m->function_type_map, a); map_init(&m->equal_procs, a); map_init(&m->hasher_procs, a); + map_init(&m->map_get_procs, a); array_init(&m->procedures_to_generate, a, 0, 1024); array_init(&m->missing_procedures_to_check, a, 0, 16); map_init(&m->debug_values, a); diff --git a/src/llvm_backend_stmt.cpp b/src/llvm_backend_stmt.cpp index 3e4846f02..6b83068ce 100644 --- a/src/llvm_backend_stmt.cpp +++ b/src/llvm_backend_stmt.cpp @@ -371,24 +371,30 @@ void lb_build_range_indexed(lbProcedure *p, lbValue expr, Type *val_type, lbValu } lbValue lb_map_cell_index_static(lbProcedure *p, Type *type, lbValue cells_ptr, lbValue index) { - i64 size, N; - i64 sz = type_size_of(type); - map_cell_size_and_len(type, &size, &N); + i64 size, len; + i64 elem_sz = type_size_of(type); + map_cell_size_and_len(type, &size, &len); - index = lb_emit_conv(p, index, t_uint); + index = lb_emit_conv(p, index, t_uintptr); - if (size == N*sz) { + if (size == len*elem_sz) { lbValue elems_ptr = lb_emit_conv(p, cells_ptr, alloc_type_pointer(type)); return lb_emit_ptr_offset(p, elems_ptr, index); } // TOOD(bill): N power of two optimization to use >> and & - lbValue N_const = lb_const_int(p->module, index.type, N); - lbValue cell_index = lb_emit_arith(p, Token_Quo, index, N_const, index.type); - lbValue data_index = lb_emit_arith(p, Token_Mod, index, N_const, index.type); - lbValue cell = lb_emit_ptr_offset(p, cells_ptr, cell_index); - lbValue elems_ptr = lb_emit_conv(p, cell, alloc_type_pointer(type)); + lbValue size_const = lb_const_int(p->module, t_uintptr, size); + lbValue len_const = lb_const_int(p->module, t_uintptr, len); + lbValue cell_index = lb_emit_arith(p, Token_Quo, index, len_const, t_uintptr); + lbValue data_index = lb_emit_arith(p, Token_Mod, index, len_const, t_uintptr); + + lbValue elems_ptr = lb_emit_conv(p, cells_ptr, t_uintptr); + lbValue cell_offset = lb_emit_arith(p, Token_Mul, size_const, cell_index, t_uintptr); + elems_ptr = lb_emit_arith(p, Token_Add, elems_ptr, cell_offset, t_uintptr); + + elems_ptr = lb_emit_conv(p, elems_ptr, alloc_type_pointer(type)); + return lb_emit_ptr_offset(p, elems_ptr, data_index); } diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index a3493f864..6d69021ce 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -1440,7 +1440,7 @@ lbValue lb_map_len(lbProcedure *p, lbValue value) { } lbValue lb_map_cap(lbProcedure *p, lbValue value) { - GB_ASSERT(is_type_map(value.type)); + GB_ASSERT_MSG(is_type_map(value.type) || are_types_identical(value.type, t_raw_map), "%s", type_to_string(value.type)); lbValue zero = lb_const_int(p->module, t_uintptr, 0); lbValue one = lb_const_int(p->module, t_uintptr, 1); @@ -1454,7 +1454,7 @@ lbValue lb_map_cap(lbProcedure *p, lbValue value) { } lbValue lb_map_data_uintptr(lbProcedure *p, lbValue value) { - GB_ASSERT(is_type_map(value.type)); + GB_ASSERT(is_type_map(value.type) || are_types_identical(value.type, t_raw_map)); lbValue data = lb_emit_struct_ev(p, value, 0); u64 mask_value = 0; if (build_context.word_size == 4) { diff --git a/src/main.cpp b/src/main.cpp index b75137613..3b0a599db 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -638,6 +638,7 @@ enum BuildFlagKind { BuildFlag_StrictStyleInitOnly, BuildFlag_ForeignErrorProcedures, BuildFlag_DisallowRTTI, + BuildFlag_UseStaticMapCalls, BuildFlag_Compact, BuildFlag_GlobalDefinitions, @@ -814,6 +815,8 @@ bool parse_build_flags(Array args) { add_flag(&build_flags, BuildFlag_DisallowRTTI, str_lit("disallow-rtti"), BuildFlagParam_None, Command__does_check); + add_flag(&build_flags, BuildFlag_UseStaticMapCalls, str_lit("use-static-map-calls"), BuildFlagParam_None, Command__does_check); + add_flag(&build_flags, BuildFlag_Compact, str_lit("compact"), BuildFlagParam_None, Command_query); add_flag(&build_flags, BuildFlag_GlobalDefinitions, str_lit("global-definitions"), BuildFlagParam_None, Command_query); @@ -1414,6 +1417,9 @@ bool parse_build_flags(Array args) { case BuildFlag_DisallowRTTI: build_context.disallow_rtti = true; break; + case BuildFlag_UseStaticMapCalls: + build_context.use_static_map_calls = true; + break; case BuildFlag_DefaultToNilAllocator: build_context.ODIN_DEFAULT_TO_NIL_ALLOCATOR = true; break; diff --git a/src/types.cpp b/src/types.cpp index ab82e87b8..74b192010 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -693,6 +693,7 @@ gb_global Type *t_raw_map_ptr = nullptr; gb_global Type *t_equal_proc = nullptr; gb_global Type *t_hasher_proc = nullptr; +gb_global Type *t_map_get_proc = nullptr; gb_global Type *t_objc_object = nullptr; gb_global Type *t_objc_selector = nullptr; From 6a4e44607ceb6098d5e3a5cd4e7277d9a4e9c3ab Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 14:59:09 +0000 Subject: [PATCH 18/62] Fix json marshal for maps --- core/encoding/json/marshal.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 79a92a73e..1dadc8ef0 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -267,8 +267,8 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: continue } - key := rawptr(runtime.map_cell_index_dynamic(ks, &info.map_info.ks, bucket_index)) - value := rawptr(runtime.map_cell_index_dynamic(vs, &info.map_info.vs, bucket_index)) + key := rawptr(runtime.map_cell_index_dynamic(ks, info.map_info.ks, bucket_index)) + value := rawptr(runtime.map_cell_index_dynamic(vs, info.map_info.vs, bucket_index)) // check for string type { From 0819d05a0b42e9e03943344bcb316ffb65a90029 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 15:07:57 +0000 Subject: [PATCH 19/62] Fix `for in` for `map` --- src/llvm_backend_stmt.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llvm_backend_stmt.cpp b/src/llvm_backend_stmt.cpp index 6b83068ce..ec7a162cb 100644 --- a/src/llvm_backend_stmt.cpp +++ b/src/llvm_backend_stmt.cpp @@ -461,8 +461,8 @@ void lb_build_range_map(lbProcedure *p, lbValue expr, Type *val_type, idx = lb_addr_load(p, index); lbValue ks = lb_map_data_uintptr(p, map_value); - lbValue vs = lb_map_cell_index_static(p, type->Map.key, ks, capacity); - lbValue hs = lb_map_cell_index_static(p, type->Map.value, vs, capacity); + lbValue vs = lb_emit_conv(p, lb_map_cell_index_static(p, type->Map.key, ks, capacity), alloc_type_pointer(type->Map.value)); + lbValue hs = lb_emit_conv(p, lb_map_cell_index_static(p, type->Map.value, vs, capacity), alloc_type_pointer(t_uintptr)); // NOTE(bill): no need to use lb_map_cell_index_static for that hashes // since it will always be packed without padding into the cells From 2f29894b456a1846cc0e7c66c286d80558188449 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 15:15:00 +0000 Subject: [PATCH 20/62] Minor change to `map_cell_index_static` --- core/runtime/dynamic_map_internal.odin | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index b9b10dd40..e7c8a6782 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -148,13 +148,13 @@ map_cell_index_static :: #force_inline proc "contextless" (cells: [^]Map_Cell($T #assert(N <= MAP_CACHE_LINE_SIZE) - // No padding case, can treat as a regular array of []T. when size_of(Map_Cell(T)) == size_of([N]T) { - return &([^]T)(cells)[index] - } + // No padding case, can treat as a regular array of []T. + + return &([^]T)(cells)[index] + } else when (N & (N - 1)) == 0 && N <= 8*size_of(uintptr) { + // Likely case, N is a power of two because T is a power of two. - // Likely case, N is a power of two because T is a power of two. - when (N & (N - 1)) == 0 { // Compute the integer log 2 of N, this is the shift amount to index the // correct cell. Odin's intrinsics.count_leading_zeros does not produce a // constant, hence this approach. We only need to check up to N = 64. @@ -170,12 +170,12 @@ map_cell_index_static :: #force_inline proc "contextless" (cells: [^]Map_Cell($T } else { return &cells[index >> SHIFT].data[index & (N - 1)] } + } else { + // Least likely (and worst case), we pay for a division operation but we + // assume the compiler does not actually generate a division. N will be in the + // range [1, CACHE_LINE_SIZE) and not a power of two. + return &cells[index / N].data[index % N] } - - // Least likely (and worst case), we pay for a division operation but we - // assume the compiler does not actually generate a division. N will be in the - // range [1, CACHE_LINE_SIZE) and not a power of two. - return &cells[index / N].data[index % N] } // len() for map From dae299b7818bef05ce22de7c1333000405faaad2 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 15:40:30 +0000 Subject: [PATCH 21/62] Make `map_free_dynamic` take the total size of the allocation --- core/mem/alloc.odin | 3 +- core/runtime/core_builtin.odin | 2 +- core/runtime/dynamic_map_internal.odin | 51 +++++++++++++++----------- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/core/mem/alloc.odin b/core/mem/alloc.odin index 2d11b523f..05f3e145d 100644 --- a/core/mem/alloc.odin +++ b/core/mem/alloc.odin @@ -125,8 +125,7 @@ delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #cal return free(raw_data(array), allocator, loc) } delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) -> Allocator_Error { - raw := transmute(Raw_Map)m - return runtime.map_free(raw, loc) + return runtime.map_free_dynamic(transmute(Raw_Map)m, runtime.map_info(T), loc) } diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 5093fbbd7..050d91d3c 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -159,7 +159,7 @@ delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #cal } @builtin delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) -> Allocator_Error { - return map_free(transmute(Raw_Map)m, loc) + return map_free_dynamic(transmute(Raw_Map)m, map_info(T), loc) } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index e7c8a6782..e56294650 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -297,6 +297,22 @@ map_kvh_data_values_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^ } +@(private) +map_total_allocation_size :: #force_inline proc "contextless" (capacity: uintptr, info: ^Map_Info) -> uintptr { + round :: #force_inline proc "contextless" (value: uintptr) -> uintptr { + CACHE_MASK :: MAP_CACHE_LINE_SIZE - 1 + return (value + CACHE_MASK) &~ CACHE_MASK + } + INFO_HS := intrinsics.type_map_cell_info(Map_Hash) + + size := uintptr(0) + size = round(map_cell_index_dynamic(size, info.ks, capacity)) + size = round(map_cell_index_dynamic(size, info.vs, capacity)) + size = round(map_cell_index_dynamic(size, INFO_HS, capacity)) + size = round(map_cell_index_dynamic(size, info.ks, 2)) // Two additional ks for scratch storage + size = round(map_cell_index_dynamic(size, info.vs, 2)) // Two additional vs for scratch storage + return size +} // The only procedure which needs access to the context is the one which allocates the map. map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator, loc := #caller_location) -> (result: Raw_Map, err: Allocator_Error) { @@ -313,18 +329,8 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc capacity := uintptr(1) << max(log2_capacity, MAP_MIN_LOG2_CAPACITY) CACHE_MASK :: MAP_CACHE_LINE_SIZE - 1 - round :: #force_inline proc "contextless" (value: uintptr) -> uintptr { - return (value + CACHE_MASK) &~ CACHE_MASK - } - INFO_HS := intrinsics.type_map_cell_info(Map_Hash) - - size := uintptr(0) - size = round(map_cell_index_dynamic(size, info.ks, capacity)) - size = round(map_cell_index_dynamic(size, info.vs, capacity)) - size = round(map_cell_index_dynamic(size, INFO_HS, capacity)) - size = round(map_cell_index_dynamic(size, info.ks, 2)) // Two additional ks for scratch storage - size = round(map_cell_index_dynamic(size, info.vs, 2)) // Two additional vs for scratch storage + size := map_total_allocation_size(capacity, info) data := mem_alloc_non_zeroed(int(size), MAP_CACHE_LINE_SIZE, allocator, loc) or_return data_ptr := uintptr(raw_data(data)) @@ -543,11 +549,10 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf // fold it into the for loop comparator as a micro-optimization. n -= 1 if n == 0 { - // break + break } } - - mem_free(rawptr(ks), allocator, loc) + map_free_dynamic(m^, info, loc) or_return m.data = resized.data // Should copy the capacity too @@ -556,7 +561,7 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf @(optimization_mode="size") -map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr) -> Allocator_Error { +map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr, loc := #caller_location) -> Allocator_Error { allocator := m.allocator if allocator.procedure == nil { allocator = context.allocator @@ -575,11 +580,11 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ log2_new_capacity := size_of(uintptr) - intrinsics.count_leading_zeros(new_capacity-1) if m.data == 0 { - m^ = map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator) or_return + m^ = map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator, loc) or_return return nil } - resized := map_alloc_dynamic(info, log2_new_capacity, allocator) or_return + resized := map_alloc_dynamic(info, log2_new_capacity, allocator, loc) or_return ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) @@ -664,10 +669,12 @@ map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I return nil } -// Single procedure for static and dynamic paths. @(require_results) -map_free :: proc "odin" (m: Raw_Map, loc := #caller_location) -> Allocator_Error { - return mem_free(rawptr(map_data(m)), m.allocator, loc) +map_free_dynamic :: proc "odin" (m: Raw_Map, info: ^Map_Info, loc := #caller_location) -> Allocator_Error { + ptr := rawptr(map_data(m)) + size := map_total_allocation_size(uintptr(map_cap(m)), info) + + return mem_free_with_size(ptr, int(size), m.allocator, loc) } @(optimization_mode="speed") @@ -832,13 +839,13 @@ __dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, } __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { - value, err := map_insert_dynamic(m, info, uintptr(key), uintptr(value)) + value, err := map_insert_dynamic(m, info, uintptr(key), uintptr(value), loc) return rawptr(value) if err == nil else nil } @(private) __dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) { - map_reserve_dynamic(m, info, uintptr(new_capacity)) + map_reserve_dynamic(m, info, uintptr(new_capacity), loc) } From 366779f8c7c7f9e522e890151df1becadbe20aba Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 16:06:10 +0000 Subject: [PATCH 22/62] Fix bug with allocator not getting set on a `map` --- core/mem/allocators.odin | 13 ++----- core/runtime/core_builtin.odin | 16 +++----- core/runtime/dynamic_map_internal.odin | 53 +++++++++++--------------- 3 files changed, 31 insertions(+), 51 deletions(-) diff --git a/core/mem/allocators.odin b/core/mem/allocators.odin index ae248e1f3..a7c50da02 100644 --- a/core/mem/allocators.odin +++ b/core/mem/allocators.odin @@ -880,7 +880,7 @@ tracking_allocator :: proc(data: ^Tracking_Allocator) -> Allocator { tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, size, alignment: int, - old_memory: rawptr, old_size: int, loc := #caller_location) -> ([]byte, Allocator_Error) { + old_memory: rawptr, old_size: int, loc := #caller_location) -> (result: []byte, err: Allocator_Error) { data := (^Tracking_Allocator)(allocator_data) if mode == .Query_Info { info := (^Allocator_Query_Info)(old_memory) @@ -892,21 +892,16 @@ tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, info.pointer = nil } - return nil, nil + return } - result: []byte - err: Allocator_Error if mode == .Free && old_memory != nil && old_memory not_in data.allocation_map { append(&data.bad_free_array, Tracking_Allocator_Bad_Free_Entry{ memory = old_memory, location = loc, }) } else { - result, err = data.backing.procedure(data.backing.data, mode, size, alignment, old_memory, old_size, loc) - if err != nil { - return result, err - } + result = data.backing.procedure(data.backing.data, mode, size, alignment, old_memory, old_size, loc) or_return } result_ptr := raw_data(result) @@ -954,6 +949,6 @@ tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, unreachable() } - return result, err + return } diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 050d91d3c..b62f66864 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -231,7 +231,7 @@ make_dynamic_array_len_cap :: proc($T: typeid/[dynamic]$E, #any_int len: int, #a return } @(builtin) -make_map :: proc($T: typeid/map[$K]$E, #any_int cap: int = DEFAULT_RESERVE_CAPACITY, allocator := context.allocator, loc := #caller_location) -> T { +make_map :: proc($T: typeid/map[$K]$E, #any_int cap: int = 1< T { make_map_expr_error_loc(loc, cap) context.allocator = allocator @@ -283,19 +283,13 @@ reserve_map :: proc(m: ^$T/map[$K]$V, capacity: int, loc := #caller_location) { } /* - Shrinks the capacity of a map down to the current length, or the given capacity. - - If `new_cap` is negative, then `len(m)` is used. - - Returns false if `cap(m) < new_cap`, or the allocator report failure. - - If `len(m) < new_cap`, then `len(m)` will be left unchanged. + Shrinks the capacity of a map down to the current length. */ @builtin -shrink_map :: proc(m: ^$T/map[$K]$V, new_cap := -1, loc := #caller_location) -> (did_shrink: bool) { +shrink_map :: proc(m: ^$T/map[$K]$V, loc := #caller_location) -> (did_shrink: bool) { if m != nil { - new_cap := new_cap if new_cap >= 0 else len(m) - return __dynamic_map_shrink(__get_map_header(m), new_cap, loc) + err := map_shrink_dynamic((^Raw_Map)(m), map_info(T), loc) + did_shrink = err == nil } return } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index e56294650..351acb88a 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -316,9 +316,9 @@ map_total_allocation_size :: #force_inline proc "contextless" (capacity: uintptr // The only procedure which needs access to the context is the one which allocates the map. map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator, loc := #caller_location) -> (result: Raw_Map, err: Allocator_Error) { + result.allocator = allocator // set the allocator always if log2_capacity == 0 { - // Empty map, but set the allocator. - return { 0, 0, allocator }, nil + return } if log2_capacity >= 64 { @@ -337,12 +337,8 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc if intrinsics.expect(data_ptr & CACHE_MASK != 0, false) { panic("allocation not aligned to a cache line", loc) } else { - result = { - // Tagged pointer representation for capacity. - data_ptr | log2_capacity, - 0, - allocator, - } + result.data = data_ptr | log2_capacity // Tagged pointer representation for capacity. + result.len = 0 map_clear_dynamic(&result, info) } @@ -513,20 +509,19 @@ map_add_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h: M @(optimization_mode="size") map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { - allocator := m.allocator - if allocator.procedure == nil { - allocator = context.allocator + if m.allocator.procedure == nil { + m.allocator = context.allocator } log2_capacity := map_log2_cap(m^) if m.data == 0 { - n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator, loc) or_return + n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, m.allocator, loc) or_return m.data = n.data return nil } - resized := map_alloc_dynamic(info, log2_capacity + 1, allocator, loc) or_return + resized := map_alloc_dynamic(info, log2_capacity + 1, m.allocator, loc) or_return old_capacity := uintptr(1) << log2_capacity @@ -554,7 +549,7 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf } map_free_dynamic(m^, info, loc) or_return - m.data = resized.data // Should copy the capacity too + m^ = resized return nil } @@ -562,9 +557,8 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf @(optimization_mode="size") map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr, loc := #caller_location) -> Allocator_Error { - allocator := m.allocator - if allocator.procedure == nil { - allocator = context.allocator + if m.allocator.procedure == nil { + m.allocator = context.allocator } new_capacity := new_capacity @@ -580,11 +574,11 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ log2_new_capacity := size_of(uintptr) - intrinsics.count_leading_zeros(new_capacity-1) if m.data == 0 { - m^ = map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, allocator, loc) or_return + m^ = map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, m.allocator, loc) or_return return nil } - resized := map_alloc_dynamic(info, log2_new_capacity, allocator, loc) or_return + resized := map_alloc_dynamic(info, log2_new_capacity, m.allocator, loc) or_return ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) @@ -609,7 +603,7 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ } } - mem_free(rawptr(ks), allocator) + map_free_dynamic(m^, info, loc) or_return m^ = resized // Should copy the capacity too @@ -618,11 +612,9 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ @(optimization_mode="size") -map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) -> Allocator_Error { - allocator := m.allocator - if allocator.procedure == nil { - // TODO(bill): is this correct behaviour? - allocator = context.allocator +map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { + if m.allocator.procedure == nil { + m.allocator = context.allocator } // Cannot shrink the capacity if the number of items in the map would exceed @@ -633,7 +625,7 @@ map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I return nil } - shrinked := map_alloc_dynamic(info, log2_capacity - 1, allocator) or_return + shrinked := map_alloc_dynamic(info, log2_capacity - 1, m.allocator) or_return capacity := uintptr(1) << log2_capacity @@ -662,9 +654,9 @@ map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I } } - mem_free(rawptr(ks), allocator) + map_free_dynamic(m^, info, loc) or_return - m.data = shrinked.data // Should copy the capacity too + m^ = shrinked return nil } @@ -672,9 +664,8 @@ map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I @(require_results) map_free_dynamic :: proc "odin" (m: Raw_Map, info: ^Map_Info, loc := #caller_location) -> Allocator_Error { ptr := rawptr(map_data(m)) - size := map_total_allocation_size(uintptr(map_cap(m)), info) - - return mem_free_with_size(ptr, int(size), m.allocator, loc) + size := int(map_total_allocation_size(uintptr(map_cap(m)), info)) + return mem_free_with_size(ptr, size, m.allocator, loc) } @(optimization_mode="speed") From 667af1be586bc64229c2a3f61ab9d59636f0c5fc Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 20:44:52 +0000 Subject: [PATCH 23/62] Correct `map_insert_hash_dynamic` and `map_insert_dynamic` --- core/runtime/dynamic_map_internal.odin | 93 ++------------------------ 1 file changed, 5 insertions(+), 88 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 351acb88a..e56dd5bb2 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -393,7 +393,7 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h return result if result != 0 else v_dst } - if pd := map_probe_distance(m, element_hash, p); pd < d { + if pd := map_probe_distance(m, element_hash, p); pd <= d { if map_hash_is_deleted(element_hash) { k_dst := map_cell_index_dynamic(ks, info.ks, p) v_dst := map_cell_index_dynamic(vs, info.vs, p) @@ -401,6 +401,10 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h return result if result != 0 else v_dst + } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, p))) { + v_dst := map_cell_index_dynamic(vs, info.vs, p) + intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(iv), info.vs.size_of_type) + return v_dst } if result == 0 { @@ -431,82 +435,6 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h } } -@(optimization_mode="speed") -map_add_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) { - capacity := uintptr(1) << map_log2_cap(m) - p := map_desired_position(m, h) - d := uintptr(0) - c := capacity - 1 // Saturating arithmetic mask - - ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) - - // Avoid redundant loads of these values - size_of_k := info.ks.size_of_type - size_of_v := info.vs.size_of_type - - // Use sk and sv scratch storage space for dynamic k and v storage here. - // - // Simulate the following at runtime - // k = ik - // v = iv - // h = h - k := map_cell_index_dynamic_const(sk, info.ks, 0) - v := map_cell_index_dynamic_const(sv, info.vs, 0) - intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(ik), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(iv), size_of_v) - h := h - - // Temporary k and v dynamic storage for swap below - tk := map_cell_index_dynamic_const(sk, info.ks, 1) - tv := map_cell_index_dynamic_const(sv, info.vs, 1) - - for { - hp := &hs[p] - element_hash := hp^ - - if map_hash_is_empty(element_hash) { - k_dst := map_cell_index_dynamic(ks, info.ks, p) - v_dst := map_cell_index_dynamic(vs, info.vs, p) - intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) - hp^ = h - return - } - - if pd := map_probe_distance(m, element_hash, p); pd < d { - if map_hash_is_deleted(element_hash) { - k_dst := map_cell_index_dynamic(ks, info.ks, p) - v_dst := map_cell_index_dynamic(vs, info.vs, p) - intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) - hp^ = h - return - } - - kp := map_cell_index_dynamic(ks, info.vs, p) - vp := map_cell_index_dynamic(vs, info.ks, p) - - // Simulate the following at runtime with dynamic storage - // - // kp^, k = k, kp^ - // vp^, v = v, vp^ - // hp^, h = h, hp^ - intrinsics.mem_copy_non_overlapping(rawptr(tk), rawptr(kp), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(tv), rawptr(vp), size_of_v) - intrinsics.mem_copy_non_overlapping(rawptr(kp), rawptr(k), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(vp), rawptr(v), size_of_v) - intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(tk), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(tv), size_of_v) - hp^, h = h, hp^ - - d = pd - } - - p = (p + 1) & c - d += 1 - } -} - @(optimization_mode="size") map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { if m.allocator.procedure == nil { @@ -729,17 +657,6 @@ map_insert_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I return } -// Same as map_insert_dynamic but does not return address to the inserted element. -@(optimization_mode="speed") -map_add_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr, loc := #caller_location) -> Allocator_Error { - if map_len(m^) + 1 >= map_resize_threshold(m^) { - map_grow_dynamic(m, info, loc) or_return - } - map_add_hash_dynamic(m^, info, info.key_hasher(rawptr(k), 0), k, v) - m.len += 1 - return nil -} - map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> bool { MASK :: 1 << (size_of(Map_Hash)*8 - 1) From 503eb470a74028052e12630fdb72aa6fa380eff4 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 21:10:38 +0000 Subject: [PATCH 24/62] Do an extra check before insertion for pre-existing keys This is test code --- core/runtime/dynamic_map_internal.odin | 49 +++++++++++++++++--------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index e56dd5bb2..df3d09375 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -354,12 +354,27 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc // This procedure returns the address of the just inserted value. @(optimization_mode="speed") map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { - p := map_desired_position(m, h) - d := uintptr(0) - c := (uintptr(1) << map_log2_cap(m)) - 1 // Saturating arithmetic mask + pos := map_desired_position(m, h) + distance := uintptr(0) + mask := (uintptr(1) << map_log2_cap(m)) - 1 // Saturating arithmetic mask ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) + get_loop: for { + element_hash := hs[pos] + if map_hash_is_empty(element_hash) { + break get_loop + } else if distance > map_probe_distance(m, element_hash, pos) { + break get_loop + } else if element_hash == h && info.key_equal(rawptr(ik), rawptr(map_cell_index_dynamic(ks, info.ks, pos))) { + result = map_cell_index_dynamic(vs, info.vs, pos) + intrinsics.mem_copy_non_overlapping(rawptr(result), rawptr(iv), info.ks.size_of_type) + return + } + pos = (pos + 1) & mask + distance += 1 + } + // Avoid redundant loads of these values size_of_k := info.ks.size_of_type size_of_v := info.vs.size_of_type @@ -381,38 +396,38 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h tv := map_cell_index_dynamic_const(sv, info.vs, 1) for { - hp := &hs[p] + hp := &hs[pos] element_hash := hp^ if map_hash_is_empty(element_hash) { - k_dst := map_cell_index_dynamic(ks, info.ks, p) - v_dst := map_cell_index_dynamic(vs, info.vs, p) + k_dst := map_cell_index_dynamic(ks, info.ks, pos) + v_dst := map_cell_index_dynamic(vs, info.vs, pos) intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h return result if result != 0 else v_dst } - if pd := map_probe_distance(m, element_hash, p); pd <= d { + if pd := map_probe_distance(m, element_hash, pos); pd <= distance { if map_hash_is_deleted(element_hash) { - k_dst := map_cell_index_dynamic(ks, info.ks, p) - v_dst := map_cell_index_dynamic(vs, info.vs, p) + k_dst := map_cell_index_dynamic(ks, info.ks, pos) + v_dst := map_cell_index_dynamic(vs, info.vs, pos) intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h return result if result != 0 else v_dst - } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, p))) { - v_dst := map_cell_index_dynamic(vs, info.vs, p) + } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, pos))) { + v_dst := map_cell_index_dynamic(vs, info.vs, pos) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(iv), info.vs.size_of_type) return v_dst } if result == 0 { - result = map_cell_index_dynamic(vs, info.vs, p) + result = map_cell_index_dynamic(vs, info.vs, pos) } - kp := map_cell_index_dynamic(ks, info.vs, p) - vp := map_cell_index_dynamic(vs, info.ks, p) + kp := map_cell_index_dynamic(ks, info.vs, pos) + vp := map_cell_index_dynamic(vs, info.ks, pos) // Simulate the following at runtime with dynamic storage // @@ -427,11 +442,11 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(tv), size_of_v) hp^, h = h, hp^ - d = pd + distance = pd } - p = (p + 1) & c - d += 1 + pos = (pos + 1) & mask + distance += 1 } } From bcf437dc11507611a3850a2d5964c6e893403967 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 21:21:19 +0000 Subject: [PATCH 25/62] Check for existence before setting Test code --- core/runtime/dynamic_map_internal.odin | 29 +++++++------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index df3d09375..7d7b5283c 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -360,21 +360,6 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) - get_loop: for { - element_hash := hs[pos] - if map_hash_is_empty(element_hash) { - break get_loop - } else if distance > map_probe_distance(m, element_hash, pos) { - break get_loop - } else if element_hash == h && info.key_equal(rawptr(ik), rawptr(map_cell_index_dynamic(ks, info.ks, pos))) { - result = map_cell_index_dynamic(vs, info.vs, pos) - intrinsics.mem_copy_non_overlapping(rawptr(result), rawptr(iv), info.ks.size_of_type) - return - } - pos = (pos + 1) & mask - distance += 1 - } - // Avoid redundant loads of these values size_of_k := info.ks.size_of_type size_of_v := info.vs.size_of_type @@ -416,10 +401,6 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h return result if result != 0 else v_dst - } else if element_hash == h && info.key_equal(rawptr(k), rawptr(map_cell_index_dynamic(ks, info.ks, pos))) { - v_dst := map_cell_index_dynamic(vs, info.vs, pos) - intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(iv), info.vs.size_of_type) - return v_dst } if result == 0 { @@ -662,7 +643,13 @@ map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, @(optimization_mode="speed") -map_insert_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr, loc := #caller_location) -> (value: uintptr, err: Allocator_Error) { +map_set_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k, v: uintptr, loc := #caller_location) -> (value: uintptr, err: Allocator_Error) { + if found := __dynamic_map_get(m^, info, rawptr(k)); found != nil { + intrinsics.mem_copy_non_overlapping(found, rawptr(v), info.vs.size_of_type) + value = uintptr(found) + return + } + if map_len(m^) + 1 >= map_resize_threshold(m^) { map_grow_dynamic(m, info, loc) or_return } @@ -762,7 +749,7 @@ __dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, } __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { - value, err := map_insert_dynamic(m, info, uintptr(key), uintptr(value), loc) + value, err := map_set_dynamic(m, info, uintptr(key), uintptr(value), loc) return rawptr(value) if err == nil else nil } From d4f343751e2a34550d2aca20c23f5c1016d07aa3 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 8 Nov 2022 21:57:18 +0000 Subject: [PATCH 26/62] Inline `__dynamic_map_set` code where possible --- core/runtime/dynamic_map_internal.odin | 40 +++++++++++--------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 7d7b5283c..632fe9d4a 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -452,7 +452,7 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) // Cache these loads to avoid hitting them in the for loop. - n := map_len(m^) + n := m.len for i in 0.. (value: uintptr, err: Allocator_Error) { - if found := __dynamic_map_get(m^, info, rawptr(k)); found != nil { - intrinsics.mem_copy_non_overlapping(found, rawptr(v), info.vs.size_of_type) - value = uintptr(found) - return - } - - if map_len(m^) + 1 >= map_resize_threshold(m^) { - map_grow_dynamic(m, info, loc) or_return - } - hashed := info.key_hasher(rawptr(k), 0) - value = map_insert_hash_dynamic(m^, info, hashed, k, v) - m.len += 1 - return -} - map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> bool { MASK :: 1 << (size_of(Map_Hash)*8 - 1) @@ -749,8 +731,20 @@ __dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, } __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { - value, err := map_set_dynamic(m, info, uintptr(key), uintptr(value), loc) - return rawptr(value) if err == nil else nil + if found := __dynamic_map_get(m^, info, key); found != nil { + intrinsics.mem_copy_non_overlapping(found, value, info.vs.size_of_type) + return found + } + + if m.len + 1 >= map_resize_threshold(m^) { + if map_grow_dynamic(m, info, loc) != nil { + return nil + } + } + hashed := info.key_hasher(key, 0) + result := map_insert_hash_dynamic(m^, info, hashed, uintptr(key), uintptr(value)) + m.len += 1 + return rawptr(result) } @(private) From 3858422f1d9c607736b2b911a3a27678d77afed5 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 9 Nov 2022 20:59:49 +0000 Subject: [PATCH 27/62] Use `mem_resize` where possible --- core/runtime/core_builtin.odin | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index b62f66864..c04d79455 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -541,10 +541,7 @@ reserve_dynamic_array :: proc(array: ^$T/[dynamic]$E, capacity: int, loc := #cal new_size := capacity * size_of(E) allocator := a.allocator - new_data, err := allocator.procedure( - allocator.data, .Resize, new_size, align_of(E), - a.data, old_size, loc, - ) + new_data, err := mem_resize(a.data, old_size, new_size, align_of(E), allocator, loc) if new_data == nil || err != nil { return false } @@ -575,10 +572,7 @@ resize_dynamic_array :: proc(array: ^$T/[dynamic]$E, length: int, loc := #caller new_size := length * size_of(E) allocator := a.allocator - new_data, err := allocator.procedure( - allocator.data, .Resize, new_size, align_of(E), - a.data, old_size, loc, - ) + new_data, err := mem_resize(a.data, old_size, new_size, align_of(E), allocator, loc) if new_data == nil || err != nil { return false } @@ -618,15 +612,7 @@ shrink_dynamic_array :: proc(array: ^$T/[dynamic]$E, new_cap := -1, loc := #call old_size := a.cap * size_of(E) new_size := new_cap * size_of(E) - new_data, err := a.allocator.procedure( - a.allocator.data, - .Resize, - new_size, - align_of(E), - a.data, - old_size, - loc, - ) + new_data, err := mem_resize(a.data, old_size, new_size, align_of(E), allocator, loc) if err != nil { return } From 0424fb486b8fee2c0853724cd3336239509b35f7 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 9 Nov 2022 21:00:17 +0000 Subject: [PATCH 28/62] Rewrite `map_insert_hash_dynamic` --- core/runtime/dynamic_map_internal.odin | 62 +++++++++++--------------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 632fe9d4a..bf0154a77 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -345,40 +345,35 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc return } -// When the type information is known we should use map_insert_hash_static for -// better performance. This procedure has to stack allocate storage to store -// local keys during the Robin Hood hashing technique where elements are swapped -// in the backing arrays to reduce variance. This swapping can only be done with -// memcpy since there is no type information. +// This procedure has to stack allocate storage to store local keys during the +// Robin Hood hashing technique where elements are swapped in the backing +// arrays to reduce variance. This swapping can only be done with memcpy since +// there is no type information. // // This procedure returns the address of the just inserted value. @(optimization_mode="speed") map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { - pos := map_desired_position(m, h) + pos := map_desired_position(m, h) distance := uintptr(0) - mask := (uintptr(1) << map_log2_cap(m)) - 1 // Saturating arithmetic mask + mask := (uintptr(1) << map_log2_cap(m)) - 1 ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) + _, _ = sk, sv // Avoid redundant loads of these values size_of_k := info.ks.size_of_type size_of_v := info.vs.size_of_type - // Use sk and sv scratch storage space for dynamic k and v storage here. - // - // Simulate the following at runtime - // k = ik - // v = iv - // h = h - k := map_cell_index_dynamic_const(sk, info.ks, 0) - v := map_cell_index_dynamic_const(sv, info.vs, 0) + k := map_cell_index_dynamic(sk, info.ks, 0) + v := map_cell_index_dynamic(sv, info.vs, 0) intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(ik), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(iv), size_of_v) - h := h // Temporary k and v dynamic storage for swap below - tk := map_cell_index_dynamic_const(sk, info.ks, 1) - tv := map_cell_index_dynamic_const(sv, info.vs, 1) + tk := map_cell_index_dynamic(sk, info.ks, 1) + tv := map_cell_index_dynamic(sv, info.vs, 1) + + h := h for { hp := &hs[pos] @@ -393,7 +388,7 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h return result if result != 0 else v_dst } - if pd := map_probe_distance(m, element_hash, pos); pd <= distance { + if probe_distance := map_probe_distance(m, element_hash, pos); probe_distance < distance { if map_hash_is_deleted(element_hash) { k_dst := map_cell_index_dynamic(ks, info.ks, pos) v_dst := map_cell_index_dynamic(vs, info.vs, pos) @@ -407,23 +402,18 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h result = map_cell_index_dynamic(vs, info.vs, pos) } - kp := map_cell_index_dynamic(ks, info.vs, pos) - vp := map_cell_index_dynamic(vs, info.ks, pos) + kp := map_cell_index_dynamic(ks, info.ks, pos) + vp := map_cell_index_dynamic(vs, info.vs, pos) - // Simulate the following at runtime with dynamic storage - // - // kp^, k = k, kp^ - // vp^, v = v, vp^ - // hp^, h = h, hp^ - intrinsics.mem_copy_non_overlapping(rawptr(tk), rawptr(kp), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(tv), rawptr(vp), size_of_v) - intrinsics.mem_copy_non_overlapping(rawptr(kp), rawptr(k), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(vp), rawptr(v), size_of_v) - intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(tk), size_of_k) - intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(tv), size_of_v) - hp^, h = h, hp^ + intrinsics.mem_copy_non_overlapping(rawptr(tk), rawptr(k), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(k), rawptr(kp), size_of_k) + intrinsics.mem_copy_non_overlapping(rawptr(kp), rawptr(tk), size_of_k) - distance = pd + intrinsics.mem_copy_non_overlapping(rawptr(tv), rawptr(v), size_of_v) + intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(vp), size_of_v) + intrinsics.mem_copy_non_overlapping(rawptr(vp), rawptr(tv), size_of_v) + + distance = probe_distance } pos = (pos + 1) & mask @@ -741,8 +731,8 @@ __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_In return nil } } - hashed := info.key_hasher(key, 0) - result := map_insert_hash_dynamic(m^, info, hashed, uintptr(key), uintptr(value)) + hash := info.key_hasher(key, 0) + result := map_insert_hash_dynamic(m^, info, hash, uintptr(key), uintptr(value)) m.len += 1 return rawptr(result) } From b035ee2bcd57bc8616858845f466deb062c999cb Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 9 Nov 2022 22:05:28 +0000 Subject: [PATCH 29/62] Swap hashes --- core/runtime/dynamic_map_internal.odin | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index bf0154a77..105e68c8c 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -351,7 +351,6 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc // there is no type information. // // This procedure returns the address of the just inserted value. -@(optimization_mode="speed") map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { pos := map_desired_position(m, h) distance := uintptr(0) @@ -413,6 +412,10 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h intrinsics.mem_copy_non_overlapping(rawptr(v), rawptr(vp), size_of_v) intrinsics.mem_copy_non_overlapping(rawptr(vp), rawptr(tv), size_of_v) + th := h + h = hp^ + hp^ = h + distance = probe_distance } @@ -421,7 +424,6 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h } } -@(optimization_mode="size") map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { if m.allocator.procedure == nil { m.allocator = context.allocator @@ -469,7 +471,6 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf } -@(optimization_mode="size") map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr, loc := #caller_location) -> Allocator_Error { if m.allocator.procedure == nil { m.allocator = context.allocator @@ -525,7 +526,6 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ } -@(optimization_mode="size") map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { if m.allocator.procedure == nil { m.allocator = context.allocator @@ -582,7 +582,6 @@ map_free_dynamic :: proc "odin" (m: Raw_Map, info: ^Map_Info, loc := #caller_loc return mem_free_with_size(ptr, size, m.allocator, loc) } -@(optimization_mode="speed") map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (index: uintptr, ok: bool) { if map_len(m) == 0 { return 0, false @@ -605,7 +604,6 @@ map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, d += 1 } } -@(optimization_mode="speed") map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (ok: bool) { if map_len(m) == 0 { return false From 1bcec3f7696243664d8bfefa24b4262d4f412755 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 9 Nov 2022 22:21:36 +0000 Subject: [PATCH 30/62] Change map internal calls to use a pointer --- src/llvm_backend.cpp | 8 ++++---- src/llvm_backend_expr.cpp | 4 ++-- src/llvm_backend_general.cpp | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 2b95c5b2f..e12a4c016 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -757,8 +757,8 @@ lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue return hashed_key; } -lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map, lbValue const &key) { - Type *map_type = base_type(map.type); +lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key) { + Type *map_type = base_type(type_deref(map_ptr.type)); GB_ASSERT(map_type->kind == Type_Map); lbValue ptr = {}; @@ -770,13 +770,13 @@ lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map, lbVa lbValue map_get_proc = lb_get_map_get_proc_for_type(p->module, map_type); auto args = array_make(permanent_allocator(), 2); - args[0] = lb_address_from_load_or_generate_local(p, map); + args[0] = map_ptr; args[1] = key_ptr; ptr = lb_emit_call(p, map_get_proc, args); } else { auto args = array_make(permanent_allocator(), 3); - args[0] = lb_emit_transmute(p, map, t_raw_map); + args[0] = lb_emit_transmute(p, lb_emit_load(p, map_ptr), t_raw_map); args[1] = lb_gen_map_info_ptr(p->module, map_type); args[2] = key_ptr; diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 7e9aa3a78..05a9fdfbf 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -1423,9 +1423,9 @@ lbValue lb_build_binary_expr(lbProcedure *p, Ast *expr) { switch (rt->kind) { case Type_Map: { - lbValue map = right; + lbValue map_ptr = lb_address_from_load_or_generate_local(p, right); lbValue key = left; - lbValue ptr = lb_internal_dynamic_map_get_ptr(p, map, key); + lbValue ptr = lb_internal_dynamic_map_get_ptr(p, map_ptr, key); if (be->op.kind == Token_in) { return lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_NotEq, ptr), t_bool); } else { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 859542fb5..a0e4d80ba 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -418,7 +418,7 @@ lbValue lb_addr_get_ptr(lbProcedure *p, lbAddr const &addr) { switch (addr.kind) { case lbAddr_Map: - return lb_internal_dynamic_map_get_ptr(p, lb_emit_load(p, addr.addr), addr.map.key); + return lb_internal_dynamic_map_get_ptr(p, addr.addr, addr.map.key); case lbAddr_RelativePointer: { Type *rel_ptr = base_type(lb_addr_type(addr)); @@ -1075,7 +1075,7 @@ lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { GB_ASSERT(map_type->kind == Type_Map); lbAddr v = lb_add_local_generated(p, map_type->Map.lookup_result_type, true); - lbValue ptr = lb_internal_dynamic_map_get_ptr(p, lb_emit_load(p, addr.addr), addr.map.key); + lbValue ptr = lb_internal_dynamic_map_get_ptr(p, addr.addr, addr.map.key); lbValue ok = lb_emit_conv(p, lb_emit_comp_against_nil(p, Token_NotEq, ptr), t_bool); lb_emit_store(p, lb_emit_struct_ep(p, v.addr, 1), ok); From f2f2d532f5bb16164cc1f0128b3f5e087c56b8d9 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 9 Nov 2022 22:31:49 +0000 Subject: [PATCH 31/62] Add extra calls to `Tracking_Allocator` --- core/mem/alloc.odin | 14 +++++++++++--- core/mem/allocators.odin | 11 +++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/core/mem/alloc.odin b/core/mem/alloc.odin index 05f3e145d..4f449bd37 100644 --- a/core/mem/alloc.odin +++ b/core/mem/alloc.odin @@ -77,6 +77,14 @@ free :: proc(ptr: rawptr, allocator := context.allocator, loc := #caller_locatio return runtime.mem_free(ptr, allocator, loc) } +free_with_size :: proc(ptr: rawptr, byte_count: int, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { + if ptr == nil || allocator.procedure == nil { + return nil + } + _, err := allocator.procedure(allocator.data, .Free, 0, 0, ptr, byte_count, loc) + return err +} + free_bytes :: proc(bytes: []byte, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { return runtime.mem_free_bytes(bytes, allocator, loc) } @@ -113,16 +121,16 @@ query_info :: proc(pointer: rawptr, allocator: Allocator, loc := #caller_locatio delete_string :: proc(str: string, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { - return free(raw_data(str), allocator, loc) + return free_with_size(raw_data(str), len(str), allocator, loc) } delete_cstring :: proc(str: cstring, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { return free((^byte)(str), allocator, loc) } delete_dynamic_array :: proc(array: $T/[dynamic]$E, loc := #caller_location) -> Allocator_Error { - return free(raw_data(array), array.allocator, loc) + return free_with_size(raw_data(array), cap(array)*size_of(E), array.allocator, loc) } delete_slice :: proc(array: $T/[]$E, allocator := context.allocator, loc := #caller_location) -> Allocator_Error { - return free(raw_data(array), allocator, loc) + return free_with_size(raw_data(array), len(array)*size_of(E), allocator, loc) } delete_map :: proc(m: $T/map[$K]$V, loc := #caller_location) -> Allocator_Error { return runtime.map_free_dynamic(transmute(Raw_Map)m, runtime.map_info(T), loc) diff --git a/core/mem/allocators.odin b/core/mem/allocators.odin index a7c50da02..364a49fba 100644 --- a/core/mem/allocators.odin +++ b/core/mem/allocators.odin @@ -864,6 +864,10 @@ tracking_allocator_init :: proc(t: ^Tracking_Allocator, backing_allocator: Alloc t.backing = backing_allocator t.allocation_map.allocator = internals_allocator t.bad_free_array.allocator = internals_allocator + + if .Free_All in query_features(t.backing) { + t.clear_on_free_all = true + } } tracking_allocator_destroy :: proc(t: ^Tracking_Allocator) { @@ -871,6 +875,13 @@ tracking_allocator_destroy :: proc(t: ^Tracking_Allocator) { delete(t.bad_free_array) } + +tracking_allocator_clear :: proc(t: ^Tracking_Allocator) { + clear(&t.allocation_map) + clear(&t.bad_free_array) +} + + tracking_allocator :: proc(data: ^Tracking_Allocator) -> Allocator { return Allocator{ data = data, From db748b7a0570d0d98b73033cced4aa8adac56a6b Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 9 Nov 2022 23:10:18 +0000 Subject: [PATCH 32/62] Correct logic for `__dynamic_map_set` --- core/runtime/dynamic_map_internal.odin | 52 ++++++++++++++++---------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 105e68c8c..7f1ea5e68 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -351,12 +351,12 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc // there is no type information. // // This procedure returns the address of the just inserted value. -map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { - pos := map_desired_position(m, h) +map_insert_hash_dynamic :: proc "odin" (m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { + pos := map_desired_position(m^, h) distance := uintptr(0) - mask := (uintptr(1) << map_log2_cap(m)) - 1 + mask := (uintptr(1) << map_log2_cap(m^)) - 1 - ks, vs, hs, sk, sv := map_kvh_data_dynamic(m, info) + ks, vs, hs, sk, sv := map_kvh_data_dynamic(m^, info) _, _ = sk, sv // Avoid redundant loads of these values @@ -384,16 +384,18 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h + return result if result != 0 else v_dst } - if probe_distance := map_probe_distance(m, element_hash, pos); probe_distance < distance { + if probe_distance := map_probe_distance(m^, element_hash, pos); distance > probe_distance { if map_hash_is_deleted(element_hash) { k_dst := map_cell_index_dynamic(ks, info.ks, pos) v_dst := map_cell_index_dynamic(vs, info.vs, pos) intrinsics.mem_copy_non_overlapping(rawptr(k_dst), rawptr(k), size_of_k) intrinsics.mem_copy_non_overlapping(rawptr(v_dst), rawptr(v), size_of_v) hp^ = h + return result if result != 0 else v_dst } @@ -414,7 +416,7 @@ map_insert_hash_dynamic :: proc "odin" (m: Raw_Map, #no_alias info: ^Map_Info, h th := h h = hp^ - hp^ = h + hp^ = th distance = probe_distance } @@ -455,7 +457,7 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf } k := map_cell_index_dynamic(ks, info.ks, i) v := map_cell_index_dynamic(vs, info.vs, i) - map_insert_hash_dynamic(resized, info, hash, k, v) + map_insert_hash_dynamic(&resized, info, hash, k, v) // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. n -= 1 @@ -465,7 +467,7 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf } map_free_dynamic(m^, info, loc) or_return - m^ = resized + m.data = resized.data return nil } @@ -509,7 +511,7 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ } k := map_cell_index_dynamic(ks, info.ks, i) v := map_cell_index_dynamic(vs, info.vs, i) - map_insert_hash_dynamic(resized, info, hash, k, v) + map_insert_hash_dynamic(&resized, info, hash, k, v) // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. n -= 1 @@ -520,7 +522,7 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ map_free_dynamic(m^, info, loc) or_return - m^ = resized // Should copy the capacity too + m.data = resized.data return nil } @@ -539,7 +541,7 @@ map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I return nil } - shrinked := map_alloc_dynamic(info, log2_capacity - 1, m.allocator) or_return + shrunk := map_alloc_dynamic(info, log2_capacity - 1, m.allocator) or_return capacity := uintptr(1) << log2_capacity @@ -558,7 +560,7 @@ map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I k := map_cell_index_dynamic(ks, info.ks, i) v := map_cell_index_dynamic(vs, info.vs, i) - map_insert_hash_dynamic(shrinked, info, hash, k, v) + map_insert_hash_dynamic(&shrunk, info, hash, k, v) // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. @@ -570,7 +572,7 @@ map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I map_free_dynamic(m^, info, loc) or_return - m^ = shrinked + m.data = shrunk.data return nil } @@ -694,12 +696,10 @@ map_get :: proc "contextless" (m: $T/map[$K]$V, key: K) -> (stored_key: K, store } } -__dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { +__dynamic_map_get_with_hash :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, key: rawptr) -> (ptr: rawptr) { if m.len == 0 { return nil } - - h := info.key_hasher(key, 0) pos := map_desired_position(m, h) distance := uintptr(0) mask := (uintptr(1) << map_log2_cap(m)) - 1 @@ -718,19 +718,31 @@ __dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, } } +__dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { + if m.len == 0 { + return nil + } + h := info.key_hasher(key, 0) + return __dynamic_map_get_with_hash(m, info, h, key) +} + __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { - if found := __dynamic_map_get(m^, info, key); found != nil { + hash := info.key_hasher(key, 0) + + if found := __dynamic_map_get_with_hash(m^, info, hash, key); found != nil { intrinsics.mem_copy_non_overlapping(found, value, info.vs.size_of_type) return found } if m.len + 1 >= map_resize_threshold(m^) { - if map_grow_dynamic(m, info, loc) != nil { + err := map_grow_dynamic(m, info, loc) + if err != nil { return nil } } - hash := info.key_hasher(key, 0) - result := map_insert_hash_dynamic(m^, info, hash, uintptr(key), uintptr(value)) + + result := map_insert_hash_dynamic(m, info, hash, uintptr(key), uintptr(value)) + assert(result != 0) m.len += 1 return rawptr(result) } From 5c106abe3fd1850c3880fcd6753e3e2273c1eded Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 10 Nov 2022 12:01:40 +0000 Subject: [PATCH 33/62] Make `map_alloc_dynamic` handle the `nil_allocator()` --- core/runtime/dynamic_map_internal.odin | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 7f1ea5e68..4a246fd18 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -334,6 +334,10 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc data := mem_alloc_non_zeroed(int(size), MAP_CACHE_LINE_SIZE, allocator, loc) or_return data_ptr := uintptr(raw_data(data)) + if data_ptr == 0 { + err = .Out_Of_Memory + return + } if intrinsics.expect(data_ptr & CACHE_MASK != 0, false) { panic("allocation not aligned to a cache line", loc) } else { From f6fc3ebe37bb1d2ab23bfbddc4ca04ffdb7781bb Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 10 Nov 2022 12:27:12 +0000 Subject: [PATCH 34/62] Add reflect/iterator.odin --- core/reflect/iterator.odin | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 core/reflect/iterator.odin diff --git a/core/reflect/iterator.odin b/core/reflect/iterator.odin new file mode 100644 index 000000000..62baff06e --- /dev/null +++ b/core/reflect/iterator.odin @@ -0,0 +1,76 @@ +package reflect + +import "core:runtime" + +iterate_array :: proc(val: any, it: ^int) -> (elem: any, index: int, ok: bool) { + if val == nil || it == nil { + return + } + + ti := type_info_base(type_info_of(val.id)) + #partial switch info in ti.variant { + case Type_Info_Pointer: + if ptr := (^rawptr)(val.data)^; ptr != nil { + return iterate_array(any{ptr, info.elem.id}, it) + } + case Type_Info_Array: + if it^ < info.count { + elem.data = rawptr(uintptr(val.data) + uintptr(it^ * info.elem_size)) + elem.id = info.elem.id + ok = true + it^ += 1 + } + case Type_Info_Slice: + array := (^runtime.Raw_Slice)(val.data) + if it^ < array.len { + elem.data = rawptr(uintptr(array.data) + uintptr(it^ * info.elem_size)) + elem.id = info.elem.id + ok = true + it^ += 1 + } + case Type_Info_Dynamic_Array: + array := (^runtime.Raw_Dynamic_Array)(val.data) + if it^ < array.len { + elem.data = rawptr(uintptr(array.data) + uintptr(it^ * info.elem_size)) + elem.id = info.elem.id + ok = true + it^ += 1 + } + } + + return +} + +iterate_map :: proc(val: any, it: ^int) -> (key, value: any, ok: bool) { + if val == nil || it == nil { + return + } + ti := type_info_base(type_info_of(val.id)) + #partial switch info in ti.variant { + case Type_Info_Pointer: + if ptr := (^rawptr)(val.data)^; ptr != nil { + return iterate_map(any{ptr, info.elem.id}, it) + } + case Type_Info_Map: + if info.map_info == nil { + break + } + rm := (^runtime.Raw_Map)(val.data) + ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(rm^, info.map_info) + for /**/ ; it^ < int(runtime.map_cap(rm^)); it^ += 1 { + if hash := hs[it^]; runtime.map_hash_is_valid(hash) { + key_ptr := runtime.map_cell_index_dynamic(ks, info.map_info.ks, uintptr(it^)) + value_ptr := runtime.map_cell_index_dynamic(vs, info.map_info.vs, uintptr(it^)) + + key.data = rawptr(key_ptr) + value.data = rawptr(value_ptr) + key.id = info.key.id + value.id = info.value.id + ok = true + return + } + + } + } + return +} \ No newline at end of file From 7b4a87d37c69759eeb91759a22daca8f23d78309 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 10 Nov 2022 12:33:49 +0000 Subject: [PATCH 35/62] Correct `iterate_map` --- core/reflect/iterator.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/reflect/iterator.odin b/core/reflect/iterator.odin index 62baff06e..9fb6489ca 100644 --- a/core/reflect/iterator.odin +++ b/core/reflect/iterator.odin @@ -67,7 +67,7 @@ iterate_map :: proc(val: any, it: ^int) -> (key, value: any, ok: bool) { key.id = info.key.id value.id = info.value.id ok = true - return + break } } From ac259ac790c7aeccf39c06caacd2e257aca307c4 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 10 Nov 2022 12:34:01 +0000 Subject: [PATCH 36/62] Unify reserve and grow code --- core/runtime/dynamic_map_internal.odin | 47 ++------------------------ 1 file changed, 3 insertions(+), 44 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 4a246fd18..f57dce885 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -431,49 +431,9 @@ map_insert_hash_dynamic :: proc "odin" (m: ^Raw_Map, #no_alias info: ^Map_Info, } map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { - if m.allocator.procedure == nil { - m.allocator = context.allocator - } - log2_capacity := map_log2_cap(m^) - - if m.data == 0 { - n := map_alloc_dynamic(info, MAP_MIN_LOG2_CAPACITY, m.allocator, loc) or_return - m.data = n.data - return nil - } - - resized := map_alloc_dynamic(info, log2_capacity + 1, m.allocator, loc) or_return - - old_capacity := uintptr(1) << log2_capacity - - ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) - - // Cache these loads to avoid hitting them in the for loop. - n := m.len - for i in 0..= map_resize_threshold(m^) { - err := map_grow_dynamic(m, info, loc) - if err != nil { + if err := map_grow_dynamic(m, info, loc); err != nil { return nil } } From 8852d090b61fab27f97787b37b7cbd5b63a52083 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 10 Nov 2022 12:46:53 +0000 Subject: [PATCH 37/62] Correct static map get; make get take a pointer to simplify compiler internals --- core/runtime/dynamic_map_internal.odin | 21 +++++----- src/llvm_backend.cpp | 58 ++++++++++++++------------ src/llvm_backend.hpp | 4 +- src/llvm_backend_expr.cpp | 2 +- src/llvm_backend_proc.cpp | 4 +- src/llvm_backend_type.cpp | 4 +- src/llvm_backend_utility.cpp | 2 +- 7 files changed, 50 insertions(+), 45 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index f57dce885..aedd3f260 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -107,7 +107,7 @@ Map_Cell_Info :: struct { map_cell_info :: intrinsics.type_map_cell_info // Same as the above procedure but at runtime with the cell Map_Cell_Info value. -map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, info: ^Map_Cell_Info, index: uintptr) -> uintptr { +map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, #no_alias info: ^Map_Cell_Info, index: uintptr) -> uintptr { // Micro-optimize the common cases to save on integer division. elements_per_cell := uintptr(info.elements_per_cell) size_of_cell := uintptr(info.size_of_cell) @@ -355,13 +355,13 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc // there is no type information. // // This procedure returns the address of the just inserted value. -map_insert_hash_dynamic :: proc "odin" (m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { +map_insert_hash_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { + h := h pos := map_desired_position(m^, h) distance := uintptr(0) mask := (uintptr(1) << map_log2_cap(m^)) - 1 ks, vs, hs, sk, sv := map_kvh_data_dynamic(m^, info) - _, _ = sk, sv // Avoid redundant loads of these values size_of_k := info.ks.size_of_type @@ -376,7 +376,6 @@ map_insert_hash_dynamic :: proc "odin" (m: ^Raw_Map, #no_alias info: ^Map_Info, tk := map_cell_index_dynamic(sk, info.ks, 1) tv := map_cell_index_dynamic(sv, info.vs, 1) - h := h for { hp := &hs[pos] @@ -660,19 +659,19 @@ map_get :: proc "contextless" (m: $T/map[$K]$V, key: K) -> (stored_key: K, store } } -__dynamic_map_get_with_hash :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, key: rawptr) -> (ptr: rawptr) { +__dynamic_map_get_with_hash :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, key: rawptr) -> (ptr: rawptr) { if m.len == 0 { return nil } - pos := map_desired_position(m, h) + pos := map_desired_position(m^, h) distance := uintptr(0) - mask := (uintptr(1) << map_log2_cap(m)) - 1 - ks, vs, hs, _, _ := map_kvh_data_dynamic(m, info) + mask := (uintptr(1) << map_log2_cap(m^)) - 1 + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) for { element_hash := hs[pos] if map_hash_is_empty(element_hash) { return nil - } else if distance > map_probe_distance(m, element_hash, pos) { + } else if distance > map_probe_distance(m^, element_hash, pos) { return nil } else if element_hash == h && info.key_equal(key, rawptr(map_cell_index_dynamic(ks, info.ks, pos))) { return rawptr(map_cell_index_dynamic(vs, info.vs, pos)) @@ -682,7 +681,7 @@ __dynamic_map_get_with_hash :: proc "contextless" (m: Raw_Map, #no_alias info: ^ } } -__dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { +__dynamic_map_get :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { if m.len == 0 { return nil } @@ -693,7 +692,7 @@ __dynamic_map_get :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { hash := info.key_hasher(key, 0) - if found := __dynamic_map_get_with_hash(m^, info, hash, key); found != nil { + if found := __dynamic_map_get_with_hash(m, info, hash, key); found != nil { intrinsics.mem_copy_non_overlapping(found, value, info.vs.size_of_type) return found } diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index e12a4c016..c313bc825 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -140,7 +140,7 @@ lbContextData *lb_push_context_onto_stack(lbProcedure *p, lbAddr ctx) { } -lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type) { +lbValue lb_equal_proc_for_type(lbModule *m, Type *type) { type = base_type(type); GB_ASSERT(is_type_comparable(type)); @@ -296,7 +296,7 @@ lbValue lb_simple_compare_hash(lbProcedure *p, Type *type, lbValue data, lbValue return lb_emit_runtime_call(p, "default_hasher_n", args); } -lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { +lbValue lb_hasher_proc_for_type(lbModule *m, Type *type) { type = core_type(type); GB_ASSERT_MSG(is_type_valid_for_keys(type), "%s", type_to_string(type)); @@ -343,7 +343,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { GB_ASSERT(type->Struct.offsets != nullptr); i64 offset = type->Struct.offsets[i]; Entity *field = type->Struct.fields[i]; - lbValue field_hasher = lb_get_hasher_proc_for_type(m, field->type); + lbValue field_hasher = lb_hasher_proc_for_type(m, field->type); lbValue ptr = lb_emit_ptr_offset(p, data, lb_const_int(m, t_uintptr, offset)); args[0] = ptr; @@ -356,7 +356,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { if (is_type_union_maybe_pointer(type)) { Type *v = type->Union.variants[0]; - lbValue variant_hasher = lb_get_hasher_proc_for_type(m, v); + lbValue variant_hasher = lb_hasher_proc_for_type(m, v); args[0] = data; args[1] = seed; @@ -379,7 +379,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { Type *v = type->Union.variants[i]; lbValue case_tag = lb_const_union_tag(p->module, type, v); - lbValue variant_hasher = lb_get_hasher_proc_for_type(m, v); + lbValue variant_hasher = lb_hasher_proc_for_type(m, v); args[0] = data; args[1] = seed; @@ -397,7 +397,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { lb_addr_store(p, pres, seed); auto args = array_make(permanent_allocator(), 2); - lbValue elem_hasher = lb_get_hasher_proc_for_type(m, type->Array.elem); + lbValue elem_hasher = lb_hasher_proc_for_type(m, type->Array.elem); auto loop_data = lb_loop_start(p, cast(isize)type->Array.count, t_i32); @@ -418,7 +418,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { lb_addr_store(p, res, seed); auto args = array_make(permanent_allocator(), 2); - lbValue elem_hasher = lb_get_hasher_proc_for_type(m, type->EnumeratedArray.elem); + lbValue elem_hasher = lb_hasher_proc_for_type(m, type->EnumeratedArray.elem); auto loop_data = lb_loop_start(p, cast(isize)type->EnumeratedArray.count, t_i32); @@ -454,7 +454,7 @@ lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type) { } -lbValue lb_get_map_get_proc_for_type(lbModule *m, Type *type) { +lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { GB_ASSERT(build_context.use_static_map_calls); type = base_type(type); GB_ASSERT(type->kind == Type_Map); @@ -468,7 +468,7 @@ lbValue lb_get_map_get_proc_for_type(lbModule *m, Type *type) { static u32 proc_index = 0; char buf[32] = {}; - isize n = gb_snprintf(buf, 32, "__$map_get%u", ++proc_index); + isize n = gb_snprintf(buf, 32, "__$map_get_%u", ++proc_index); char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); String proc_name = make_string_c(str); @@ -489,9 +489,23 @@ lbValue lb_get_map_get_proc_for_type(lbModule *m, Type *type) { LLVMAddAttributeAtIndex(p->value, 1+1, nonnull_attr); LLVMAddAttributeAtIndex(p->value, 1+1, noalias_attr); + lbBlock *loop_block = lb_create_block(p, "loop"); + lbBlock *hash_block = lb_create_block(p, "hash"); + lbBlock *probe_block = lb_create_block(p, "probe"); + lbBlock *increment_block = lb_create_block(p, "increment"); + lbBlock *hash_compare_block = lb_create_block(p, "hash_compare"); + lbBlock *key_compare_block = lb_create_block(p, "key_compare"); + lbBlock *value_block = lb_create_block(p, "value"); + lbBlock *nil_block = lb_create_block(p, "nil"); + map_ptr = lb_emit_conv(p, map_ptr, t_raw_map_ptr); lbValue map = lb_emit_load(p, map_ptr); + lbValue length = lb_map_len(p, map); + + lb_emit_if(p, lb_emit_comp(p, Token_CmpEq, length, lb_const_nil(m, t_int)), nil_block, hash_block); + lb_start_block(p, hash_block); + key_ptr = lb_emit_conv(p, key_ptr, alloc_type_pointer(type->Map.key)); lbValue key = lb_emit_load(p, key_ptr); @@ -521,16 +535,8 @@ lbValue lb_get_map_get_proc_for_type(lbModule *m, Type *type) { // lbValue res = // LLVMBuildRet(p->builder, res.value); - lbBlock *loop = lb_create_block(p, "loop"); - lbBlock *probe_block = lb_create_block(p, "probe"); - lbBlock *increment_block = lb_create_block(p, "increment"); - lbBlock *hash_compare_block = lb_create_block(p, "hash_compare"); - lbBlock *key_compare_block = lb_create_block(p, "key_compare"); - lbBlock *value_block = lb_create_block(p, "value"); - lbBlock *nil_block = lb_create_block(p, "nil"); - - lb_emit_jump(p, loop); - lb_start_block(p, loop); + lb_emit_jump(p, loop_block); + lb_start_block(p, loop_block); lbValue element_hash = lb_emit_load(p, lb_emit_ptr_offset(p, hs, lb_addr_load(p, pos))); { @@ -577,7 +583,7 @@ lbValue lb_get_map_get_proc_for_type(lbModule *m, Type *type) { lb_addr_store(p, pos, pp); lb_emit_increment(p, distance.addr); } - lb_emit_jump(p, loop); + lb_emit_jump(p, loop_block); lb_start_block(p, nil_block); { @@ -678,8 +684,8 @@ lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type) { LLVMValueRef const_values[4] = {}; const_values[0] = key_cell_info; const_values[1] = value_cell_info; - const_values[2] = lb_get_hasher_proc_for_type(m, map_type->Map.key).value; - const_values[3] = lb_get_equal_proc_for_type(m, map_type->Map.key).value; + const_values[2] = lb_hasher_proc_for_type(m, map_type->Map.key).value; + const_values[3] = lb_equal_proc_for_type(m, map_type->Map.key).value; LLVMValueRef llvm_res = llvm_const_named_struct(m, t_map_info, const_values, gb_count_of(const_values)); lbValue res = {llvm_res, t_map_info}; @@ -746,7 +752,7 @@ lbValue lb_gen_map_key_hash(lbProcedure *p, lbValue key, Type *key_type, lbValue lbValue hashed_key = lb_const_hash(p->module, key, key_type); if (hashed_key.value == nullptr) { - lbValue hasher = lb_get_hasher_proc_for_type(p->module, key_type); + lbValue hasher = lb_hasher_proc_for_type(p->module, key_type); auto args = array_make(permanent_allocator(), 2); args[0] = key_ptr; @@ -767,16 +773,16 @@ lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, key_ptr = lb_emit_conv(p, key_ptr, t_rawptr); if (build_context.use_static_map_calls) { - lbValue map_get_proc = lb_get_map_get_proc_for_type(p->module, map_type); + lbValue map_get_proc = lb_map_get_proc_for_type(p->module, map_type); auto args = array_make(permanent_allocator(), 2); - args[0] = map_ptr; + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); args[1] = key_ptr; ptr = lb_emit_call(p, map_get_proc, args); } else { auto args = array_make(permanent_allocator(), 3); - args[0] = lb_emit_transmute(p, lb_emit_load(p, map_ptr), t_raw_map); + args[0] = lb_emit_transmute(p, map_ptr, t_raw_map_ptr); args[1] = lb_gen_map_info_ptr(p->module, map_type); args[2] = key_ptr; diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index f9fe6cff0..c4333e949 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -463,8 +463,8 @@ lbValue lb_emit_source_code_location_const(lbProcedure *p, String const &procedu lbValue lb_handle_param_value(lbProcedure *p, Type *parameter_type, ParameterValue const ¶m_value, TokenPos const &pos); -lbValue lb_get_equal_proc_for_type(lbModule *m, Type *type); -lbValue lb_get_hasher_proc_for_type(lbModule *m, Type *type); +lbValue lb_equal_proc_for_type(lbModule *m, Type *type); +lbValue lb_hasher_proc_for_type(lbModule *m, Type *type); lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t); LLVMMetadataRef lb_debug_type(lbModule *m, Type *type); diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 05a9fdfbf..aee87242c 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -2215,7 +2215,7 @@ lbValue lb_compare_records(lbProcedure *p, TokenKind op_kind, lbValue left, lbVa args[2] = lb_const_int(p->module, t_int, type_size_of(type)); res = lb_emit_runtime_call(p, "memory_equal", args); } else { - lbValue value = lb_get_equal_proc_for_type(p->module, type); + lbValue value = lb_equal_proc_for_type(p->module, type); auto args = array_make(permanent_allocator(), 2); args[0] = lb_emit_conv(p, left_ptr, t_rawptr); args[1] = lb_emit_conv(p, right_ptr, t_rawptr); diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 6d7d7eecb..4b0323855 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -2319,10 +2319,10 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, case BuiltinProc_type_equal_proc: - return lb_get_equal_proc_for_type(p->module, ce->args[0]->tav.type); + return lb_equal_proc_for_type(p->module, ce->args[0]->tav.type); case BuiltinProc_type_hasher_proc: - return lb_get_hasher_proc_for_type(p->module, ce->args[0]->tav.type); + return lb_hasher_proc_for_type(p->module, ce->args[0]->tav.type); case BuiltinProc_type_map_info: return lb_gen_map_info_ptr(p->module, ce->args[0]->tav.type); diff --git a/src/llvm_backend_type.cpp b/src/llvm_backend_type.cpp index 26f89f985..307d9304b 100644 --- a/src/llvm_backend_type.cpp +++ b/src/llvm_backend_type.cpp @@ -666,7 +666,7 @@ void lb_setup_type_info_data(lbProcedure *p) { // NOTE(bill): Setup type_info da } if (is_type_comparable(t) && !is_type_simple_compare(t)) { - vals[3] = lb_get_equal_proc_for_type(m, t).value; + vals[3] = lb_equal_proc_for_type(m, t).value; } vals[4] = lb_const_bool(m, t_bool, t->Union.custom_align != 0).value; @@ -702,7 +702,7 @@ void lb_setup_type_info_data(lbProcedure *p) { // NOTE(bill): Setup type_info da vals[6] = is_raw_union.value; vals[7] = is_custom_align.value; if (is_type_comparable(t) && !is_type_simple_compare(t)) { - vals[8] = lb_get_equal_proc_for_type(m, t).value; + vals[8] = lb_equal_proc_for_type(m, t).value; } diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index 6d69021ce..30c531d71 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -1434,7 +1434,7 @@ lbValue lb_dynamic_array_allocator(lbProcedure *p, lbValue da) { } lbValue lb_map_len(lbProcedure *p, lbValue value) { - GB_ASSERT(is_type_map(value.type)); + GB_ASSERT_MSG(is_type_map(value.type) || are_types_identical(value.type, t_raw_map), "%s", type_to_string(value.type)); lbValue len = lb_emit_struct_ev(p, value, 1); return lb_emit_conv(p, len, t_int); } From 033525fe13762237a17ba1f08c59a346ff3cf326 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 11:10:26 +0000 Subject: [PATCH 38/62] Force inline of hasher proc where possible --- src/llvm_backend.cpp | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index c313bc825..79db03f85 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -166,6 +166,9 @@ lbValue lb_equal_proc_for_type(lbModule *m, Type *type) { map_set(&m->equal_procs, type, p); lb_begin_procedure_body(p); + lb_add_attribute_to_proc(m, p->value, "readonly"); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + LLVMValueRef x = LLVMGetParam(p->value, 0); LLVMValueRef y = LLVMGetParam(p->value, 1); x = LLVMBuildPointerCast(p->builder, x, ptr_type, ""); @@ -173,6 +176,8 @@ lbValue lb_equal_proc_for_type(lbModule *m, Type *type) { lbValue lhs = {x, pt}; lbValue rhs = {y, pt}; + lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+1, "nonnull"); lbBlock *block_same_ptr = lb_create_block(p, "same_ptr"); lbBlock *block_diff_ptr = lb_create_block(p, "diff_ptr"); @@ -296,6 +301,10 @@ lbValue lb_simple_compare_hash(lbProcedure *p, Type *type, lbValue data, lbValue return lb_emit_runtime_call(p, "default_hasher_n", args); } +void lb_add_callsite_force_inline(lbProcedure *p, lbValue ret_value) { + LLVMAddCallSiteAttribute(ret_value.value, LLVMAttributeIndex_FunctionIndex, lb_create_enum_attribute(p->module->ctx, "alwaysinline")); +} + lbValue lb_hasher_proc_for_type(lbModule *m, Type *type) { type = core_type(type); GB_ASSERT_MSG(is_type_valid_for_keys(type), "%s", type_to_string(type)); @@ -320,16 +329,20 @@ lbValue lb_hasher_proc_for_type(lbModule *m, Type *type) { lb_begin_procedure_body(p); defer (lb_end_procedure_body(p)); + lb_add_attribute_to_proc(m, p->value, "readonly"); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + LLVMValueRef x = LLVMGetParam(p->value, 0); LLVMValueRef y = LLVMGetParam(p->value, 1); lbValue data = {x, t_rawptr}; lbValue seed = {y, t_uintptr}; - LLVMAttributeRef nonnull_attr = lb_create_enum_attribute(m->ctx, "nonnull"); - LLVMAddAttributeAtIndex(p->value, 1+0, nonnull_attr); + lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+0, "readonly"); if (is_type_simple_compare(type)) { lbValue res = lb_simple_compare_hash(p, type, data, seed); + lb_add_callsite_force_inline(p, res); LLVMBuildRet(p->builder, res.value); return {p->value, p->type}; } @@ -361,6 +374,7 @@ lbValue lb_hasher_proc_for_type(lbModule *m, Type *type) { args[0] = data; args[1] = seed; lbValue res = lb_emit_call(p, variant_hasher, args); + lb_add_callsite_force_inline(p, res); LLVMBuildRet(p->builder, res.value); } @@ -439,12 +453,14 @@ lbValue lb_hasher_proc_for_type(lbModule *m, Type *type) { args[0] = data; args[1] = seed; lbValue res = lb_emit_runtime_call(p, "default_hasher_cstring", args); + lb_add_callsite_force_inline(p, res); LLVMBuildRet(p->builder, res.value); } else if (is_type_string(type)) { auto args = array_make(permanent_allocator(), 2); args[0] = data; args[1] = seed; lbValue res = lb_emit_runtime_call(p, "default_hasher_string", args); + lb_add_callsite_force_inline(p, res); LLVMBuildRet(p->builder, res.value); } else { GB_PANIC("Unhandled type for hasher: %s", type_to_string(type)); @@ -477,17 +493,21 @@ lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { lb_begin_procedure_body(p); defer (lb_end_procedure_body(p)); + lb_add_attribute_to_proc(m, p->value, "readonly"); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + LLVMValueRef x = LLVMGetParam(p->value, 0); LLVMValueRef y = LLVMGetParam(p->value, 1); lbValue map_ptr = {x, t_rawptr}; lbValue key_ptr = {y, t_rawptr}; - LLVMAttributeRef nonnull_attr = lb_create_enum_attribute(m->ctx, "nonnull"); - LLVMAttributeRef noalias_attr = lb_create_enum_attribute(m->ctx, "noalias"); - LLVMAddAttributeAtIndex(p->value, 1+0, nonnull_attr); - LLVMAddAttributeAtIndex(p->value, 1+0, noalias_attr); - LLVMAddAttributeAtIndex(p->value, 1+1, nonnull_attr); - LLVMAddAttributeAtIndex(p->value, 1+1, noalias_attr); + lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+0, "noalias"); + lb_add_proc_attribute_at_index(p, 1+0, "readonly"); + + lb_add_proc_attribute_at_index(p, 1+1, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+1, "noalias"); + lb_add_proc_attribute_at_index(p, 1+1, "readonly"); lbBlock *loop_block = lb_create_block(p, "loop"); lbBlock *hash_block = lb_create_block(p, "hash"); @@ -848,6 +868,8 @@ lbProcedure *lb_create_startup_type_info(lbModule *m) { p->is_startup = true; LLVMSetLinkage(p->value, LLVMInternalLinkage); + lb_add_attribute_to_proc(m, p->value, "nounwind"); + lb_begin_procedure_body(p); lb_setup_type_info_data(p); @@ -872,6 +894,7 @@ lbProcedure *lb_create_objc_names(lbModule *main_module) { } Type *proc_type = alloc_type_proc(nullptr, nullptr, 0, nullptr, 0, false, ProcCC_CDecl); lbProcedure *p = lb_create_dummy_procedure(main_module, str_lit("__$init_objc_names"), proc_type); + lb_add_attribute_to_proc(p->module, p->value, "nounwind"); p->is_startup = true; return p; } From b47548178811860afaffbba0690c644823b72d5e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 11:19:34 +0000 Subject: [PATCH 39/62] Get deleted key and value for `delete_key` --- core/runtime/core_builtin.odin | 7 +++++-- core/runtime/dynamic_map_internal.odin | 13 ++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index c04d79455..754316dab 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -300,8 +300,11 @@ shrink_map :: proc(m: ^$T/map[$K]$V, loc := #caller_location) -> (did_shrink: bo delete_key :: proc(m: ^$T/map[$K]$V, key: K) -> (deleted_key: K, deleted_value: V) { if m != nil { key := key - _ = map_erase_dynamic((^Raw_Map)(m), map_info(T), uintptr(&key)) - // TODO(bill) old key and value + old_k, old_v, ok := map_erase_dynamic((^Raw_Map)(m), map_info(T), uintptr(&key)) + if ok { + deleted_key = (^K)(old_k)^ + deleted_value = (^V)(old_v)^ + } } return } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index aedd3f260..303f566ed 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -594,14 +594,18 @@ map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, -map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> bool { +@(require_results) +map_erase_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (old_k, old_v: uintptr, ok: bool) { MASK :: 1 << (size_of(Map_Hash)*8 - 1) index := map_lookup_dynamic(m^, info, k) or_return - _, _, hs, _, _ := map_kvh_data_dynamic(m^, info) + ks, vs, hs, _, _ := map_kvh_data_dynamic(m^, info) hs[index] |= MASK + old_k = map_cell_index_dynamic(ks, info.ks, index) + old_v = map_cell_index_dynamic(vs, info.vs, index) m.len -= 1 - return true + ok = true + return } map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info) { @@ -681,6 +685,7 @@ __dynamic_map_get_with_hash :: proc "contextless" (#no_alias m: ^Raw_Map, #no_al } } +// IMPORTANT: USED WITHIN THE COMPILER __dynamic_map_get :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { if m.len == 0 { return nil @@ -689,6 +694,7 @@ __dynamic_map_get :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: return __dynamic_map_get_with_hash(m, info, h, key) } +// IMPORTANT: USED WITHIN THE COMPILER __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { hash := info.key_hasher(key, 0) @@ -709,6 +715,7 @@ __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_In return rawptr(result) } +// IMPORTANT: USED WITHIN THE COMPILER @(private) __dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) { map_reserve_dynamic(m, info, uintptr(new_capacity), loc) From 5d47e2a166ada673ed08784f47120abe2433f742 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 11:24:34 +0000 Subject: [PATCH 40/62] Change `map_reserve_dynamic` no do anything when current capacity is greater than specified for the reserve --- core/runtime/core_builtin.odin | 2 +- core/runtime/dynamic_map_internal.odin | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 754316dab..dab177f46 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -300,7 +300,7 @@ shrink_map :: proc(m: ^$T/map[$K]$V, loc := #caller_location) -> (did_shrink: bo delete_key :: proc(m: ^$T/map[$K]$V, key: K) -> (deleted_key: K, deleted_value: V) { if m != nil { key := key - old_k, old_v, ok := map_erase_dynamic((^Raw_Map)(m), map_info(T), uintptr(&key)) + old_k, old_v, ok := map_erase_dynamic((^Raw_Map)(m), map_info(T), uintptr(&key)) if ok { deleted_key = (^K)(old_k)^ deleted_value = (^V)(old_v)^ diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 303f566ed..007df4365 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -442,7 +442,6 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ } new_capacity := new_capacity - new_capacity = max(new_capacity, uintptr(1)<= new_capacity { return nil } + + new_capacity = max(new_capacity, uintptr(1)< Date: Fri, 11 Nov 2022 11:41:28 +0000 Subject: [PATCH 41/62] Add minor optimization for `lb_map_cell_index_static` --- src/llvm_backend.cpp | 3 --- src/llvm_backend_stmt.cpp | 14 +++++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 79db03f85..ee3d36896 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -552,9 +552,6 @@ lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { vs = lb_emit_conv(p, vs, alloc_type_pointer(type->Map.value)); hs = lb_emit_conv(p, hs, alloc_type_pointer(t_uintptr)); - // lbValue res = - // LLVMBuildRet(p->builder, res.value); - lb_emit_jump(p, loop_block); lb_start_block(p, loop_block); diff --git a/src/llvm_backend_stmt.cpp b/src/llvm_backend_stmt.cpp index ec7a162cb..c8f244181 100644 --- a/src/llvm_backend_stmt.cpp +++ b/src/llvm_backend_stmt.cpp @@ -382,12 +382,20 @@ lbValue lb_map_cell_index_static(lbProcedure *p, Type *type, lbValue cells_ptr, return lb_emit_ptr_offset(p, elems_ptr, index); } - // TOOD(bill): N power of two optimization to use >> and & + lbValue cell_index = {}; + lbValue data_index = {}; lbValue size_const = lb_const_int(p->module, t_uintptr, size); lbValue len_const = lb_const_int(p->module, t_uintptr, len); - lbValue cell_index = lb_emit_arith(p, Token_Quo, index, len_const, t_uintptr); - lbValue data_index = lb_emit_arith(p, Token_Mod, index, len_const, t_uintptr); + + if (is_power_of_two(len)) { + u64 log2_len = floor_log2(cast(u64)len); + cell_index = log2_len == 0 ? index : lb_emit_arith(p, Token_Shr, index, lb_const_int(p->module, t_uintptr, log2_len), t_uintptr); + data_index = lb_emit_arith(p, Token_And, index, lb_const_int(p->module, t_uintptr, len-1), t_uintptr); + } else { + cell_index = lb_emit_arith(p, Token_Quo, index, len_const, t_uintptr); + data_index = lb_emit_arith(p, Token_Mod, index, len_const, t_uintptr); + } lbValue elems_ptr = lb_emit_conv(p, cells_ptr, t_uintptr); lbValue cell_offset = lb_emit_arith(p, Token_Mul, size_const, cell_index, t_uintptr); From a0bd31646bd5ab0fa4b8e3e8766d59f70c5f0d4c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 13:02:23 +0000 Subject: [PATCH 42/62] Make `map` get internal calls take the hash value rather than compute it internally --- core/runtime/dynamic_map_internal.odin | 21 +++++++++------------ src/checker.cpp | 4 ++-- src/llvm_backend.cpp | 26 ++++++++++++++------------ 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 007df4365..65f883c7c 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -665,7 +665,8 @@ map_get :: proc "contextless" (m: $T/map[$K]$V, key: K) -> (stored_key: K, store } } -__dynamic_map_get_with_hash :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, key: rawptr) -> (ptr: rawptr) { +// IMPORTANT: USED WITHIN THE COMPILER +__dynamic_map_get :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, key: rawptr) -> (ptr: rawptr) { if m.len == 0 { return nil } @@ -687,28 +688,24 @@ __dynamic_map_get_with_hash :: proc "contextless" (#no_alias m: ^Raw_Map, #no_al } } -// IMPORTANT: USED WITHIN THE COMPILER -__dynamic_map_get :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key: rawptr) -> (ptr: rawptr) { - if m.len == 0 { - return nil +__dynamic_map_check_grow :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { + if m.len + 1 >= map_resize_threshold(m^) { + return map_grow_dynamic(m, info, loc) } - h := info.key_hasher(key, 0) - return __dynamic_map_get_with_hash(m, info, h, key) + return nil } // IMPORTANT: USED WITHIN THE COMPILER __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { hash := info.key_hasher(key, 0) - if found := __dynamic_map_get_with_hash(m, info, hash, key); found != nil { + if found := __dynamic_map_get(m, info, hash, key); found != nil { intrinsics.mem_copy_non_overlapping(found, value, info.vs.size_of_type) return found } - if m.len + 1 >= map_resize_threshold(m^) { - if err := map_grow_dynamic(m, info, loc); err != nil { - return nil - } + if __dynamic_map_check_grow(m, info, loc) != nil { + return nil } result := map_insert_hash_dynamic(m, info, hash, uintptr(key), uintptr(value)) diff --git a/src/checker.cpp b/src/checker.cpp index d48b37b26..0c23f2627 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -927,8 +927,8 @@ void init_universal(void) { Type *hasher_args[2] = {t_rawptr, t_uintptr}; t_hasher_proc = alloc_type_proc_from_types(hasher_args, 2, t_uintptr, false, ProcCC_Contextless); - Type *map_get_args[2] = {/*map*/t_rawptr, /*key*/t_rawptr}; - t_map_get_proc = alloc_type_proc_from_types(map_get_args, 2, t_rawptr, false, ProcCC_Contextless); + Type *map_get_args[3] = {/*map*/t_rawptr, /*hash*/t_uintptr, /*key*/t_rawptr}; + t_map_get_proc = alloc_type_proc_from_types(map_get_args, 3, t_rawptr, false, ProcCC_Contextless); } diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index ee3d36896..f0a123e5e 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -498,16 +498,18 @@ lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { LLVMValueRef x = LLVMGetParam(p->value, 0); LLVMValueRef y = LLVMGetParam(p->value, 1); + LLVMValueRef z = LLVMGetParam(p->value, 2); lbValue map_ptr = {x, t_rawptr}; - lbValue key_ptr = {y, t_rawptr}; + lbValue h = {y, t_uintptr}; + lbValue key_ptr = {z, t_rawptr}; lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); lb_add_proc_attribute_at_index(p, 1+0, "noalias"); lb_add_proc_attribute_at_index(p, 1+0, "readonly"); - lb_add_proc_attribute_at_index(p, 1+1, "nonnull"); - lb_add_proc_attribute_at_index(p, 1+1, "noalias"); - lb_add_proc_attribute_at_index(p, 1+1, "readonly"); + lb_add_proc_attribute_at_index(p, 1+2, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+2, "noalias"); + lb_add_proc_attribute_at_index(p, 1+2, "readonly"); lbBlock *loop_block = lb_create_block(p, "loop"); lbBlock *hash_block = lb_create_block(p, "hash"); @@ -529,7 +531,6 @@ lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { key_ptr = lb_emit_conv(p, key_ptr, alloc_type_pointer(type->Map.key)); lbValue key = lb_emit_load(p, key_ptr); - lbValue h = lb_gen_map_key_hash(p, key, type->Map.key, nullptr); lbAddr pos = lb_add_local_generated(p, t_uintptr, false); lbAddr distance = lb_add_local_generated(p, t_uintptr, true); lbValue capacity = lb_map_cap(p, map); @@ -785,23 +786,24 @@ lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, GB_ASSERT(map_type->kind == Type_Map); lbValue ptr = {}; - - lbValue key_ptr = lb_address_from_load_or_generate_local(p, key); - key_ptr = lb_emit_conv(p, key_ptr, t_rawptr); + lbValue key_ptr = {}; + lbValue hash = lb_gen_map_key_hash(p, key, map_type->Map.key, &key_ptr); if (build_context.use_static_map_calls) { lbValue map_get_proc = lb_map_get_proc_for_type(p->module, map_type); - auto args = array_make(permanent_allocator(), 2); + auto args = array_make(permanent_allocator(), 3); args[0] = lb_emit_conv(p, map_ptr, t_rawptr); - args[1] = key_ptr; + args[1] = hash; + args[2] = key_ptr; ptr = lb_emit_call(p, map_get_proc, args); } else { - auto args = array_make(permanent_allocator(), 3); + auto args = array_make(permanent_allocator(), 4); args[0] = lb_emit_transmute(p, map_ptr, t_raw_map_ptr); args[1] = lb_gen_map_info_ptr(p->module, map_type); - args[2] = key_ptr; + args[2] = hash; + args[3] = key_ptr; ptr = lb_emit_runtime_call(p, "__dynamic_map_get", args); } From d2701d8b13bce7ac684c215e5ec4bd1dd65f670a Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 13:04:38 +0000 Subject: [PATCH 43/62] Make `__dynamic_map_set` take the `hash` rather than compute it internally --- src/llvm_backend.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index f0a123e5e..5306ed35e 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -815,17 +815,19 @@ void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, map_type = base_type(map_type); GB_ASSERT(map_type->kind == Type_Map); - lbValue key_ptr = lb_address_from_load_or_generate_local(p, map_key); + lbValue key_ptr = {}; + lbValue hash = lb_gen_map_key_hash(p, map_key, map_type->Map.key, &key_ptr); lbValue v = lb_emit_conv(p, map_value, map_type->Map.value); lbValue value_ptr = lb_address_from_load_or_generate_local(p, v); - auto args = array_make(permanent_allocator(), 5); + auto args = array_make(permanent_allocator(), 6); args[0] = lb_emit_conv(p, map_ptr, t_raw_map_ptr); args[1] = lb_gen_map_info_ptr(p->module, map_type); - args[2] = lb_emit_conv(p, key_ptr, t_rawptr); - args[3] = lb_emit_conv(p, value_ptr, t_rawptr); - args[4] = lb_emit_source_code_location_as_global(p, node); + args[2] = hash; + args[3] = lb_emit_conv(p, key_ptr, t_rawptr); + args[4] = lb_emit_conv(p, value_ptr, t_rawptr); + args[5] = lb_emit_source_code_location_as_global(p, node); lb_emit_runtime_call(p, "__dynamic_map_set", args); } From 16fc96101049ef884401ab6182ea860390abd6a9 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 14:45:22 +0000 Subject: [PATCH 44/62] Begin work on map static set --- core/runtime/dynamic_map_internal.odin | 4 +- src/check_expr.cpp | 52 +++++--- src/checker.cpp | 9 +- src/llvm_backend.cpp | 162 +++++++++++++++++++++++-- src/llvm_backend.hpp | 3 +- src/llvm_backend_const.cpp | 17 ++- src/llvm_backend_expr.cpp | 2 +- src/llvm_backend_general.cpp | 3 +- src/llvm_backend_utility.cpp | 7 ++ src/types.cpp | 1 + 10 files changed, 218 insertions(+), 42 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 65f883c7c..9721340f6 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -696,9 +696,7 @@ __dynamic_map_check_grow :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: } // IMPORTANT: USED WITHIN THE COMPILER -__dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { - hash := info.key_hasher(key, 0) - +__dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, hash: Map_Hash, key, value: rawptr, loc := #caller_location) -> rawptr { if found := __dynamic_map_get(m, info, hash, key); found != nil { intrinsics.mem_copy_non_overlapping(found, value, info.vs.size_of_type) return found diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 045b22ca2..c58aac609 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -285,6 +285,37 @@ void error_operand_no_value(Operand *o) { } } +void add_map_get_dependencies(CheckerContext *c) { + if (build_context.use_static_map_calls) { + add_package_dependency(c, "runtime", "map_desired_position"); + add_package_dependency(c, "runtime", "map_probe_distance"); + } else { + add_package_dependency(c, "runtime", "__dynamic_map_get"); + } +} + +void add_map_set_dependencies(CheckerContext *c) { + init_core_source_code_location(c->checker); + + if (t_map_set_proc == nullptr) { + Type *map_set_args[5] = {/*map*/t_rawptr, /*hash*/t_uintptr, /*key*/t_rawptr, /*value*/t_rawptr, /*#caller_location*/t_source_code_location}; + t_map_set_proc = alloc_type_proc_from_types(map_set_args, gb_count_of(map_set_args), t_rawptr, false, ProcCC_Odin); + } + + if (build_context.use_static_map_calls) { + add_package_dependency(c, "runtime", "__dynamic_map_check_grow"); + add_package_dependency(c, "runtime", "map_insert_hash_dynamic"); + } else { + add_package_dependency(c, "runtime", "__dynamic_map_set"); + } +} + +void add_map_reserve_dependencies(CheckerContext *c) { + init_core_source_code_location(c->checker); + add_package_dependency(c, "runtime", "__dynamic_map_reserve"); +} + + void check_scope_decls(CheckerContext *c, Slice const &nodes, isize reserve_size) { Scope *s = c->scope; @@ -3244,12 +3275,7 @@ void check_binary_expr(CheckerContext *c, Operand *x, Ast *node, Type *type_hint check_assignment(c, x, yt->Map.key, str_lit("map 'not_in'")); } - if (build_context.use_static_map_calls) { - add_package_dependency(c, "runtime", "map_desired_position"); - add_package_dependency(c, "runtime", "map_probe_distance"); - } else { - add_package_dependency(c, "runtime", "__dynamic_map_get"); - } + add_map_get_dependencies(c); } else if (is_type_bit_set(rhs_type)) { Type *yt = base_type(rhs_type); @@ -8560,8 +8586,8 @@ ExprKind check_compound_literal(CheckerContext *c, Operand *o, Ast *node, Type * if (build_context.no_dynamic_literals && cl->elems.count) { error(node, "Compound literals of dynamic types have been disabled"); } else { - add_package_dependency(c, "runtime", "__dynamic_map_reserve"); - add_package_dependency(c, "runtime", "__dynamic_map_set"); + add_map_reserve_dependencies(c); + add_map_set_dependencies(c); } break; } @@ -8997,14 +9023,8 @@ ExprKind check_index_expr(CheckerContext *c, Operand *o, Ast *node, Type *type_h o->type = t->Map.value; o->expr = node; - - add_package_dependency(c, "runtime", "__dynamic_map_set"); - if (build_context.use_static_map_calls) { - add_package_dependency(c, "runtime", "map_desired_position"); - add_package_dependency(c, "runtime", "map_probe_distance"); - } else { - add_package_dependency(c, "runtime", "__dynamic_map_get"); - } + add_map_get_dependencies(c); + add_map_set_dependencies(c); return Expr_Expr; } diff --git a/src/checker.cpp b/src/checker.cpp index 0c23f2627..4d1ef4614 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -922,14 +922,13 @@ void init_universal(void) { { Type *equal_args[2] = {t_rawptr, t_rawptr}; - t_equal_proc = alloc_type_proc_from_types(equal_args, 2, t_bool, false, ProcCC_Contextless); + t_equal_proc = alloc_type_proc_from_types(equal_args, gb_count_of(equal_args), t_bool, false, ProcCC_Contextless); Type *hasher_args[2] = {t_rawptr, t_uintptr}; - t_hasher_proc = alloc_type_proc_from_types(hasher_args, 2, t_uintptr, false, ProcCC_Contextless); + t_hasher_proc = alloc_type_proc_from_types(hasher_args, gb_count_of(hasher_args), t_uintptr, false, ProcCC_Contextless); Type *map_get_args[3] = {/*map*/t_rawptr, /*hash*/t_uintptr, /*key*/t_rawptr}; - t_map_get_proc = alloc_type_proc_from_types(map_get_args, 3, t_rawptr, false, ProcCC_Contextless); - + t_map_get_proc = alloc_type_proc_from_types(map_get_args, gb_count_of(map_get_args), t_rawptr, false, ProcCC_Contextless); } // Constants @@ -2844,7 +2843,7 @@ void init_core_source_code_location(Checker *c) { return; } t_source_code_location = find_core_type(c, str_lit("Source_Code_Location")); - t_source_code_location_ptr = alloc_type_pointer(t_allocator); + t_source_code_location_ptr = alloc_type_pointer(t_source_code_location); } void init_core_map_type(Checker *c) { diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 5306ed35e..4c34a56e9 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -484,7 +484,7 @@ lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { static u32 proc_index = 0; char buf[32] = {}; - isize n = gb_snprintf(buf, 32, "__$map_get_%u", ++proc_index); + isize n = gb_snprintf(buf, 32, "__$map_get-%u", ++proc_index); char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); String proc_name = make_string_c(str); @@ -613,6 +613,129 @@ lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { return {p->value, p->type}; } +void lb_debug_print(lbProcedure *p, String const &str) { + auto args = array_make(heap_allocator(), 1); + args[0] = lb_const_string(p->module, str); + lb_emit_runtime_call(p, "print_string", args); +} + +lbValue lb_map_set_proc_for_type(lbModule *m, Type *type) { + GB_ASSERT(build_context.use_static_map_calls); + type = base_type(type); + GB_ASSERT(type->kind == Type_Map); + + + lbProcedure **found = map_get(&m->map_set_procs, type); + if (found) { + GB_ASSERT(*found != nullptr); + return {(*found)->value, (*found)->type}; + } + static u32 proc_index = 0; + + char buf[32] = {}; + isize n = gb_snprintf(buf, 32, "__$map_set-%u", ++proc_index); + char *str = gb_alloc_str_len(permanent_allocator(), buf, n-1); + String proc_name = make_string_c(str); + + lbProcedure *p = lb_create_dummy_procedure(m, proc_name, t_map_set_proc); + map_set(&m->map_set_procs, type, p); + lb_begin_procedure_body(p); + defer (lb_end_procedure_body(p)); + + lb_add_attribute_to_proc(m, p->value, "nounwind"); + lb_add_attribute_to_proc(m, p->value, "noinline"); + + lbValue map_ptr = {LLVMGetParam(p->value, 0), t_rawptr}; + lbValue hash = {LLVMGetParam(p->value, 1), t_uintptr}; + lbValue key_ptr = {LLVMGetParam(p->value, 2), t_rawptr}; + lbValue value_ptr = {LLVMGetParam(p->value, 3), t_rawptr}; + lbValue location_ptr = {LLVMGetParam(p->value, 4), t_source_code_location_ptr}; + + map_ptr = lb_emit_conv(p, map_ptr, alloc_type_pointer(type)); + key_ptr = lb_emit_conv(p, key_ptr, alloc_type_pointer(type->Map.key)); + + lb_add_proc_attribute_at_index(p, 1+0, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+0, "noalias"); + + lb_add_proc_attribute_at_index(p, 1+2, "nonnull"); + if (!are_types_identical(type->Map.key, type->Map.value)) { + lb_add_proc_attribute_at_index(p, 1+2, "noalias"); + } + lb_add_proc_attribute_at_index(p, 1+2, "readonly"); + + lb_add_proc_attribute_at_index(p, 1+3, "nonnull"); + if (!are_types_identical(type->Map.key, type->Map.value)) { + lb_add_proc_attribute_at_index(p, 1+3, "noalias"); + } + lb_add_proc_attribute_at_index(p, 1+3, "readonly"); + + lb_add_proc_attribute_at_index(p, 1+4, "nonnull"); + lb_add_proc_attribute_at_index(p, 1+4, "noalias"); + lb_add_proc_attribute_at_index(p, 1+4, "readonly"); + + //// + lbValue found_ptr = {}; + { + lbValue map_get_proc = lb_map_get_proc_for_type(m, type); + + auto args = array_make(permanent_allocator(), 3); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = hash; + args[2] = key_ptr; + + found_ptr = lb_emit_call(p, map_get_proc, args); + } + + + lbBlock *found_block = lb_create_block(p, "found"); + lbBlock *check_grow_block = lb_create_block(p, "check-grow"); + lbBlock *grow_fail_block = lb_create_block(p, "grow-fail"); + lbBlock *insert_block = lb_create_block(p, "insert"); + + lb_emit_if(p, lb_emit_comp_against_nil(p, Token_NotEq, found_ptr), found_block, check_grow_block); + lb_start_block(p, found_block); + { + lb_mem_copy_non_overlapping(p, found_ptr, value_ptr, lb_const_int(m, t_int, type_size_of(type->Map.value))); + LLVMBuildRet(p->builder, lb_emit_conv(p, found_ptr, t_rawptr).value); + } + lb_start_block(p, check_grow_block); + + + lbValue map_info = lb_gen_map_info_ptr(p->module, type); + + { + auto args = array_make(permanent_allocator(), 3); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = map_info; + args[2] = lb_emit_load(p, location_ptr); + lbValue grow_err = lb_emit_runtime_call(p, "__dynamic_map_check_grow", args); + + lb_emit_if(p, lb_emit_comp_against_nil(p, Token_NotEq, grow_err), grow_fail_block, insert_block); + + lb_start_block(p, grow_fail_block); + LLVMBuildRet(p->builder, LLVMConstNull(lb_type(m, t_rawptr))); + } + + lb_start_block(p, insert_block); + { + auto args = array_make(permanent_allocator(), 5); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = map_info; + args[2] = hash; + args[3] = lb_emit_conv(p, key_ptr, t_uintptr); + args[4] = lb_emit_conv(p, value_ptr, t_uintptr); + + lbValue result = lb_emit_runtime_call(p, "map_insert_hash_dynamic", args); + + lb_emit_increment(p, lb_map_len_ptr(p, map_ptr)); + + LLVMBuildRet(p->builder, lb_emit_conv(p, result, t_rawptr).value); + } + + return {p->value, p->type}; +} + + lbValue lb_generate_anonymous_proc_lit(lbModule *m, String const &prefix_name, Ast *expr, lbProcedure *parent) { lbProcedure **found = map_get(&m->gen->anonymous_proc_lits, expr); if (found) { @@ -810,8 +933,8 @@ lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, return lb_emit_conv(p, ptr, alloc_type_pointer(map_type->Map.value)); } -void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, Type *map_type, - lbValue const &map_key, lbValue const &map_value, Ast *node) { +void lb_internal_dynamic_map_set(lbProcedure *p, lbValue const &map_ptr, Type *map_type, + lbValue const &map_key, lbValue const &map_value, Ast *node) { map_type = base_type(map_type); GB_ASSERT(map_type->kind == Type_Map); @@ -821,14 +944,27 @@ void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, lbValue v = lb_emit_conv(p, map_value, map_type->Map.value); lbValue value_ptr = lb_address_from_load_or_generate_local(p, v); - auto args = array_make(permanent_allocator(), 6); - args[0] = lb_emit_conv(p, map_ptr, t_raw_map_ptr); - args[1] = lb_gen_map_info_ptr(p->module, map_type); - args[2] = hash; - args[3] = lb_emit_conv(p, key_ptr, t_rawptr); - args[4] = lb_emit_conv(p, value_ptr, t_rawptr); - args[5] = lb_emit_source_code_location_as_global(p, node); - lb_emit_runtime_call(p, "__dynamic_map_set", args); + if (build_context.use_static_map_calls) { + lbValue map_set_proc = lb_map_set_proc_for_type(p->module, map_type); + + auto args = array_make(permanent_allocator(), 5); + args[0] = lb_emit_conv(p, map_ptr, t_rawptr); + args[1] = hash; + args[2] = lb_emit_conv(p, key_ptr, t_rawptr); + args[3] = lb_emit_conv(p, value_ptr, t_rawptr); + args[4] = lb_emit_source_code_location_as_global(p, node); + + lb_emit_call(p, map_set_proc, args); + } else { + auto args = array_make(permanent_allocator(), 6); + args[0] = lb_emit_conv(p, map_ptr, t_raw_map_ptr); + args[1] = lb_gen_map_info_ptr(p->module, map_type); + args[2] = hash; + args[3] = lb_emit_conv(p, key_ptr, t_rawptr); + args[4] = lb_emit_conv(p, value_ptr, t_rawptr); + args[5] = lb_emit_source_code_location_as_global(p, node); + lb_emit_runtime_call(p, "__dynamic_map_set", args); + } } void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos) { @@ -1386,6 +1522,10 @@ WORKER_TASK_PROC(lb_llvm_function_pass_worker_proc) { lbProcedure *p = m->map_get_procs.entries[i].value; lb_run_function_pass_manager(default_function_pass_manager, p); } + for_array(i, m->map_set_procs.entries) { + lbProcedure *p = m->map_set_procs.entries[i].value; + lb_run_function_pass_manager(default_function_pass_manager, p); + } return 0; } diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index c4333e949..9074de42a 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -145,6 +145,7 @@ struct lbModule { PtrMap equal_procs; PtrMap hasher_procs; PtrMap map_get_procs; + PtrMap map_set_procs; u32 nested_type_name_guid; @@ -451,7 +452,7 @@ lbValue lb_gen_map_cell_info_ptr(lbModule *m, Type *type); lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type); lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key); -void lb_insert_dynamic_map_key_and_value(lbProcedure *p, lbValue const &map_ptr, Type *map_type, lbValue const &map_key, lbValue const &map_value, Ast *node); +void lb_internal_dynamic_map_set(lbProcedure *p, lbValue const &map_ptr, Type *map_type, lbValue const &map_key, lbValue const &map_value, Ast *node); void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos); lbValue lb_find_procedure_value_from_entity(lbModule *m, Entity *e); diff --git a/src/llvm_backend_const.cpp b/src/llvm_backend_const.cpp index a8b66a0ea..dff5298c5 100644 --- a/src/llvm_backend_const.cpp +++ b/src/llvm_backend_const.cpp @@ -283,19 +283,28 @@ lbValue lb_emit_source_code_location_const(lbProcedure *p, Ast *node) { return lb_emit_source_code_location_const(p, proc_name, pos); } -lbValue lb_emit_source_code_location_as_global(lbProcedure *p, String const &procedure, TokenPos const &pos) { + +lbValue lb_emit_source_code_location_as_global_ptr(lbProcedure *p, String const &procedure, TokenPos const &pos) { lbValue loc = lb_emit_source_code_location_const(p, procedure, pos); lbAddr addr = lb_add_global_generated(p->module, loc.type, loc, nullptr); lb_make_global_private_const(addr); - return lb_addr_load(p, addr); + return addr.addr; } -lbValue lb_emit_source_code_location_as_global(lbProcedure *p, Ast *node) { +lbValue lb_emit_source_code_location_as_global_ptr(lbProcedure *p, Ast *node) { lbValue loc = lb_emit_source_code_location_const(p, node); lbAddr addr = lb_add_global_generated(p->module, loc.type, loc, nullptr); lb_make_global_private_const(addr); - return lb_addr_load(p, addr); + return addr.addr; +} + +lbValue lb_emit_source_code_location_as_global(lbProcedure *p, String const &procedure, TokenPos const &pos) { + return lb_emit_load(p, lb_emit_source_code_location_as_global_ptr(p, procedure, pos)); +} + +lbValue lb_emit_source_code_location_as_global(lbProcedure *p, Ast *node) { + return lb_emit_load(p, lb_emit_source_code_location_as_global_ptr(p, node)); } diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index aee87242c..e58c84c9c 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -4139,7 +4139,7 @@ lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { lbValue key = lb_build_expr(p, fv->field); lbValue value = lb_build_expr(p, fv->value); - lb_insert_dynamic_map_key_and_value(p, v.addr, type, key, value, elem); + lb_internal_dynamic_map_set(p, v.addr, type, key, value, elem); } break; } diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index a0e4d80ba..f36dc1842 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -68,6 +68,7 @@ void lb_init_module(lbModule *m, Checker *c) { map_init(&m->equal_procs, a); map_init(&m->hasher_procs, a); map_init(&m->map_get_procs, a); + map_init(&m->map_set_procs, a); array_init(&m->procedures_to_generate, a, 0, 1024); array_init(&m->missing_procedures_to_check, a, 0, 16); map_init(&m->debug_values, a); @@ -727,7 +728,7 @@ void lb_addr_store(lbProcedure *p, lbAddr addr, lbValue value) { return; } else if (addr.kind == lbAddr_Map) { - lb_insert_dynamic_map_key_and_value(p, addr.addr, addr.map.type, addr.map.key, value, p->curr_stmt); + lb_internal_dynamic_map_set(p, addr.addr, addr.map.type, addr.map.key, value, p->curr_stmt); return; } else if (addr.kind == lbAddr_Context) { lbAddr old_addr = lb_find_or_generate_context_ptr(p); diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index 30c531d71..101b9dbfb 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -1438,6 +1438,13 @@ lbValue lb_map_len(lbProcedure *p, lbValue value) { lbValue len = lb_emit_struct_ev(p, value, 1); return lb_emit_conv(p, len, t_int); } +lbValue lb_map_len_ptr(lbProcedure *p, lbValue map_ptr) { + Type *type = map_ptr.type; + GB_ASSERT(is_type_pointer(type)); + type = type_deref(type); + GB_ASSERT_MSG(is_type_map(type) || are_types_identical(type, t_raw_map), "%s", type_to_string(type)); + return lb_emit_struct_ep(p, map_ptr, 1); +} lbValue lb_map_cap(lbProcedure *p, lbValue value) { GB_ASSERT_MSG(is_type_map(value.type) || are_types_identical(value.type, t_raw_map), "%s", type_to_string(value.type)); diff --git a/src/types.cpp b/src/types.cpp index 74b192010..47007491d 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -694,6 +694,7 @@ gb_global Type *t_raw_map_ptr = nullptr; gb_global Type *t_equal_proc = nullptr; gb_global Type *t_hasher_proc = nullptr; gb_global Type *t_map_get_proc = nullptr; +gb_global Type *t_map_set_proc = nullptr; gb_global Type *t_objc_object = nullptr; gb_global Type *t_objc_selector = nullptr; From f9576c2f5bc6e1354a9e736b32bdfb193593173b Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 15:28:20 +0000 Subject: [PATCH 45/62] Add internal linkage to static map calls --- src/llvm_backend.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 4c34a56e9..4d2f95bdf 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -493,6 +493,7 @@ lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { lb_begin_procedure_body(p); defer (lb_end_procedure_body(p)); + LLVMSetLinkage(p->value, LLVMInternalLinkage); lb_add_attribute_to_proc(m, p->value, "readonly"); lb_add_attribute_to_proc(m, p->value, "nounwind"); @@ -642,8 +643,8 @@ lbValue lb_map_set_proc_for_type(lbModule *m, Type *type) { lb_begin_procedure_body(p); defer (lb_end_procedure_body(p)); + LLVMSetLinkage(p->value, LLVMInternalLinkage); lb_add_attribute_to_proc(m, p->value, "nounwind"); - lb_add_attribute_to_proc(m, p->value, "noinline"); lbValue map_ptr = {LLVMGetParam(p->value, 0), t_rawptr}; lbValue hash = {LLVMGetParam(p->value, 1), t_uintptr}; From 22840ddf9769c47f8ac0f68b5b12f75200229bd9 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 15:35:05 +0000 Subject: [PATCH 46/62] Add `noinline` LLVM attribute to static map procedures --- src/llvm_backend.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 4d2f95bdf..5b9db7f2b 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -496,6 +496,9 @@ lbValue lb_map_get_proc_for_type(lbModule *m, Type *type) { LLVMSetLinkage(p->value, LLVMInternalLinkage); lb_add_attribute_to_proc(m, p->value, "readonly"); lb_add_attribute_to_proc(m, p->value, "nounwind"); + if (build_context.ODIN_DEBUG) { + lb_add_attribute_to_proc(m, p->value, "noinline"); + } LLVMValueRef x = LLVMGetParam(p->value, 0); LLVMValueRef y = LLVMGetParam(p->value, 1); @@ -645,6 +648,9 @@ lbValue lb_map_set_proc_for_type(lbModule *m, Type *type) { LLVMSetLinkage(p->value, LLVMInternalLinkage); lb_add_attribute_to_proc(m, p->value, "nounwind"); + if (build_context.ODIN_DEBUG) { + lb_add_attribute_to_proc(m, p->value, "noinline"); + } lbValue map_ptr = {LLVMGetParam(p->value, 0), t_rawptr}; lbValue hash = {LLVMGetParam(p->value, 1), t_uintptr}; From 04a1e7d638ec20d52fab954fa432bd76d2eeb395 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 11 Nov 2022 16:15:21 +0000 Subject: [PATCH 47/62] Correct json/unmarshal.odin --- core/encoding/json/unmarshal.odin | 2 +- core/runtime/dynamic_map_internal.odin | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index 8edf1a016..11cae6160 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -427,7 +427,7 @@ unmarshal_object :: proc(p: ^Parser, v: any, end_token: Token_Kind) -> (err: Unm key_ptr = &key_cstr } - set_ptr := runtime.__dynamic_map_set(raw_map, t.map_info, key_ptr, map_backing_value.data) + set_ptr := runtime.__dynamic_map_set_without_hash(raw_map, t.map_info, key_ptr, map_backing_value.data) if set_ptr == nil { delete(key, p.allocator) } diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 9721340f6..5543a62dc 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -688,6 +688,7 @@ __dynamic_map_get :: proc "contextless" (#no_alias m: ^Raw_Map, #no_alias info: } } +// IMPORTANT: USED WITHIN THE COMPILER __dynamic_map_check_grow :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { if m.len + 1 >= map_resize_threshold(m^) { return map_grow_dynamic(m, info, loc) @@ -695,6 +696,11 @@ __dynamic_map_check_grow :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: return nil } +__dynamic_map_set_without_hash :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, key, value: rawptr, loc := #caller_location) -> rawptr { + return __dynamic_map_set(m, info, info.key_hasher(key, 0), key, value, loc) +} + + // IMPORTANT: USED WITHIN THE COMPILER __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, hash: Map_Hash, key, value: rawptr, loc := #caller_location) -> rawptr { if found := __dynamic_map_get(m, info, hash, key); found != nil { From 9c1b464c945b607a104f5e01ae734373f9300316 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sat, 12 Nov 2022 17:25:42 +0100 Subject: [PATCH 48/62] Add tests for new map implementation. --- .github/workflows/ci.yml | 7 ++ tests/internal/test_map.odin | 122 +++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 tests/internal/test_map.odin diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 882c21e75..689c1d67a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,13 @@ jobs: cd tests\vendor call build.bat timeout-minutes: 10 + - name: Odin internals tests + shell: cmd + run: | + call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat + cd tests\internal + call build.bat + timeout-minutes: 10 - name: core:math/big tests shell: cmd run: | diff --git a/tests/internal/test_map.odin b/tests/internal/test_map.odin new file mode 100644 index 000000000..c318b6bdc --- /dev/null +++ b/tests/internal/test_map.odin @@ -0,0 +1,122 @@ +package test_internal_map + +import "core:fmt" +import "core:intrinsics" +import "core:math/rand" +import "core:mem" +import "core:os" +import "core:testing" + +seed: u64 + +ENTRY_COUNTS := []int{11, 101} // , 1_001, 10_001, 100_001, 1_000_001} + +@test +map_insert_random_key_value :: proc(t: ^testing.T) { + for entries in ENTRY_COUNTS { + fmt.printf("[map_insert_random_key_value] Testing %v entries.\n", entries) + m: map[i64]i64 + + unique_keys := 0 + r := rand.create(seed) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected value. Expected m[%v] = %v, got %v", k, v, m[k])) + } + } + clear(&m) + delete(m) + } +} + +// -------- -------- -------- -------- -------- -------- -------- -------- -------- -------- + +main :: proc() { + t := testing.T{} + + // Allow tests to be repeatable + SEED :: #config(SEED, -1) + when SEED > 0 { + seed = u64(SEED) + } else { + seed = u64(intrinsics.read_cycle_counter()) + } + fmt.println("Initialized seed to", seed) + + mem_track_test(&t, map_insert_random_key_value) + + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + if TEST_fail > 0 { + os.exit(1) + } +} + +mem_track_test :: proc(t: ^testing.T, test: proc(t: ^testing.T)) { + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + test(t) + + expect(t, len(track.allocation_map) == 0, "Expected no leaks.") + expect(t, len(track.bad_free_array) == 0, "Expected no leaks.") + + for _, leak in track.allocation_map { + fmt.printf("%v leaked %v bytes\n", leak.location, leak.size) + } + for bad_free in track.bad_free_array { + fmt.printf("%v allocation %p was freed badly\n", bad_free.location, bad_free.memory) + } +} + +TEST_count := 0 +TEST_fail := 0 + +when ODIN_TEST { + expect :: testing.expect + log :: testing.log +} else { + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] ", loc) + fmt.printf("log: %v\n", v) + } +} \ No newline at end of file From 7207f4b0c5e7c171f3fdec19e13db651001bc0a3 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sat, 12 Nov 2022 17:31:26 +0100 Subject: [PATCH 49/62] Add tests/internal/build.bat --- tests/internal/build.bat | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 tests/internal/build.bat diff --git a/tests/internal/build.bat b/tests/internal/build.bat new file mode 100644 index 000000000..45e1acc72 --- /dev/null +++ b/tests/internal/build.bat @@ -0,0 +1,2 @@ +@echo off +odin run test_map.odin -file -vet -strict-style -o:minimal -define:SEED=42 \ No newline at end of file From 699cabeb1cff9334759d8f936c0a2b0e0a49369c Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sat, 12 Nov 2022 17:36:20 +0100 Subject: [PATCH 50/62] Update tests/internal/build.bat --- tests/internal/build.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/internal/build.bat b/tests/internal/build.bat index 45e1acc72..d747ebfef 100644 --- a/tests/internal/build.bat +++ b/tests/internal/build.bat @@ -1,2 +1,3 @@ @echo off -odin run test_map.odin -file -vet -strict-style -o:minimal -define:SEED=42 \ No newline at end of file +set PATH_TO_ODIN==..\..\odin +%PATH_TO_ODIN% run test_map.odin -file -vet -strict-style -o:minimal -define:SEED=42 \ No newline at end of file From ad0f11668b4ddd74f719c0861d2e1e4cd6812ac3 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Nov 2022 14:53:58 +0000 Subject: [PATCH 51/62] Correct `map_reserve_dynamic` caused by an bizarre code generation bug --- core/mem/alloc.odin | 8 +++----- core/runtime/core_builtin.odin | 6 +++--- core/runtime/dynamic_map_internal.odin | 28 ++++++++++++++++---------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/core/mem/alloc.odin b/core/mem/alloc.odin index 4f449bd37..c7a21dcd4 100644 --- a/core/mem/alloc.odin +++ b/core/mem/alloc.odin @@ -164,8 +164,6 @@ new_clone :: proc(data: $T, allocator := context.allocator, loc := #caller_locat return nil, .Out_Of_Memory } -DEFAULT_RESERVE_CAPACITY :: 16 - make_aligned :: proc($T: typeid/[]$E, #any_int len: int, alignment: int, allocator := context.allocator, loc := #caller_location) -> (slice: T, err: Allocator_Error) { runtime.make_slice_error_loc(loc, len) data := alloc_bytes(size_of(E)*len, alignment, allocator, loc) or_return @@ -179,7 +177,7 @@ make_slice :: proc($T: typeid/[]$E, #any_int len: int, allocator := context.allo return make_aligned(T, len, align_of(E), allocator, loc) } make_dynamic_array :: proc($T: typeid/[dynamic]$E, allocator := context.allocator, loc := #caller_location) -> (T, Allocator_Error) { - return make_dynamic_array_len_cap(T, 0, DEFAULT_RESERVE_CAPACITY, allocator, loc) + return make_dynamic_array_len_cap(T, 0, 16, allocator, loc) } make_dynamic_array_len :: proc($T: typeid/[dynamic]$E, #any_int len: int, allocator := context.allocator, loc := #caller_location) -> (T, Allocator_Error) { return make_dynamic_array_len_cap(T, len, len, allocator, loc) @@ -194,12 +192,12 @@ make_dynamic_array_len_cap :: proc($T: typeid/[dynamic]$E, #any_int len: int, #a array = transmute(T)s return } -make_map :: proc($T: typeid/map[$K]$E, #any_int cap: int = DEFAULT_RESERVE_CAPACITY, allocator := context.allocator, loc := #caller_location) -> T { +make_map :: proc($T: typeid/map[$K]$E, #any_int cap: int = 1< T { runtime.make_map_expr_error_loc(loc, cap) context.allocator = allocator m: T - reserve_map(&m, cap) + reserve_map(&m, cap, loc) return m } make_multi_pointer :: proc($T: typeid/[^]$E, #any_int len: int, allocator := context.allocator, loc := #caller_location) -> (mp: T, err: Allocator_Error) { diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index dab177f46..24a53dfbd 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -231,12 +231,12 @@ make_dynamic_array_len_cap :: proc($T: typeid/[dynamic]$E, #any_int len: int, #a return } @(builtin) -make_map :: proc($T: typeid/map[$K]$E, #any_int cap: int = 1< T { - make_map_expr_error_loc(loc, cap) +make_map :: proc($T: typeid/map[$K]$E, #any_int capacity: int = 1< T { + make_map_expr_error_loc(loc, capacity) context.allocator = allocator m: T - reserve_map(&m, cap) + reserve_map(&m, capacity, loc) return m } @(builtin) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index 5543a62dc..e943a3e24 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -437,36 +437,42 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr, loc := #caller_location) -> Allocator_Error { + ceil_log2 :: #force_inline proc "contextless" (x: uintptr) -> uintptr { + z := intrinsics.count_leading_zeros(x) + if z > 0 && x & (x-1) != 0 { + z -= 1 + } + return size_of(uintptr)*8 - 1 - z + } + if m.allocator.procedure == nil { m.allocator = context.allocator } new_capacity := new_capacity + old_capacity := uintptr(map_cap(m^)) - log2_capacity := map_log2_cap(m^) - capacity := uintptr(1) << log2_capacity - - if capacity >= new_capacity { + if old_capacity >= new_capacity { return nil } - new_capacity = max(new_capacity, uintptr(1)< Allocator_Error { - if m.len + 1 >= map_resize_threshold(m^) { + if m.len >= map_resize_threshold(m^) { return map_grow_dynamic(m, info, loc) } return nil From 16a494347c82ea22686146b5e932d6683d761751 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 13 Nov 2022 16:24:20 +0100 Subject: [PATCH 52/62] map: Add tests for update + delete. --- tests/internal/test_map.odin | 153 +++++++++++++++++++++++++++++++++-- 1 file changed, 148 insertions(+), 5 deletions(-) diff --git a/tests/internal/test_map.odin b/tests/internal/test_map.odin index c318b6bdc..597f3ea1c 100644 --- a/tests/internal/test_map.odin +++ b/tests/internal/test_map.odin @@ -9,16 +9,18 @@ import "core:testing" seed: u64 -ENTRY_COUNTS := []int{11, 101} // , 1_001, 10_001, 100_001, 1_000_001} +ENTRY_COUNTS := []int{11, 101, 1_001, 10_001, 100_001, 1_000_001} @test map_insert_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) for entries in ENTRY_COUNTS { fmt.printf("[map_insert_random_key_value] Testing %v entries.\n", entries) m: map[i64]i64 + defer delete(m) unique_keys := 0 - r := rand.create(seed) + r := rand.create(seed + seed_incr) for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected value. Expected m[%v] = %v, got %v", k, v, m[k])) + } + } + seed_incr += 1 + } +} + +@test +map_delete_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) + for entries in ENTRY_COUNTS { + fmt.printf("[map_delete_random_key_value] Testing %v entries.\n", entries) + m: map[i64]i64 + defer delete(m) + + unique_keys := 0 + r := rand.create(seed + seed_incr) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected key present. Expected m[%v] to have been deleted, got %v", k, m[k])) + } + } else { + if k not_in m { + num_fails += 1 + if num_fails > 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Expected key not present. Expected m[%v] = %v", k, v)) + } else if m[k] != v { + num_fails += 1 + if num_fails > 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected value. Expected m[%v] = %v, got %v", k, v, m[k])) + } + } + } + seed_incr += 1 } } @@ -75,6 +216,8 @@ main :: proc() { fmt.println("Initialized seed to", seed) mem_track_test(&t, map_insert_random_key_value) + mem_track_test(&t, map_update_random_key_value) + mem_track_test(&t, map_delete_random_key_value) fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) if TEST_fail > 0 { From 9b88a38e54818daf6fda4e2dda4ab4aa6c6dbd5d Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 13 Nov 2022 16:32:24 +0100 Subject: [PATCH 53/62] map tests for Linux and Mac --- .github/workflows/ci.yml | 10 ++++++++++ tests/internal/Makefile | 6 ++++++ tests/internal/build.bat | 3 ++- 3 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 tests/internal/Makefile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 689c1d67a..0c43690ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,11 @@ jobs: cd tests/vendor make timeout-minutes: 10 + - name: Odin internals tests + run: | + cd tests/internal + make + timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 timeout-minutes: 10 @@ -87,6 +92,11 @@ jobs: cd tests/vendor make timeout-minutes: 10 + - name: Odin internals tests + run: | + cd tests/internals + make + timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 timeout-minutes: 10 diff --git a/tests/internal/Makefile b/tests/internal/Makefile new file mode 100644 index 000000000..7328968f0 --- /dev/null +++ b/tests/internal/Makefile @@ -0,0 +1,6 @@ +ODIN=../../odin + +all: map_test + +map_test: + $(ODIN) run test_map.odin -file -vet -strict-style -o:minimal \ No newline at end of file diff --git a/tests/internal/build.bat b/tests/internal/build.bat index d747ebfef..313e1dbb5 100644 --- a/tests/internal/build.bat +++ b/tests/internal/build.bat @@ -1,3 +1,4 @@ @echo off set PATH_TO_ODIN==..\..\odin -%PATH_TO_ODIN% run test_map.odin -file -vet -strict-style -o:minimal -define:SEED=42 \ No newline at end of file +%PATH_TO_ODIN% run test_map.odin -file -vet -strict-style -o:minimal +rem -define:SEED=42 \ No newline at end of file From 7dfbda58d9c51932fa91a04727b821860358289d Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 13 Nov 2022 16:38:22 +0100 Subject: [PATCH 54/62] Fix CI typo. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c43690ad..dc2691d80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,7 @@ jobs: timeout-minutes: 10 - name: Odin internals tests run: | - cd tests/internals + cd tests/internal make timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 From a705a2e38bee035d6800999ba6eddd82792594f7 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Nov 2022 22:55:32 +0000 Subject: [PATCH 55/62] Minor improvement to multi return value reducing stack usage --- src/check_expr.cpp | 3 +++ src/check_stmt.cpp | 4 ++++ src/checker.hpp | 1 + src/llvm_backend_general.cpp | 24 ++++++++++++--------- src/llvm_backend_opt.cpp | 35 +++++++++++++++++++++++++++++- src/llvm_backend_proc.cpp | 42 +++++++++++++++++++++++++++++------- 6 files changed, 90 insertions(+), 19 deletions(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index c58aac609..c819267fd 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -6739,6 +6739,9 @@ ExprKind check_call_expr(CheckerContext *c, Operand *operand, Ast *call, Ast *pr if (initial_entity != nullptr && initial_entity->kind == Entity_Procedure) { if (initial_entity->Procedure.deferred_procedure.entity != nullptr) { call->viral_state_flags |= ViralStateFlag_ContainsDeferredProcedure; + if (c->decl != nullptr) { + c->decl->defer_use_count += 1; + } } } diff --git a/src/check_stmt.cpp b/src/check_stmt.cpp index 9cacb4a35..8271ca451 100644 --- a/src/check_stmt.cpp +++ b/src/check_stmt.cpp @@ -2014,6 +2014,10 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { if (is_ast_decl(ds->stmt)) { error(ds->token, "You cannot defer a declaration"); } else { + if (ctx->decl != nullptr) { + ctx->decl->defer_use_count += 1; + } + bool out_in_defer = ctx->in_defer; ctx->in_defer = true; check_stmt(ctx, ds->stmt, 0); diff --git a/src/checker.hpp b/src/checker.hpp index badcd93d5..ded4a0ad6 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -158,6 +158,7 @@ struct DeclInfo { bool is_using; bool where_clauses_evaluated; bool proc_checked; + isize defer_use_count; CommentGroup *comment; CommentGroup *docs; diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index f36dc1842..6f2253d01 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -936,23 +936,27 @@ void lb_emit_store(lbProcedure *p, lbValue ptr, lbValue value) { LLVMValueRef src_ptr_original = LLVMGetOperand(value.value, 0); LLVMValueRef src_ptr = LLVMBuildPointerCast(p->builder, src_ptr_original, LLVMTypeOf(dst_ptr), ""); - LLVMBuildMemMove(p->builder, - dst_ptr, lb_try_get_alignment(dst_ptr, 1), - src_ptr, lb_try_get_alignment(src_ptr_original, 1), - LLVMConstInt(LLVMInt64TypeInContext(p->module->ctx), lb_sizeof(LLVMTypeOf(value.value)), false)); + if (dst_ptr != src_ptr && dst_ptr != src_ptr_original) { + LLVMBuildMemMove(p->builder, + dst_ptr, lb_try_get_alignment(dst_ptr, 1), + src_ptr, lb_try_get_alignment(src_ptr_original, 1), + LLVMConstInt(LLVMInt64TypeInContext(p->module->ctx), lb_sizeof(LLVMTypeOf(value.value)), false)); + } return; } else if (LLVMIsConstant(value.value)) { lbAddr addr = lb_add_global_generated(p->module, value.type, value, nullptr); lb_make_global_private_const(addr); LLVMValueRef dst_ptr = ptr.value; - LLVMValueRef src_ptr = addr.addr.value; - src_ptr = LLVMBuildPointerCast(p->builder, src_ptr, LLVMTypeOf(dst_ptr), ""); + LLVMValueRef src_ptr_original = addr.addr.value; + LLVMValueRef src_ptr = LLVMBuildPointerCast(p->builder, src_ptr_original, LLVMTypeOf(dst_ptr), ""); - LLVMBuildMemMove(p->builder, - dst_ptr, lb_try_get_alignment(dst_ptr, 1), - src_ptr, lb_try_get_alignment(src_ptr, 1), - LLVMConstInt(LLVMInt64TypeInContext(p->module->ctx), lb_sizeof(LLVMTypeOf(value.value)), false)); + if (dst_ptr != src_ptr && dst_ptr != src_ptr_original) { + LLVMBuildMemMove(p->builder, + dst_ptr, lb_try_get_alignment(dst_ptr, 1), + src_ptr, lb_try_get_alignment(src_ptr, 1), + LLVMConstInt(LLVMInt64TypeInContext(p->module->ctx), lb_sizeof(LLVMTypeOf(value.value)), false)); + } return; } } diff --git a/src/llvm_backend_opt.cpp b/src/llvm_backend_opt.cpp index e2f51b868..ba5a41871 100644 --- a/src/llvm_backend_opt.cpp +++ b/src/llvm_backend_opt.cpp @@ -267,6 +267,8 @@ void lb_populate_module_pass_manager(LLVMTargetMachineRef target_machine, LLVMPa **************************************************************************/ void lb_run_remove_dead_instruction_pass(lbProcedure *p) { + LLVMTypeRef llvm_void = LLVMVoidTypeInContext(p->module->ctx); + isize removal_count = 0; isize pass_count = 0; isize const max_pass_count = 10; @@ -322,7 +324,7 @@ void lb_run_remove_dead_instruction_pass(lbProcedure *p) { case LLVMOr: case LLVMXor: case LLVMAlloca: - case LLVMLoad: + case LLVMLoad: // TODO: should LLVMLoad be removed? case LLVMGetElementPtr: case LLVMTrunc: case LLVMZExt: @@ -347,6 +349,37 @@ void lb_run_remove_dead_instruction_pass(lbProcedure *p) { LLVMInstructionEraseFromParent(curr_instr); was_dead_instructions = true; break; + + case LLVMCall: + if (LLVMTypeOf(curr_instr) == llvm_void) { + LLVMValueRef the_proc = LLVMGetCalledValue(curr_instr); + unsigned id = LLVMGetIntrinsicID(the_proc); + if (id != 0) { + size_t text_len = 0; + char const *text = LLVMIntrinsicGetName(id, &text_len); + String name = make_string(cast(u8 const *)text, cast(isize)text_len); + if (name == "llvm.memmove" || name == "llvm.memcpy") { + LLVMValueRef dst = LLVMGetOperand(curr_instr, 0); + LLVMValueRef src = LLVMGetOperand(curr_instr, 1); + LLVMValueRef sz = LLVMGetOperand(curr_instr, 2); + if ((dst == src) || (LLVMIsConstant(sz) && LLVMConstIntGetZExtValue(sz) == 0)) { + removal_count += 1; + LLVMInstructionEraseFromParent(curr_instr); + was_dead_instructions = true; + break; + } + } else if (name == "llvm.memset") { + LLVMValueRef sz = LLVMGetOperand(curr_instr, 2); + if (LLVMIsConstant(sz) && LLVMConstIntGetZExtValue(sz) == 0) { + removal_count += 1; + LLVMInstructionEraseFromParent(curr_instr); + was_dead_instructions = true; + break; + } + } + } + } + break; } } } diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index eaff6edc0..2e508a939 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -9,9 +9,15 @@ LLVMValueRef lb_call_intrinsic(lbProcedure *p, const char *name, LLVMValueRef* a } void lb_mem_copy_overlapping(lbProcedure *p, lbValue dst, lbValue src, lbValue len, bool is_volatile) { + if (dst.value == src.value) { + return; + } dst = lb_emit_conv(p, dst, t_rawptr); src = lb_emit_conv(p, src, t_rawptr); len = lb_emit_conv(p, len, t_int); + if (dst.value == src.value) { + return; + } char const *name = "llvm.memmove"; if (LLVMIsConstant(len.value)) { @@ -38,9 +44,16 @@ void lb_mem_copy_overlapping(lbProcedure *p, lbValue dst, lbValue src, lbValue l void lb_mem_copy_non_overlapping(lbProcedure *p, lbValue dst, lbValue src, lbValue len, bool is_volatile) { + if (dst.value == src.value) { + return; + } dst = lb_emit_conv(p, dst, t_rawptr); src = lb_emit_conv(p, src, t_rawptr); len = lb_emit_conv(p, len, t_int); + if (dst.value == src.value) { + return; + } + char const *name = "llvm.memcpy"; if (LLVMIsConstant(len.value)) { @@ -580,18 +593,31 @@ void lb_begin_procedure_body(lbProcedure *p) { if (e->token.string != "") { GB_ASSERT(!is_blank_ident(e->token)); - // NOTE(bill): Don't even bother trying to optimize this with the return ptr value - // This will violate the defer rules if you do: - // foo :: proc() -> (x, y: T) { - // defer x = ... // defer is executed after the `defer` - // return // the values returned should be zeroed - // } - lbAddr res = lb_add_local(p, e->type, e); + lbAddr res = {}; + if (p->return_ptr.addr.value != nullptr && + p->entity != nullptr && + p->entity->decl_info != nullptr && + p->entity->decl_info->defer_use_count == 0) { + lbValue val = lb_emit_struct_ep(p, p->return_ptr.addr, cast(i32)i); + + lb_add_entity(p->module, e, val); + lb_add_debug_local_variable(p, val.value, e->type, e->token); + + // NOTE(bill): no need to zero initialize due to caller will zero return value + res = lb_addr(val); + } else { + // NOTE(bill): Don't even bother trying to optimize this with the return ptr value + // This will violate the defer rules if you do: + // foo :: proc() -> (x, y: T) { + // defer x = ... // defer is executed after the `defer` + // return // the values returned should be zeroed + // } + res = lb_add_local(p, e->type, e); + } if (e->Variable.param_value.kind != ParameterValue_Invalid) { lbValue c = lb_handle_param_value(p, e->type, e->Variable.param_value, e->token.pos); lb_addr_store(p, res, c); } - } } From 3edb3d8d8c16a24371bb401fe0d69ec93e667b27 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Nov 2022 23:24:08 +0000 Subject: [PATCH 56/62] Simplify the handling of the hashing calls for `map`s --- core/runtime/dynamic_map_internal.odin | 96 ++++++-------------------- src/check_type.cpp | 20 ++---- src/llvm_backend.cpp | 14 +--- 3 files changed, 30 insertions(+), 100 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index e943a3e24..ac09c3e2b 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -732,92 +732,42 @@ __dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Ma +// NOTE: the default hashing algorithm derives from fnv64a, with some minor modifications to work for `map` type: +// +// * Convert a `0` result to `1` +// * "empty entry" +// * Prevent the top bit from being set +// * "deleted entry" +// +// Both of these modification are necessary for the implementation of the `map` INITIAL_HASH_SEED :: 0xcbf29ce484222325 HASH_MASK :: 1 << (8*size_of(uintptr) - 1) -1 -_fnv64a :: proc "contextless" (data: []byte, seed: u64 = INITIAL_HASH_SEED) -> u64 { - h: u64 = seed - for b in data { - h = (h ~ u64(b)) * 0x100000001b3 - } - h &= HASH_MASK - return h | u64(h == 0) -} - -default_hash :: #force_inline proc "contextless" (data: []byte) -> uintptr { - return uintptr(_fnv64a(data)) -} -default_hash_string :: #force_inline proc "contextless" (s: string) -> uintptr { - return default_hash(transmute([]byte)(s)) -} -default_hash_ptr :: #force_inline proc "contextless" (data: rawptr, size: int) -> uintptr { - s := Raw_Slice{data, size} - return default_hash(transmute([]byte)(s)) -} - -@(private) -_default_hasher_const :: #force_inline proc "contextless" (data: rawptr, seed: uintptr, $N: uint) -> uintptr where N <= 16 { - h := u64(seed) + 0xcbf29ce484222325 - p := uintptr(data) - #unroll for _ in 0.. uintptr { - h := u64(seed) + 0xcbf29ce484222325 - p := uintptr(data) +default_hasher :: #force_inline proc "contextless" (data: rawptr, seed: uintptr, N: int) -> uintptr { + h := u64(seed) + INITIAL_HASH_SEED + p := ([^]byte)(data) for _ in 0.. uintptr { return #force_inline _default_hasher_const(data, seed, 1) } -default_hasher2 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 2) } -default_hasher3 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 3) } -default_hasher4 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 4) } -default_hasher5 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 5) } -default_hasher6 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 6) } -default_hasher7 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 7) } -default_hasher8 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 8) } -default_hasher9 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 9) } -default_hasher10 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 10) } -default_hasher11 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 11) } -default_hasher12 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 12) } -default_hasher13 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 13) } -default_hasher14 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 14) } -default_hasher15 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 15) } -default_hasher16 :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { return #force_inline _default_hasher_const(data, seed, 16) } - default_hasher_string :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { - h := u64(seed) + 0xcbf29ce484222325 - str := (^[]byte)(data)^ - for b in str { - h = (h ~ u64(b)) * 0x100000001b3 - } - h &= HASH_MASK - return uintptr(h) | uintptr(h == 0) + str := (^[]byte)(data) + return default_hasher(raw_data(str^), seed, len(str)) } default_hasher_cstring :: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr { - h := u64(seed) + 0xcbf29ce484222325 - ptr := (^uintptr)(data)^ - for (^byte)(ptr)^ != 0 { - b := (^byte)(ptr)^ - h = (h ~ u64(b)) * 0x100000001b3 - ptr += 1 + h := u64(seed) + INITIAL_HASH_SEED + if ptr := (^[^]byte)(data)^; ptr != nil { + for ptr[0] != 0 { + h = (h ~ u64(ptr[0])) * 0x100000001b3 + ptr = ptr[1:] + } } h &= HASH_MASK - return uintptr(h) | uintptr(h == 0) + return uintptr(h) | uintptr(uintptr(h) == 0) } diff --git a/src/check_type.cpp b/src/check_type.cpp index 5d7f8d7b5..4d94fce6c 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2221,35 +2221,27 @@ void add_map_key_type_dependencies(CheckerContext *ctx, Type *key) { } if (is_type_simple_compare(key)) { - i64 sz = type_size_of(key); - if (1 <= sz && sz <= 16) { - char buf[20] = {}; - gb_snprintf(buf, 20, "default_hasher%d", cast(i32)sz); - add_package_dependency(ctx, "runtime", buf); - return; - } else { - add_package_dependency(ctx, "runtime", "default_hasher_n"); - return; - } + add_package_dependency(ctx, "runtime", "default_hasher"); + return; } if (key->kind == Type_Struct) { - add_package_dependency(ctx, "runtime", "default_hasher_n"); + add_package_dependency(ctx, "runtime", "default_hasher"); for_array(i, key->Struct.fields) { Entity *field = key->Struct.fields[i]; add_map_key_type_dependencies(ctx, field->type); } } else if (key->kind == Type_Union) { - add_package_dependency(ctx, "runtime", "default_hasher_n"); + add_package_dependency(ctx, "runtime", "default_hasher"); for_array(i, key->Union.variants) { Type *v = key->Union.variants[i]; add_map_key_type_dependencies(ctx, v); } } else if (key->kind == Type_EnumeratedArray) { - add_package_dependency(ctx, "runtime", "default_hasher_n"); + add_package_dependency(ctx, "runtime", "default_hasher"); add_map_key_type_dependencies(ctx, key->EnumeratedArray.elem); } else if (key->kind == Type_Array) { - add_package_dependency(ctx, "runtime", "default_hasher_n"); + add_package_dependency(ctx, "runtime", "default_hasher"); add_map_key_type_dependencies(ctx, key->Array.elem); } } diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 5b9db7f2b..2ee292880 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -282,23 +282,11 @@ lbValue lb_equal_proc_for_type(lbModule *m, Type *type) { lbValue lb_simple_compare_hash(lbProcedure *p, Type *type, lbValue data, lbValue seed) { GB_ASSERT_MSG(is_type_simple_compare(type), "%s", type_to_string(type)); - i64 sz = type_size_of(type); - - if (1 <= sz && sz <= 16) { - char name[32] = {}; - gb_snprintf(name, 32, "default_hasher%d", cast(i32)sz); - - auto args = array_make(permanent_allocator(), 2); - args[0] = data; - args[1] = seed; - return lb_emit_runtime_call(p, name, args); - } - auto args = array_make(permanent_allocator(), 3); args[0] = data; args[1] = seed; args[2] = lb_const_int(p->module, t_int, type_size_of(type)); - return lb_emit_runtime_call(p, "default_hasher_n", args); + return lb_emit_runtime_call(p, "default_hasher", args); } void lb_add_callsite_force_inline(lbProcedure *p, lbValue ret_value) { From 489e8dc59272cd0db755da53a2c1de362e4175b3 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Nov 2022 23:47:00 +0000 Subject: [PATCH 57/62] Add @(require_results) to map procedures where possible --- core/runtime/dynamic_map_internal.odin | 50 ++++++++++++++++++-------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/core/runtime/dynamic_map_internal.odin b/core/runtime/dynamic_map_internal.odin index ac09c3e2b..0ecd0b73b 100644 --- a/core/runtime/dynamic_map_internal.odin +++ b/core/runtime/dynamic_map_internal.odin @@ -107,6 +107,7 @@ Map_Cell_Info :: struct { map_cell_info :: intrinsics.type_map_cell_info // Same as the above procedure but at runtime with the cell Map_Cell_Info value. +@(require_results) map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, #no_alias info: ^Map_Cell_Info, index: uintptr) -> uintptr { // Micro-optimize the common cases to save on integer division. elements_per_cell := uintptr(info.elements_per_cell) @@ -128,6 +129,7 @@ map_cell_index_dynamic :: #force_inline proc "contextless" (base: uintptr, #no_a } // Same as above procedure but with compile-time constant index. +@(require_results) map_cell_index_dynamic_const :: proc "contextless" (base: uintptr, #no_alias info: ^Map_Cell_Info, $INDEX: uintptr) -> uintptr { elements_per_cell := uintptr(info.elements_per_cell) size_of_cell := uintptr(info.size_of_cell) @@ -143,6 +145,7 @@ map_cell_index_dynamic_const :: proc "contextless" (base: uintptr, #no_alias inf // The following compile-time procedure indexes such a [N]Cell(T) structure as // if it were a flat array accounting for the internal padding introduced by the // Cell structure. +@(require_results) map_cell_index_static :: #force_inline proc "contextless" (cells: [^]Map_Cell($T), index: uintptr) -> ^T #no_bounds_check { N :: size_of(Map_Cell(T){}.data) / size_of(T) when size_of(T) > 0 else 1 @@ -179,11 +182,13 @@ map_cell_index_static :: #force_inline proc "contextless" (cells: [^]Map_Cell($T } // len() for map +@(require_results) map_len :: #force_inline proc "contextless" (m: Raw_Map) -> int { return m.len } // cap() for map +@(require_results) map_cap :: #force_inline proc "contextless" (m: Raw_Map) -> int { // The data uintptr stores the capacity in the lower six bits which gives the // a maximum value of 2^6-1, or 63. We store the integer log2 of capacity @@ -196,10 +201,12 @@ map_cap :: #force_inline proc "contextless" (m: Raw_Map) -> int { // some math is needed to compute it. Compute it as a fixed point percentage to // avoid floating point operations. This division can be optimized out by // multiplying by the multiplicative inverse of 100. +@(require_results) map_load_factor :: #force_inline proc "contextless" (log2_capacity: uintptr) -> uintptr { return ((uintptr(1) << log2_capacity) * MAP_LOAD_FACTOR) / 100 } +@(require_results) map_resize_threshold :: #force_inline proc "contextless" (m: Raw_Map) -> int { return int(map_load_factor(map_log2_cap(m))) } @@ -208,12 +215,14 @@ map_resize_threshold :: #force_inline proc "contextless" (m: Raw_Map) -> int { // used in the implementation rather than map_cap since the check for data = 0 // isn't necessary in the implementation. cap() on the otherhand needs to work // when called on an empty map. +@(require_results) map_log2_cap :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr { return m.data & (64 - 1) } // Canonicalize the data by removing the tagged capacity stored in the lower six // bits of the data uintptr. +@(require_results) map_data :: #force_inline proc "contextless" (m: Raw_Map) -> uintptr { return m.data &~ uintptr(64 - 1) } @@ -224,15 +233,18 @@ Map_Hash :: uintptr // Procedure to check if a slot is empty for a given hash. This is represented // by the zero value to make the zero value useful. This is a procedure just // for prose reasons. +@(require_results) map_hash_is_empty :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { return hash == 0 } +@(require_results) map_hash_is_deleted :: #force_no_inline proc "contextless" (hash: Map_Hash) -> bool { // The MSB indicates a tombstone N :: size_of(Map_Hash)*8 - 1 return hash >> N != 0 } +@(require_results) map_hash_is_valid :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { // The MSB indicates a tombstone N :: size_of(Map_Hash)*8 - 1 @@ -242,12 +254,14 @@ map_hash_is_valid :: #force_inline proc "contextless" (hash: Map_Hash) -> bool { // Computes the desired position in the array. This is just index % capacity, // but a procedure as there's some math involved here to recover the capacity. +@(require_results) map_desired_position :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Hash) -> uintptr { // We do not use map_cap since we know the capacity will not be zero here. capacity := uintptr(1) << map_log2_cap(m) return uintptr(hash & Map_Hash(capacity - 1)) } +@(require_results) map_probe_distance :: #force_inline proc "contextless" (m: Raw_Map, hash: Map_Hash, slot: uintptr) -> uintptr { // We do not use map_cap since we know the capacity will not be zero here. capacity := uintptr(1) << map_log2_cap(m) @@ -275,6 +289,7 @@ Map_Info :: struct { // map_info :: proc "contextless" ($T: typeid/map[$K]$V) -> ^Map_Info {...} map_info :: intrinsics.type_map_info +@(require_results) map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (ks: uintptr, vs: uintptr, hs: [^]Map_Hash, sk: uintptr, sv: uintptr) { INFO_HS := intrinsics.type_map_cell_info(Map_Hash) @@ -291,13 +306,14 @@ map_kvh_data_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Inf return } +@(require_results) map_kvh_data_values_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info) -> (vs: uintptr) { capacity := uintptr(1) << map_log2_cap(m) return map_cell_index_dynamic(map_data(m), info.ks, capacity) // Skip past ks to get start of vs } -@(private) +@(private, require_results) map_total_allocation_size :: #force_inline proc "contextless" (capacity: uintptr, info: ^Map_Info) -> uintptr { round :: #force_inline proc "contextless" (value: uintptr) -> uintptr { CACHE_MASK :: MAP_CACHE_LINE_SIZE - 1 @@ -315,6 +331,7 @@ map_total_allocation_size :: #force_inline proc "contextless" (capacity: uintptr } // The only procedure which needs access to the context is the one which allocates the map. +@(require_results) map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, allocator := context.allocator, loc := #caller_location) -> (result: Raw_Map, err: Allocator_Error) { result.allocator = allocator // set the allocator always if log2_capacity == 0 { @@ -355,6 +372,7 @@ map_alloc_dynamic :: proc "odin" (info: ^Map_Info, log2_capacity: uintptr, alloc // there is no type information. // // This procedure returns the address of the just inserted value. +@(require_results) map_insert_hash_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, h: Map_Hash, ik: uintptr, iv: uintptr) -> (result: uintptr) { h := h pos := map_desired_position(m^, h) @@ -429,6 +447,7 @@ map_insert_hash_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^ } } +@(require_results) map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { log2_capacity := map_log2_cap(m^) new_capacity := uintptr(1) << max(log2_capacity + 1, MAP_MIN_LOG2_CAPACITY) @@ -436,7 +455,9 @@ map_grow_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Inf } +@(require_results) map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uintptr, loc := #caller_location) -> Allocator_Error { + @(require_results) ceil_log2 :: #force_inline proc "contextless" (x: uintptr) -> uintptr { z := intrinsics.count_leading_zeros(x) if z > 0 && x & (x-1) != 0 { @@ -482,7 +503,7 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ } k := map_cell_index_dynamic(ks, info.ks, i) v := map_cell_index_dynamic(vs, info.vs, i) - map_insert_hash_dynamic(&resized, info, hash, k, v) + _ = map_insert_hash_dynamic(&resized, info, hash, k, v) // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. n -= 1 @@ -499,6 +520,7 @@ map_reserve_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_ } +@(require_results) map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, loc := #caller_location) -> Allocator_Error { if m.allocator.procedure == nil { m.allocator = context.allocator @@ -530,9 +552,7 @@ map_shrink_dynamic :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_I k := map_cell_index_dynamic(ks, info.ks, i) v := map_cell_index_dynamic(vs, info.vs, i) - - map_insert_hash_dynamic(&shrunk, info, hash, k, v) - + _ = map_insert_hash_dynamic(&shrunk, info, hash, k, v) // Only need to do this comparison on each actually added pair, so do not // fold it into the for loop comparator as a micro-optimization. n -= 1 @@ -555,6 +575,7 @@ map_free_dynamic :: proc "odin" (m: Raw_Map, info: ^Map_Info, loc := #caller_loc return mem_free_with_size(ptr, size, m.allocator, loc) } +@(require_results) map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (index: uintptr, ok: bool) { if map_len(m) == 0 { return 0, false @@ -577,6 +598,7 @@ map_lookup_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, d += 1 } } +@(require_results) map_exists_dynamic :: proc "contextless" (m: Raw_Map, #no_alias info: ^Map_Info, k: uintptr) -> (ok: bool) { if map_len(m) == 0 { return false @@ -626,16 +648,17 @@ map_clear_dynamic :: #force_inline proc "contextless" (#no_alias m: ^Raw_Map, #n } -map_kvh_data_static :: #force_inline proc "contextless" (m: $T/map[$K]$V) -> ([^]Map_Cell(K), [^]Map_Cell(V), [^]Map_Hash) { - H :: Map_Hash +@(require_results) +map_kvh_data_static :: #force_inline proc "contextless" (m: $T/map[$K]$V) -> (ks: [^]Map_Cell(K), vs: [^]Map_Cell(V), hs: [^]Map_Hash) { capacity := uintptr(cap(m)) - ks := ([^]Map_Cell(K))(map_data(transmute(Raw_Map)m)) - vs := ([^]Map_Cell(V))(map_cell_index_static(ks, capacity)) - hs := ([^]Map_Cell(H))(map_cell_index_static(vs, capacity)) - return ks, vs, ([^]Map_Hash)(hs) + ks = ([^]Map_Cell(K))(map_data(transmute(Raw_Map)m)) + vs = ([^]Map_Cell(V))(map_cell_index_static(ks, capacity)) + hs = ([^]Map_Hash)(map_cell_index_static(vs, capacity)) + return } +@(require_results) map_get :: proc "contextless" (m: $T/map[$K]$V, key: K) -> (stored_key: K, stored_value: V, ok: bool) { rm := transmute(Raw_Map)m if rm.len == 0 { @@ -719,15 +742,14 @@ __dynamic_map_set :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_In } result := map_insert_hash_dynamic(m, info, hash, uintptr(key), uintptr(value)) - assert(result != 0) m.len += 1 return rawptr(result) } // IMPORTANT: USED WITHIN THE COMPILER @(private) -__dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) { - map_reserve_dynamic(m, info, uintptr(new_capacity), loc) +__dynamic_map_reserve :: proc "odin" (#no_alias m: ^Raw_Map, #no_alias info: ^Map_Info, new_capacity: uint, loc := #caller_location) -> Allocator_Error { + return map_reserve_dynamic(m, info, uintptr(new_capacity), loc) } From d2019e3e4d4b45c34bdc0ef7cf7d630ee61a02fb Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Nov 2022 23:50:45 +0000 Subject: [PATCH 58/62] Enforce pointer cast --- src/llvm_backend.cpp | 4 ++-- src/llvm_backend_expr.cpp | 3 ++- src/llvm_backend_proc.cpp | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 2ee292880..594224e6a 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -962,7 +962,7 @@ void lb_internal_dynamic_map_set(lbProcedure *p, lbValue const &map_ptr, Type *m } } -void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos) { +lbValue lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos) { GB_ASSERT(!build_context.no_dynamic_literals); String proc_name = {}; @@ -975,7 +975,7 @@ void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const args[1] = lb_gen_map_info_ptr(p->module, type_deref(map_ptr.type)); args[2] = lb_const_int(p->module, t_uint, capacity); args[3] = lb_emit_source_code_location_as_global(p, proc_name, pos); - lb_emit_runtime_call(p, "__dynamic_map_reserve", args); + return lb_emit_runtime_call(p, "__dynamic_map_reserve", args); } diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index e58c84c9c..034682855 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -4131,7 +4131,8 @@ lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { } GB_ASSERT(!build_context.no_dynamic_literals); - lb_dynamic_map_reserve(p, v.addr, 2*cl->elems.count, pos); + lbValue err = lb_dynamic_map_reserve(p, v.addr, 2*cl->elems.count, pos); + gb_unused(err); for_array(field_index, cl->elems) { Ast *elem = cl->elems[field_index]; diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 2e508a939..510479440 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -599,6 +599,7 @@ void lb_begin_procedure_body(lbProcedure *p) { p->entity->decl_info != nullptr && p->entity->decl_info->defer_use_count == 0) { lbValue val = lb_emit_struct_ep(p, p->return_ptr.addr, cast(i32)i); + val = lb_emit_conv(p, val, alloc_type_pointer(e->type)); lb_add_entity(p->module, e, val); lb_add_debug_local_variable(p, val.value, e->type, e->token); From 81f83d5780121cc0840c911aca3cf82a0b1d7598 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Nov 2022 23:51:59 +0000 Subject: [PATCH 59/62] Fix prototype --- src/llvm_backend.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index 9074de42a..72dfdda54 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -453,7 +453,7 @@ lbValue lb_gen_map_info_ptr(lbModule *m, Type *map_type); lbValue lb_internal_dynamic_map_get_ptr(lbProcedure *p, lbValue const &map_ptr, lbValue const &key); void lb_internal_dynamic_map_set(lbProcedure *p, lbValue const &map_ptr, Type *map_type, lbValue const &map_key, lbValue const &map_value, Ast *node); -void lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos); +lbValue lb_dynamic_map_reserve(lbProcedure *p, lbValue const &map_ptr, isize const capacity, TokenPos const &pos); lbValue lb_find_procedure_value_from_entity(lbModule *m, Entity *e); lbValue lb_find_value_from_entity(lbModule *m, Entity *e); From 25bec19b1f91cac47544ec434b825bd2b9727a39 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 13 Nov 2022 23:56:05 +0000 Subject: [PATCH 60/62] Revert "Minor improvement to multi return value reducing stack usage" --- src/check_expr.cpp | 3 --- src/check_stmt.cpp | 4 ---- src/checker.hpp | 1 - src/llvm_backend_general.cpp | 24 ++++++++------------ src/llvm_backend_opt.cpp | 35 +--------------------------- src/llvm_backend_proc.cpp | 44 ++++++++---------------------------- 6 files changed, 20 insertions(+), 91 deletions(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index c819267fd..c58aac609 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -6739,9 +6739,6 @@ ExprKind check_call_expr(CheckerContext *c, Operand *operand, Ast *call, Ast *pr if (initial_entity != nullptr && initial_entity->kind == Entity_Procedure) { if (initial_entity->Procedure.deferred_procedure.entity != nullptr) { call->viral_state_flags |= ViralStateFlag_ContainsDeferredProcedure; - if (c->decl != nullptr) { - c->decl->defer_use_count += 1; - } } } diff --git a/src/check_stmt.cpp b/src/check_stmt.cpp index 8271ca451..9cacb4a35 100644 --- a/src/check_stmt.cpp +++ b/src/check_stmt.cpp @@ -2014,10 +2014,6 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { if (is_ast_decl(ds->stmt)) { error(ds->token, "You cannot defer a declaration"); } else { - if (ctx->decl != nullptr) { - ctx->decl->defer_use_count += 1; - } - bool out_in_defer = ctx->in_defer; ctx->in_defer = true; check_stmt(ctx, ds->stmt, 0); diff --git a/src/checker.hpp b/src/checker.hpp index ded4a0ad6..badcd93d5 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -158,7 +158,6 @@ struct DeclInfo { bool is_using; bool where_clauses_evaluated; bool proc_checked; - isize defer_use_count; CommentGroup *comment; CommentGroup *docs; diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 6f2253d01..f36dc1842 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -936,27 +936,23 @@ void lb_emit_store(lbProcedure *p, lbValue ptr, lbValue value) { LLVMValueRef src_ptr_original = LLVMGetOperand(value.value, 0); LLVMValueRef src_ptr = LLVMBuildPointerCast(p->builder, src_ptr_original, LLVMTypeOf(dst_ptr), ""); - if (dst_ptr != src_ptr && dst_ptr != src_ptr_original) { - LLVMBuildMemMove(p->builder, - dst_ptr, lb_try_get_alignment(dst_ptr, 1), - src_ptr, lb_try_get_alignment(src_ptr_original, 1), - LLVMConstInt(LLVMInt64TypeInContext(p->module->ctx), lb_sizeof(LLVMTypeOf(value.value)), false)); - } + LLVMBuildMemMove(p->builder, + dst_ptr, lb_try_get_alignment(dst_ptr, 1), + src_ptr, lb_try_get_alignment(src_ptr_original, 1), + LLVMConstInt(LLVMInt64TypeInContext(p->module->ctx), lb_sizeof(LLVMTypeOf(value.value)), false)); return; } else if (LLVMIsConstant(value.value)) { lbAddr addr = lb_add_global_generated(p->module, value.type, value, nullptr); lb_make_global_private_const(addr); LLVMValueRef dst_ptr = ptr.value; - LLVMValueRef src_ptr_original = addr.addr.value; - LLVMValueRef src_ptr = LLVMBuildPointerCast(p->builder, src_ptr_original, LLVMTypeOf(dst_ptr), ""); + LLVMValueRef src_ptr = addr.addr.value; + src_ptr = LLVMBuildPointerCast(p->builder, src_ptr, LLVMTypeOf(dst_ptr), ""); - if (dst_ptr != src_ptr && dst_ptr != src_ptr_original) { - LLVMBuildMemMove(p->builder, - dst_ptr, lb_try_get_alignment(dst_ptr, 1), - src_ptr, lb_try_get_alignment(src_ptr, 1), - LLVMConstInt(LLVMInt64TypeInContext(p->module->ctx), lb_sizeof(LLVMTypeOf(value.value)), false)); - } + LLVMBuildMemMove(p->builder, + dst_ptr, lb_try_get_alignment(dst_ptr, 1), + src_ptr, lb_try_get_alignment(src_ptr, 1), + LLVMConstInt(LLVMInt64TypeInContext(p->module->ctx), lb_sizeof(LLVMTypeOf(value.value)), false)); return; } } diff --git a/src/llvm_backend_opt.cpp b/src/llvm_backend_opt.cpp index ba5a41871..e2f51b868 100644 --- a/src/llvm_backend_opt.cpp +++ b/src/llvm_backend_opt.cpp @@ -267,8 +267,6 @@ void lb_populate_module_pass_manager(LLVMTargetMachineRef target_machine, LLVMPa **************************************************************************/ void lb_run_remove_dead_instruction_pass(lbProcedure *p) { - LLVMTypeRef llvm_void = LLVMVoidTypeInContext(p->module->ctx); - isize removal_count = 0; isize pass_count = 0; isize const max_pass_count = 10; @@ -324,7 +322,7 @@ void lb_run_remove_dead_instruction_pass(lbProcedure *p) { case LLVMOr: case LLVMXor: case LLVMAlloca: - case LLVMLoad: // TODO: should LLVMLoad be removed? + case LLVMLoad: case LLVMGetElementPtr: case LLVMTrunc: case LLVMZExt: @@ -349,37 +347,6 @@ void lb_run_remove_dead_instruction_pass(lbProcedure *p) { LLVMInstructionEraseFromParent(curr_instr); was_dead_instructions = true; break; - - case LLVMCall: - if (LLVMTypeOf(curr_instr) == llvm_void) { - LLVMValueRef the_proc = LLVMGetCalledValue(curr_instr); - unsigned id = LLVMGetIntrinsicID(the_proc); - if (id != 0) { - size_t text_len = 0; - char const *text = LLVMIntrinsicGetName(id, &text_len); - String name = make_string(cast(u8 const *)text, cast(isize)text_len); - if (name == "llvm.memmove" || name == "llvm.memcpy") { - LLVMValueRef dst = LLVMGetOperand(curr_instr, 0); - LLVMValueRef src = LLVMGetOperand(curr_instr, 1); - LLVMValueRef sz = LLVMGetOperand(curr_instr, 2); - if ((dst == src) || (LLVMIsConstant(sz) && LLVMConstIntGetZExtValue(sz) == 0)) { - removal_count += 1; - LLVMInstructionEraseFromParent(curr_instr); - was_dead_instructions = true; - break; - } - } else if (name == "llvm.memset") { - LLVMValueRef sz = LLVMGetOperand(curr_instr, 2); - if (LLVMIsConstant(sz) && LLVMConstIntGetZExtValue(sz) == 0) { - removal_count += 1; - LLVMInstructionEraseFromParent(curr_instr); - was_dead_instructions = true; - break; - } - } - } - } - break; } } } diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 510479440..7c83125ca 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -9,15 +9,9 @@ LLVMValueRef lb_call_intrinsic(lbProcedure *p, const char *name, LLVMValueRef* a } void lb_mem_copy_overlapping(lbProcedure *p, lbValue dst, lbValue src, lbValue len, bool is_volatile) { - if (dst.value == src.value) { - return; - } dst = lb_emit_conv(p, dst, t_rawptr); src = lb_emit_conv(p, src, t_rawptr); len = lb_emit_conv(p, len, t_int); - if (dst.value == src.value) { - return; - } char const *name = "llvm.memmove"; if (LLVMIsConstant(len.value)) { @@ -44,16 +38,9 @@ void lb_mem_copy_overlapping(lbProcedure *p, lbValue dst, lbValue src, lbValue l void lb_mem_copy_non_overlapping(lbProcedure *p, lbValue dst, lbValue src, lbValue len, bool is_volatile) { - if (dst.value == src.value) { - return; - } dst = lb_emit_conv(p, dst, t_rawptr); src = lb_emit_conv(p, src, t_rawptr); len = lb_emit_conv(p, len, t_int); - if (dst.value == src.value) { - return; - } - char const *name = "llvm.memcpy"; if (LLVMIsConstant(len.value)) { @@ -593,32 +580,19 @@ void lb_begin_procedure_body(lbProcedure *p) { if (e->token.string != "") { GB_ASSERT(!is_blank_ident(e->token)); - lbAddr res = {}; - if (p->return_ptr.addr.value != nullptr && - p->entity != nullptr && - p->entity->decl_info != nullptr && - p->entity->decl_info->defer_use_count == 0) { - lbValue val = lb_emit_struct_ep(p, p->return_ptr.addr, cast(i32)i); - val = lb_emit_conv(p, val, alloc_type_pointer(e->type)); - - lb_add_entity(p->module, e, val); - lb_add_debug_local_variable(p, val.value, e->type, e->token); - - // NOTE(bill): no need to zero initialize due to caller will zero return value - res = lb_addr(val); - } else { - // NOTE(bill): Don't even bother trying to optimize this with the return ptr value - // This will violate the defer rules if you do: - // foo :: proc() -> (x, y: T) { - // defer x = ... // defer is executed after the `defer` - // return // the values returned should be zeroed - // } - res = lb_add_local(p, e->type, e); - } + // NOTE(bill): Don't even bother trying to optimize this with the return ptr value + // This will violate the defer rules if you do: + // foo :: proc() -> (x, y: T) { + // defer x = ... // defer is executed after the `defer` + // return // the values returned should be zeroed + // } + // NOTE(bill): REALLY, don't even bother. + lbAddr res = lb_add_local(p, e->type, e); if (e->Variable.param_value.kind != ParameterValue_Invalid) { lbValue c = lb_handle_param_value(p, e->type, e->Variable.param_value, e->token.pos); lb_addr_store(p, res, c); } + } } From bbe44b49bcd66b7bfc74795a69142f8b94919107 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 14 Nov 2022 11:46:41 +0000 Subject: [PATCH 61/62] Correct `map_insert` --- core/runtime/core_builtin.odin | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 24a53dfbd..6792734d2 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -629,10 +629,7 @@ shrink_dynamic_array :: proc(array: ^$T/[dynamic]$E, new_cap := -1, loc := #call @builtin map_insert :: proc(m: ^$T/map[$K]$V, key: K, value: V, loc := #caller_location) -> (ptr: ^V) { key, value := key, value - h := __get_map_header_table(T) - - e := __dynamic_map_set(m, h, __get_map_key_hash(&key), &key, &value, loc) - return (^V)(uintptr(e) + h.value_offset) + return (^V)(__dynamic_map_set_without_hash((^Raw_Map)(m), map_info(T), rawptr(&key), rawptr(&value), loc)) } From 3949e2220feca6c718a27ecc0fd5cb1cde56f7b7 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 15 Nov 2022 01:27:29 +0100 Subject: [PATCH 62/62] Test new map when used as a set. map[K]struct{} works fine. --- tests/internal/test_map.odin | 117 +++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/tests/internal/test_map.odin b/tests/internal/test_map.odin index 597f3ea1c..781fbad74 100644 --- a/tests/internal/test_map.odin +++ b/tests/internal/test_map.odin @@ -201,6 +201,120 @@ map_delete_random_key_value :: proc(t: ^testing.T) { } } +@test +set_insert_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) + for entries in ENTRY_COUNTS { + fmt.printf("[set_insert_random_key_value] Testing %v entries.\n", entries) + m: map[i64]struct{} + defer delete(m) + + unique_keys := 0 + r := rand.create(seed + seed_incr) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected value. Expected m[%v] to exist", k)) + } + } + seed_incr += 1 + } +} + +@test +set_delete_random_key_value :: proc(t: ^testing.T) { + seed_incr := u64(0) + for entries in ENTRY_COUNTS { + fmt.printf("[set_delete_random_key_value] Testing %v entries.\n", entries) + m: map[i64]struct{} + defer delete(m) + + unique_keys := 0 + r := rand.create(seed + seed_incr) + for _ in 0.. 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Unexpected key present. Expected m[%v] to have been deleted", k)) + } + } else { + if k not_in m { + num_fails += 1 + if num_fails > 5 { + fmt.println("... and more") + break + } + expect(t, false, fmt.tprintf("Expected key not present. Expected m[%v] to exist", k)) + } + } + } + seed_incr += 1 + } +} + // -------- -------- -------- -------- -------- -------- -------- -------- -------- -------- main :: proc() { @@ -219,6 +333,9 @@ main :: proc() { mem_track_test(&t, map_update_random_key_value) mem_track_test(&t, map_delete_random_key_value) + mem_track_test(&t, set_insert_random_key_value) + mem_track_test(&t, set_delete_random_key_value) + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) if TEST_fail > 0 { os.exit(1)