ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-08-05 23:28:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Vectorize runtime.memory_* comparison procedures

Browse Source
This commit is contained in:
Feoramund
2025-05-29 16:29:13 -04:00
parent 827a6f9045
commit 34698288b8
1 changed files with 136 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
base/runtime/internal.odin
+136 -54
View File
@@ -234,91 +234,173 @@ memory_equal :: proc "contextless" (x, y: rawptr, n: int) -> bool {
case n == 0: return true case n == 0: return true
case x == y: return true case x == y: return true
} }
a, b := ([^]byte)(x), ([^]byte)(y) a, b := cast([^]byte)x, cast([^]byte)y
length := uint(n)
for i := uint(0); i < length; i += 1 { n := uint(n)
i := uint(0)
m := uint(0)
if n >= 8 {
when !SIMD_IS_EMULATED {
// Avoid using 256-bit SIMD on platforms where its emulation is
// likely to be less than ideal.
when ODIN_ARCH == .amd64 && intrinsics.has_target_feature("avx2") {
m = n / 32 * 32
for /**/; i < m; i += 32 {
load_a := intrinsics.unaligned_load(cast(^#simd[32]u8)&a[i])
load_b := intrinsics.unaligned_load(cast(^#simd[32]u8)&b[i])
ne := intrinsics.simd_lanes_ne(load_a, load_b)
if intrinsics.simd_reduce_or(ne) != 0 {
return false
}
}
}
}
m = (n-i) / 16 * 16
for /**/; i < m; i += 16 {
load_a := intrinsics.unaligned_load(cast(^#simd[16]u8)&a[i])
load_b := intrinsics.unaligned_load(cast(^#simd[16]u8)&b[i])
ne := intrinsics.simd_lanes_ne(load_a, load_b)
if intrinsics.simd_reduce_or(ne) != 0 {
return false
}
}
m = (n-i) / 8 * 8
for /**/; i < m; i += 8 {
if intrinsics.unaligned_load(cast(^uintptr)&a[i]) != intrinsics.unaligned_load(cast(^uintptr)&b[i]) {
return false
}
}
}
for /**/; i < n; i += 1 {
if a[i] != b[i] { if a[i] != b[i] {
return false return false
} }
} }
return true return true
} }
memory_compare :: proc "contextless" (a, b: rawptr, n: int) -> int #no_bounds_check {
memory_compare :: proc "contextless" (x, y: rawptr, n: int) -> int #no_bounds_check {
switch { switch {
case a == b: return 0 case x == y: return 0
case a == nil: return -1 case x == nil: return -1
case b == nil: return +1 case y == nil: return +1
} }
a, b := cast([^]byte)x, cast([^]byte)y
n := uint(n)
i := uint(0)
m := uint(0)
x := uintptr(a) when !SIMD_IS_EMULATED {
y := uintptr(b) when ODIN_ARCH == .amd64 && intrinsics.has_target_feature("avx2") {
n := uintptr(n) m = n / 32 * 32
for /**/; i < m; i += 32 {
SU :: size_of(uintptr) load_a := intrinsics.unaligned_load(cast(^#simd[32]u8)&a[i])
fast := n/SU + 1 load_b := intrinsics.unaligned_load(cast(^#simd[32]u8)&b[i])
offset := (fast-1)*SU comparison := intrinsics.simd_lanes_ne(load_a, load_b)
curr_block := uintptr(0) if intrinsics.simd_reduce_or(comparison) != 0 {
if n < SU { sentinel: #simd[32]u8 = u8(0xFF)
fast = 0 indices := intrinsics.simd_indices(#simd[32]u8)
} index_select := intrinsics.simd_select(comparison, indices, sentinel)
index_reduce := cast(uint)intrinsics.simd_reduce_min(index_select)
for /**/; curr_block < fast; curr_block += 1 { return -1 if a[i+index_reduce] < b[i+index_reduce] else +1
va := (^uintptr)(x + curr_block * size_of(uintptr))^
vb := (^uintptr)(y + curr_block * size_of(uintptr))^
if va ~ vb != 0 {
for pos := curr_block*SU; pos < n; pos += 1 {
a := (^byte)(x+pos)^
b := (^byte)(y+pos)^
if a ~ b != 0 {
return -1 if (int(a) - int(b)) < 0 else +1
} }
} }
} }
} }
for /**/; offset < n; offset += 1 { m = (n-i) / 16 * 16
a := (^byte)(x+offset)^ for /**/; i < m; i += 16 {
b := (^byte)(y+offset)^ load_a := intrinsics.unaligned_load(cast(^#simd[16]u8)&a[i])
if a ~ b != 0 { load_b := intrinsics.unaligned_load(cast(^#simd[16]u8)&b[i])
return -1 if (int(a) - int(b)) < 0 else +1 comparison := intrinsics.simd_lanes_ne(load_a, load_b)
if intrinsics.simd_reduce_or(comparison) != 0 {
sentinel: #simd[16]u8 = u8(0xFF)
indices := intrinsics.simd_indices(#simd[16]u8)
index_select := intrinsics.simd_select(comparison, indices, sentinel)
index_reduce := cast(uint)intrinsics.simd_reduce_min(index_select)
return -1 if a[i+index_reduce] < b[i+index_reduce] else +1
} }
} }
// 64-bit SIMD is faster than using a `uintptr` to detect a difference then
// re-iterating with the byte-by-byte loop, at least on AMD64.
m = (n-i) / 8 * 8
for /**/; i < m; i += 8 {
load_a := intrinsics.unaligned_load(cast(^#simd[8]u8)&a[i])
load_b := intrinsics.unaligned_load(cast(^#simd[8]u8)&b[i])
comparison := intrinsics.simd_lanes_ne(load_a, load_b)
if intrinsics.simd_reduce_or(comparison) != 0 {
sentinel: #simd[8]u8 = u8(0xFF)
indices := intrinsics.simd_indices(#simd[8]u8)
index_select := intrinsics.simd_select(comparison, indices, sentinel)
index_reduce := cast(uint)intrinsics.simd_reduce_min(index_select)
return -1 if a[i+index_reduce] < b[i+index_reduce] else +1
}
}
for /**/; i < n; i += 1 {
if a[i] ~ b[i] != 0 {
return -1 if int(a[i]) - int(b[i]) < 0 else +1
}
}
return 0 return 0
} }
memory_compare_zero :: proc "contextless" (a: rawptr, n: int) -> int #no_bounds_check { memory_compare_zero :: proc "contextless" (a: rawptr, n: int) -> int #no_bounds_check {
x := uintptr(a) n := uint(n)
n := uintptr(n) i := uint(0)
m := uint(0)
SU :: size_of(uintptr) // Because we're comparing against zero, we never return -1, as that would
fast := n/SU + 1 // indicate the compared value is less than zero.
offset := (fast-1)*SU //
curr_block := uintptr(0) // Note that a zero return value here means equality.
if n < SU {
fast = 0
}
for /**/; curr_block < fast; curr_block += 1 { bytes := ([^]u8)(a)
va := (^uintptr)(x + curr_block * size_of(uintptr))^
if va ~ 0 != 0 { if n >= 8 {
for pos := curr_block*SU; pos < n; pos += 1 { when !SIMD_IS_EMULATED {
a := (^byte)(x+pos)^ when ODIN_ARCH == .amd64 && intrinsics.has_target_feature("avx2") {
if a ~ 0 != 0 { scanner32: #simd[32]u8
return -1 if int(a) < 0 else +1 m = n / 32 * 32
for /**/; i < m; i += 32 {
load := intrinsics.unaligned_load(cast(^#simd[32]u8)&bytes[i])
ne := intrinsics.simd_lanes_ne(scanner32, load)
if intrinsics.simd_reduce_or(ne) > 0 {
return 1
}
} }
} }
} }
scanner16: #simd[16]u8
m = (n-i) / 16 * 16
for /**/; i < m; i += 16 {
load := intrinsics.unaligned_load(cast(^#simd[16]u8)&bytes[i])
ne := intrinsics.simd_lanes_ne(scanner16, load)
if intrinsics.simd_reduce_or(ne) != 0 {
return 1
}
}
m = (n-i) / 8 * 8
for /**/; i < m; i += 8 {
if intrinsics.unaligned_load(cast(^uintptr)&bytes[i]) != 0 {
return 1
}
}
} }
for /**/; offset < n; offset += 1 { for /**/; i < n; i += 1 {
a := (^byte)(x+offset)^ if bytes[i] != 0 {
if a ~ 0 != 0 { return 1
return -1 if int(a) < 0 else +1
} }
} }
return 0 return 0
} }