ed/Odin
1
0
Fork 0
mirror of https://github.com/Ed94/Odin.git synced 2026-08-03 22:28:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify and make simd_util cross-platform

Browse Source 
This new algorithm uses a Scalar->Vector->Scalar iteration loop which
requires no masking off of any incomplete data chunks.

Also, the width was reduced to 32 bytes instead of 64, as I found this
to be about as fast as the previous 64-byte x86 version.
This commit is contained in:
Feoramund
2024-08-09 18:54:04 -04:00
parent 793811b219
commit 12dd0cb72a
5 changed files with 101 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/bytes/bytes.odin
+4 -14
View File
@@ -309,14 +309,8 @@ index_byte :: proc(s: []byte, c: byte) -> int {
// NOTE(Feoramund): On my Alder Lake CPU, I have only witnessed a // NOTE(Feoramund): On my Alder Lake CPU, I have only witnessed a
// significant speedup when compiling in either Size or Speed mode. // significant speedup when compiling in either Size or Speed mode.
// The SIMD version is usually 2-3x slower without optimizations on. // The SIMD version is usually 2-3x slower without optimizations on.
when ODIN_OPTIMIZATION_MODE > .Minimal && intrinsics.has_target_feature("sse2") { when ODIN_OPTIMIZATION_MODE > .Minimal {
// SIMD's benefits are noticeable only past a certain threshold of data. return #force_inline simd_util.index_byte(s, c)
// For small data, use the plain old algorithm.
if len(s) >= simd_util.RECOMMENDED_SCAN_SIZE {
return simd_util.index_byte(s, c)
} else {
return _index_byte(s, c)
}
} else { } else {
return _index_byte(s, c) return _index_byte(s, c)
} }
@@ -333,12 +327,8 @@ last_index_byte :: proc(s: []byte, c: byte) -> int {
return -1 return -1
} }
when ODIN_OPTIMIZATION_MODE > .Minimal && intrinsics.has_target_feature("sse2") { when ODIN_OPTIMIZATION_MODE > .Minimal {
if len(s) >= simd_util.RECOMMENDED_SCAN_SIZE { return #force_inline simd_util.last_index_byte(s, c)
return simd_util.last_index_byte(s, c)
} else {
return _last_index_byte(s, c)
}
} else { } else {
return _last_index_byte(s, c) return _last_index_byte(s, c)
} }
core/simd/util/util.odin
+93 -121
View File
@@ -8,26 +8,24 @@
// package simd_util implements compositions of SIMD operations for optimizing // package simd_util implements compositions of SIMD operations for optimizing
// the core library where available. // the core library where available.
//+build i386, amd64
package simd_util package simd_util
import "base:intrinsics" import "base:intrinsics"
import "core:simd/x86"
@private SCAN_REGISTER_SIZE :: 16 @private SCAN_WIDTH :: 32
@private SCAN_REGISTERS :: 4
@private SCAN_WIDTH :: SCAN_REGISTERS * SCAN_REGISTER_SIZE
// How long should a string be before using any of the `index_*` procedures in @(private, rodata)
// this package. simd_scanner_indices := #simd[SCAN_WIDTH]u8 {
RECOMMENDED_SCAN_SIZE :: SCAN_REGISTER_SIZE 0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31,
}
/* /*
Scan a slice of bytes for a specific byte. Scan a slice of bytes for a specific byte.
This procedure safely handles padding out slices of any length, including empty This procedure safely handles slices of any length, including empty slices.
slices.
Inputs: Inputs:
- data: A slice of bytes. - data: A slice of bytes.
@@ -36,83 +34,54 @@ Inputs:
Returns: Returns:
- index: The index of the byte `c`, or -1 if it was not found. - index: The index of the byte `c`, or -1 if it was not found.
*/ */
@(enable_target_feature="sse2")
index_byte :: proc(data: []u8, c: byte) -> (index: int) #no_bounds_check { index_byte :: proc(data: []u8, c: byte) -> (index: int) #no_bounds_check {
scanner_data: [SCAN_REGISTER_SIZE]u8 = c
scanner := intrinsics.unaligned_load(cast(^x86.__m128i)&scanner_data[0])
i: int
length := len(data) length := len(data)
full_chunks_length := length - length % SCAN_WIDTH i := 0
for /**/; i < full_chunks_length; i += SCAN_WIDTH { // Guard against small strings.
simd_load := intrinsics.unaligned_load(cast(^[SCAN_REGISTERS]x86.__m128i)&data[i]) if length < SCAN_WIDTH {
for /**/; i < length; i += 1 {
#unroll for j in 0..<SCAN_REGISTERS { if data[i] == c {
cmp := x86._mm_cmpeq_epi8(simd_load[j], scanner) return i
mask := x86._mm_movemask_epi8(cmp)
// NOTE(Feoramund): I experimented with ORing all the masks onto a
// 128-bit integer before performing the `mask != 0` check to see
// if that might be faster. However, the cost to avoid 3
// compares resulted in a marginally slower runtime on my machine.
//
// Simpler won out here.
if mask != 0 {
ctz := intrinsics.count_trailing_zeros(mask)
return i + j * SCAN_REGISTER_SIZE + cast(int)ctz
} }
} }
return -1
}
ptr := cast(int)cast(uintptr)raw_data(data)
alignment_start := (SCAN_WIDTH - ptr % SCAN_WIDTH) % SCAN_WIDTH
// Iterate as a scalar until the data is aligned on a `SCAN_WIDTH` boundary.
//
// This way, every load in the vector loop will be aligned, which should be
// the fastest possible scenario.
for /**/; i < alignment_start; i += 1 {
if data[i] == c {
return i
}
} }
if i < length { // Iterate as a vector over every aligned chunk, evaluating each byte simultaneously at the CPU level.
// The data is not exactly divisible by SCAN_WIDTH, and we haven't found scanner: #simd[SCAN_WIDTH]u8 = c
// what we're looking for yet, so we must pad out the end, then run our tail := length - (length - alignment_start) % SCAN_WIDTH
// algorithm on it.
padded_data_end: [SCAN_WIDTH]u8 = ---
remnant_length := length % SCAN_WIDTH
intrinsics.mem_copy_non_overlapping(
&padded_data_end[0],
&raw_data(data)[full_chunks_length],
remnant_length,
)
simd_load := intrinsics.unaligned_load(cast(^[SCAN_REGISTERS]x86.__m128i)&padded_data_end[0]) for /**/; i < tail; i += SCAN_WIDTH {
load := (cast(^#simd[SCAN_WIDTH]u8)(&data[i]))^
#unroll for j in 0..<SCAN_REGISTERS { comparison := intrinsics.simd_lanes_eq(load, scanner)
cmp := x86._mm_cmpeq_epi8(simd_load[j], scanner) match := intrinsics.simd_reduce_or(comparison)
mask := x86._mm_movemask_epi8(cmp) if match > 0 {
sentinel: #simd[SCAN_WIDTH]u8 = u8(0xFF)
// Because this data is padded out, it's possible that we could index_select := intrinsics.simd_select(comparison, simd_scanner_indices, sentinel)
// match on uninitialized memory, so we must guard against that. index_reduce := intrinsics.simd_reduce_min(index_select)
return i + cast(int)index_reduce
// Create a relevancy mask: (Example) }
// }
// max(u64) = 0xFFFF_FFFF_FFFF_FFFF
// // Iterate as a scalar over the remaining unaligned portion.
// Convert an integer into a stream of on-bits by using the for /**/; i < length; i += 1 {
// shifted negation of the maximum. The subtraction selects which if data[i] == c {
// section of the overall mask we should apply. return i
//
// << 17 - (1 * SCAN_REGISTER_SIZE)
// = 0xFFFF_FFFF_FFFF_FFFE
//
submask := max(u64) << u64(remnant_length - (j * SCAN_REGISTER_SIZE))
//
// ~submask = 0x0000_0000_0000_0001
// (submask >> 63) = 0x0000_0000_0000_0001
//
// The multiplication is a guard against zero.
//
submask = ~submask * (submask >> 63)
//
// Finally, mask out any irrelevant bits with the submask.
mask &= i32(submask)
if mask != 0 {
ctz := int(intrinsics.count_trailing_zeros(mask))
return i + j * SCAN_REGISTER_SIZE + ctz
}
} }
} }
@@ -123,8 +92,7 @@ index_byte :: proc(data: []u8, c: byte) -> (index: int) #no_bounds_check {
Scan a slice of bytes for a specific byte, starting from the end and working Scan a slice of bytes for a specific byte, starting from the end and working
backwards to the start. backwards to the start.
This procedure safely handles padding out slices of any length, including empty This procedure safely handles slices of any length, including empty slices.
slices.
Inputs: Inputs:
- data: A slice of bytes. - data: A slice of bytes.
@@ -133,54 +101,58 @@ Inputs:
Returns: Returns:
- index: The index of the byte `c`, or -1 if it was not found. - index: The index of the byte `c`, or -1 if it was not found.
*/ */
@(enable_target_feature="sse2")
last_index_byte :: proc(data: []u8, c: byte) -> int #no_bounds_check { last_index_byte :: proc(data: []u8, c: byte) -> int #no_bounds_check {
scanner_data: [SCAN_REGISTER_SIZE]u8 = c length := len(data)
scanner := intrinsics.unaligned_load(cast(^x86.__m128i)&scanner_data[0]) i := length - 1
i := len(data) - SCAN_WIDTH // Guard against small strings.
if length < SCAN_WIDTH {
for /**/; i >= 0; i -= SCAN_WIDTH { for /**/; i >= 0; i -= 1 {
simd_load := intrinsics.unaligned_load(cast(^[SCAN_REGISTERS]x86.__m128i)&data[i]) if data[i] == c {
return i
// There is no #reverse #unroll at the time of this writing, so we use
// `j` to count down by subtraction.
#unroll for j in 1..=SCAN_REGISTERS {
cmp := x86._mm_cmpeq_epi8(simd_load[SCAN_REGISTERS-j], scanner)
mask := x86._mm_movemask_epi8(cmp)
if mask != 0 {
// CLZ is used instead to get the on-bit from the other end.
clz := (8 * size_of(mask) - 1) - int(intrinsics.count_leading_zeros(mask))
return i + SCAN_WIDTH - j * SCAN_REGISTER_SIZE + clz
} }
} }
return -1
}
ptr := cast(int)cast(uintptr)raw_data(data)
tail := length - (ptr + length) % SCAN_WIDTH
// Iterate as a scalar until the data is aligned on a `SCAN_WIDTH` boundary.
//
// This way, every load in the vector loop will be aligned, which should be
// the fastest possible scenario.
for /**/; i >= tail; i -= 1 {
if data[i] == c {
return i
}
} }
if i < 0 { // Iterate as a vector over every aligned chunk, evaluating each byte simultaneously at the CPU level.
padded_data_end: [SCAN_WIDTH]u8 = --- scanner: #simd[SCAN_WIDTH]u8 = c
remnant_length := len(data) % SCAN_WIDTH alignment_start := (SCAN_WIDTH - ptr % SCAN_WIDTH) % SCAN_WIDTH
intrinsics.mem_copy_non_overlapping(
&padded_data_end[0],
&raw_data(data)[0],
remnant_length,
)
simd_load := intrinsics.unaligned_load(cast(^[SCAN_REGISTERS]x86.__m128i)&padded_data_end[0]) i -= SCAN_WIDTH - 1
#unroll for j in 1..=SCAN_REGISTERS { for /**/; i >= alignment_start; i -= SCAN_WIDTH {
cmp := x86._mm_cmpeq_epi8(simd_load[SCAN_REGISTERS-j], scanner) load := (cast(^#simd[SCAN_WIDTH]u8)(&data[i]))^
mask := x86._mm_movemask_epi8(cmp) comparison := intrinsics.simd_lanes_eq(load, scanner)
match := intrinsics.simd_reduce_or(comparison)
if match > 0 {
sentinel: #simd[SCAN_WIDTH]u8
index_select := intrinsics.simd_select(comparison, simd_scanner_indices, sentinel)
index_reduce := intrinsics.simd_reduce_max(index_select)
return i + cast(int)index_reduce
}
}
submask := max(u64) << u64(remnant_length - (SCAN_REGISTERS-j) * SCAN_REGISTER_SIZE) // Iterate as a scalar over the remaining unaligned portion.
submask = ~submask * (submask >> 63) i += SCAN_WIDTH - 1
mask &= i32(submask) for /**/; i >= 0; i -= 1 {
if data[i] == c {
if mask != 0 { return i
clz := (8 * size_of(mask) - 1) - int(intrinsics.count_leading_zeros(mask))
return SCAN_WIDTH - j * SCAN_REGISTER_SIZE + clz
}
} }
} }
core/strings/strings.odin
+4 -14
View File
@@ -1438,14 +1438,8 @@ index_byte :: proc(s: string, c: byte) -> (res: int) {
// NOTE(Feoramund): On my Alder Lake CPU, I have only witnessed a // NOTE(Feoramund): On my Alder Lake CPU, I have only witnessed a
// significant speedup when compiling in either Size or Speed mode. // significant speedup when compiling in either Size or Speed mode.
// The SIMD version is usually 2-3x slower without optimizations on. // The SIMD version is usually 2-3x slower without optimizations on.
when ODIN_OPTIMIZATION_MODE > .Minimal && intrinsics.has_target_feature("sse2") { when ODIN_OPTIMIZATION_MODE > .Minimal {
// SIMD's benefits are noticeable only past a certain threshold of data. return #force_inline simd_util.index_byte(transmute([]u8)s, c)
// For small data, use the plain old algorithm.
if len(s) >= simd_util.RECOMMENDED_SCAN_SIZE {
return simd_util.index_byte(transmute([]u8)s, c)
} else {
return _index_byte(s, c)
}
} else { } else {
return _index_byte(s, c) return _index_byte(s, c)
} }
@@ -1492,12 +1486,8 @@ last_index_byte :: proc(s: string, c: byte) -> (res: int) {
return -1 return -1
} }
when ODIN_OPTIMIZATION_MODE > .Minimal && intrinsics.has_target_feature("sse2") { when ODIN_OPTIMIZATION_MODE > .Minimal {
if len(s) >= simd_util.RECOMMENDED_SCAN_SIZE { return #force_inline simd_util.last_index_byte(transmute([]u8)s, c)
return simd_util.last_index_byte(transmute([]u8)s, c)
} else {
return _last_index_byte(s, c)
}
} else { } else {
return _last_index_byte(s, c) return _last_index_byte(s, c)
} }
tests/benchmark/simd/util/benchmark_simd_util.odin
-1
View File
@@ -1,4 +1,3 @@
//+build i386, amd64
package benchmark_simd_util package benchmark_simd_util
import "core:fmt" import "core:fmt"
tests/core/simd/util/test_core_simd_util.odin
-1
View File
@@ -1,4 +1,3 @@
//+build i386, amd64
package test_core_simd_util package test_core_simd_util
import simd_util "core:simd/util" import simd_util "core:simd/util"