Better str8_fmt_ktl_buf (using simd)

2026-09-14 03:39:23 +00:00 · 2026-09-08 20:45:57 -04:00
parent 749a712ceb
commit b9b1ed460a
8 changed files with 387 additions and 296 deletions
@@ -0,0 +1,127 @@
+#ifdef INTELLISENSE_DIRECTIVES
+#	pragma once
+#	include "dsl.h"
+#endif
+
+#define asm_out     /* outputs  */
+#define asm_in      /* inputs   */
+#define asm_clobber /* clobbers */
+
+#define asm_out_r(name)  [name] "=r"(name)
+#define asm_in_r(name)   [name] "r"(name)
+#define asm_out_x(name)  [name] "=&x"(name) /* XMM, earlyclobber */
+#define asm_out_x0(name) [name] "=x"(name)  /* XMM, no earlyclobber */
+#define asm_in_x(name)   [name] "x"(name)
+
+#define x64_r(name)   "%[" #name "]"
+#define x64_m(name)   "(%[" #name "])"
+#define x64_xmm(name) "%x[" #name "]"
+
+typedef U8 U8x2  attribute(vector_size(16));
+typedef U8 U8x4  attribute(vector_size(32));
+typedef U1 U1x16 attribute(vector_size(16));
+
+#define x64_u4_from_byte_hits(dst, src)        "pmovmskb "        x64_r(src) ", " x64_r(dst) "\n"
+#define x64_u4_count_trailing_zeros(dst, src)  "tzcntl "          x64_r(src) ", " x64_r(dst) "\n"
+
+#define x64_u1x16_repeat_dwords(dst, src, n)   "pshufd $" #n ", " x64_r(src) ", " x64_r(dst) "\n"
+#define x64_u1x16_load_mem4(dst, src)          "movd "            x64_r(src) ", " x64_r(dst) "\n"
+#define x64_u1x16_load_mem8(dst, src)          "movq "            x64_m(src) ", " x64_r(dst) "\n"
+#define x64_u1x16_load_mem16(dst, src)         "movdqu "          x64_m(src) ", " x64_r(dst) "\n"
+#define x64_u1x16_match_bytes(dst, src)        "pcmpeqb "         x64_r(src) ", " x64_r(dst) "\n"
+
+#define x64_u8x2_load_u8(dst, src)             "vmovq "                x64_m(src) ", " x64_r(dst) "\n"
+#define x64_u8x2_insert_u8(dst, src, n)        "vpinsrq $" #n ", "     x64_m(src) ", " x64_r(dst) ", " x64_r(dst) "\n"
+#define x64_u8x4_insert_xmm(dst, src, n)       "vinserti128 $" #n ", " x64_r(src) ", " x64_r(dst) ", " x64_r(dst) "\n"
+#define x64_u8x4_broadcast(dst, src)           "vpbroadcastq "         x64_r(src) ", " x64_r(dst) "\n"
+#define x64_u8x4_match(dst, a, b)              "vpcmpeqq "             x64_r(b)   ", " x64_r(a)   ", " x64_r(dst) "\n"
+#define x64_u4_from_qword_hits(dst, src)       "vmovmskpd "            x64_r(src) ", " x64_r(dst) "\n"
+
+I_ U4 count_trailing_zeros_u4(U4 mask) {
+	U4 n;
+	asm volatile(
+		x64_u4_count_trailing_zeros(n, mask)
+		asm_out : asm_out_r(n)
+		asm_in  : asm_in_r(mask)
+	);
+	return n;
+}
+
+I_ U1x16 splat_u4_u1x16(U4 b4) {
+	U1x16 v;
+	asm volatile(
+		x64_u1x16_load_mem4    (v, b4)
+		x64_u1x16_repeat_dwords(v, v, 0)
+		asm_out : asm_out_x0(v)
+		asm_in  : asm_in_r(b4)
+	);
+	return v;
+}
+
+I_ U4 mask_eq8_u1x16(U1_R p, U1x16 needle) {
+	U1x16 chunk;
+	U4    mask;
+	asm volatile(
+		x64_u1x16_load_mem8  (chunk, p)
+		x64_u1x16_match_bytes(chunk, needle)
+		x64_u4_from_byte_hits(mask, chunk)
+		asm_out : asm_out_x(chunk), asm_out_r(mask)
+		asm_in  : asm_in_r(p), asm_in_x(needle)
+		asm_clobber : "memory"
+	);
+	return mask; /* movq zero-fills the high 8 bytes; bits 8–15 stay 0 */
+}
+
+I_ U4 mask_eq16_u1x16(U1_R p, U1x16 needle) {
+	U1x16 chunk;
+	U4    mask;
+	asm volatile(
+		x64_u1x16_load_mem16 (chunk, p)
+		x64_u1x16_match_bytes(chunk, needle)
+		x64_u4_from_byte_hits(mask, chunk)
+		asm_out : asm_out_x(chunk), asm_out_r(mask)
+		asm_in  : asm_in_r(p), asm_in_x(needle)
+		asm_clobber : "memory"
+	);
+	return mask;
+}
+
+I_ U8 find_u1_via_u1x16(U1_R p, U8 len, U1 ch, U1x16 needle) {
+	U8 i = 0;
+	while (len - i >= 16) {
+		U4 mask = mask_eq16_u1x16(p + i, needle);
+		if (mask) return i + C_(U8, count_trailing_zeros_u4(mask));
+		i += 16;
+	}
+	while (len - i >= 8) {
+		U4 mask = mask_eq8_u1x16(p + i, needle);
+		if (mask) return i + C_(U8, count_trailing_zeros_u4(mask));
+		i += 8;
+	}
+	while (i < len) {
+		if (p[i] == ch) return i;
+		++ i;
+	}
+	return len; /* not found */
+}
+
+I_ U4 find_aos_keys_mask_u8x4(U8_R p0, U8_R p1, U8_R p2, U8_R p3, U8 key) {
+	U8x2 lo, hi;
+	U8x4 keys, splat, eq;
+	U4   mask;
+	asm volatile(
+		x64_u8x2_load_u8  (lo, p0)
+		x64_u8x2_insert_u8(lo, p1, 1)
+		x64_u8x2_load_u8  (hi, p2)
+		x64_u8x2_insert_u8(hi, p3, 1)
+		x64_u8x4_insert_xmm(keys, lo, 0)
+		x64_u8x4_insert_xmm(keys, hi, 1)
+		x64_u8x4_broadcast(splat, key)
+		x64_u8x4_match(eq, keys, splat)
+		x64_u4_from_qword_hits(mask, eq)
+		asm_out : asm_out_x(lo), asm_out_x(hi), asm_out_x(keys), asm_out_x(splat), asm_out_x(eq), asm_out_r(mask)
+		asm_in  : asm_in_r(p0), asm_in_r(p1), asm_in_r(p2), asm_in_r(p3), asm_in_r(key)
+		asm_clobber : "memory"
+	);
+	return mask;
+}
@@ -54,16 +54,19 @@ Standard: c23
 #define LP_      static // static data within procedure scope
 #define internal static // internal

+
+#define attribute(directive) __attribute__((directive))
+
 #define asm           __asm__

-#define align_(value) __attribute__((aligned (value)))             // for easy alignment
-#define C_(type,data) ((type)(data))                               // for enforced precedence
-#define expect_(x, y) __builtin_expect(x, y)                       // so compiler knows the common path
+#define align_(value) attribute(aligned(value))              // for easy alignment
+#define C_(type,data) ((type)(data))                         // for enforced precedence
+#define expect_(x, y) __builtin_expect(x, y)                 // so compiler knows the common path
 #define cexpr_        __builtin_constant_p
 #define I_            internal inline
-#define FI_           inline   __attribute__((always_inline))      // inline always
-#define NI_           internal __attribute__((noinline))           // inline never
-#define RO_           __attribute__((section(".rodata")))          // Read only data allocation
+#define FI_           inline   attribute(always_inline)      // inline always
+#define NI_           internal attribute(noinline)           // inline never
+#define RO_           attribute(section(".rodata"))          // Read only data allocation
 #define T_            typeof                                       // 
 #define T_same(a,b)   _Generic((a), typeof((b)): 1, default: 0)

@@ -262,6 +265,10 @@ FI_ U8 atm_swap_u8(U8_R addr, U8 value){asm volatile("lock xchgq %0,%1":"=r"(val
 #pragma endregion Thread Coherence

 #pragma region Misc
+#define byte_pos(pos)         (pos * 8)
+#define byte_shift(value,pos) (value << byte_pos(pos))
+#define u4_byte_fill(value)   byte_shift(value,0) | byte_shift(value,1) | byte_shift(value,2) | byte_shift(value,3)
+
 enum {
 	Bitmask_3  = 0x00000007,
 	Bitmask_4  = 0x0000000f,
@@ -270,32 +277,8 @@ enum {
 	Bitmask_10 = 0x000003ff,
 };

-typedef Enum_(U4, WeekDay) {
-	WeekDay_Sun,
-	WeekDay_Mon,
-	WeekDay_Tue,
-	WeekDay_Wed,
-	WeekDay_Thu,
-	WeekDay_Fri,
-	WeekDay_Sat,
-	WeekDay_Num,
-};
-
-typedef Enum_(U4, Month) {
-	Month_Jan,
-	Month_Feb,
-	Month_Mar,
-	Month_Apr,
-	Month_May,
-	Month_Jun,
-	Month_Jul,
-	Month_Aug,
-	Month_Sep,
-	Month_Oct,
-	Month_Nov,
-	Month_Dec,
-	Month_Num,
-};
+typedef Enum_(U4,WeekDay){ WeekDay_Sun, WeekDay_Mon, WeekDay_Tue, WeekDay_Wed, WeekDay_Thu, WeekDay_Fri, WeekDay_Sat, WeekDay_Num, };
+typedef Enum_(U4,Month)  { Month_Jan, Month_Feb, Month_Mar, Month_Apr, Month_May, Month_Jun, Month_Jul, Month_Aug, Month_Sep, Month_Oct, Month_Nov, Month_Dec, Month_Num, };

 typedef U8 DenseTime;

@@ -39,8 +39,17 @@ FI_ B4 mem_match  (U8 a, U8 b, U8 z)   { return mem_compare(a, b, z) == 0; }
 #define mem_match_struct(a,b)  mem_match(C_(U8,a), C_(U8,b), S_((a)[0]))
 #define mem_zero_struct(s) mem_zero(u8_(& s), S_(s))

-#pragma region DAG
+I_ void mem_push_aligned_typed(U8 start, U8 capacity, U8_R used, U8 amount, U4 alignment, U4 type_width, U8_R out_ptr, U8_R out_len) {
+	if (amount == 0) { out_ptr[0] = 0; out_len[0] = 0; return;  }
+	U8 desired   = amount * (type_width == 0 ? 1 : type_width);
+	U8 to_commit = align_pow2(desired, alignment ?  alignment : MEM_ALIGNMENT_DEFAULT);
+	U8 ptr       = start + used[0];
+	mem_bump_u8(start, capacity, used, to_commit);
+	out_ptr[0] = ptr;
+	out_len[0] = to_commit;
+}

+#pragma region DAG
 #define check_nil(nil, p) ((p) == 0 || (p) == nil)
 #define set_nil(nil, p)   ((p) = nil)

@@ -59,11 +68,9 @@ FI_ B4 mem_match  (U8 a, U8 b, U8 z)   { return mem_compare(a, b, z) == 0; }
 	)                         \
 )
 #define sll_queue_push_n(f, l, n, next) sll_queue_push_nz(0, f, l, n, next)
-
 #pragma endregion DAG

 #pragma region Slice
-
 typedef unsigned char TSet_(UTF8);
 typedef Struct_(Str8)         { UTF8* ptr; U8 len; }; typedef Str8 Slice_UTF8;
 typedef Struct_(Slice_Str8)   { Str8* ptr; U8 len; };
@@ -104,20 +111,9 @@ typedef Slice_(U1);
 typedef Slice_(U2);
 typedef Slice_(U4);
 typedef Slice_(U8); 
-
 #pragma endregion Slice

-I_ Slice mem_push_aligned_typed(U8 start, U8 capacity, U8_R used, U8 amount, U4 alignment, U4 type_width) {
-	if (amount == 0) { return (Slice){}; }
-	U8 desired   = amount * (type_width == 0 ? 1 : type_width);
-	U8 to_commit = align_pow2(desired, alignment ?  alignment : MEM_ALIGNMENT_DEFAULT);
-	U8 ptr       = start + used[0];
-	mem_bump_u8(start, capacity, used, to_commit);
-	return (Slice){ ptr, to_commit };
-}
-
 #pragma region FArena
-
 typedef Opt_(farena)    { U8 alignment, type_width; };
 typedef Struct_(FArena) { U8 start, capacity, used; };
 FI_ void farena_init(FArena_R arena, Slice mem) {  assert(arena != nullptr);
@@ -126,8 +122,10 @@ FI_ void farena_init(FArena_R arena, Slice mem) {  assert(arena != nullptr);
 	arena->used     = 0;
 }
 FI_ FArena farena_make(Slice mem) { FArena a; farena_init(& a, mem); return a; }
-FI_ Slice  farena_push(FArena_R arena, U8 amount, Opt_farena o) {
-	return mem_push_aligned_typed(arena->start, arena->capacity, & arena->used, amount, o.alignment, o.type_width);
+FI_ Slice  farena_push(FArena_R arena, U8 amount, Opt_farena o) { Slice res; 
+	mem_push_aligned_typed(arena->start, arena->capacity, & arena->used, amount, o.alignment, o.type_width
+		, & res.ptr, & res.len);
+	return res;
 }
 FI_ void farena_reset (FArena_R arena) { arena->used = 0; }
 FI_ void farena_rewind(FArena_R arena, U8 save_point) {
@@ -138,14 +136,15 @@ FI_ U8 farena_save(FArena arena) { return arena.used; }
 #define farena_push_(arena, amount, ...)                                         farena_push((arena),(amount),opt_(farena,__VA_ARGS__))
 #define farena_push_type(arena, type, ...)                              C_(type*,farena_push((arena),1,       opt_(farena,.type_width=S_(type),__VA_ARGS__)).ptr)
 #define farena_push_array(arena, type, amount, ...) (tmpl(Slice,type)){ C_(type*,farena_push((arena),(amount),opt_(farena,.type_width=S_(type),__VA_ARGS__)).ptr),(amount) }
-
 #pragma endregion FArena

 #pragma region FStack
 #define FStack_(name, type, width) Struct_(name) { U8 top; type arr[width]; }

-FI_ Slice fstack_push(Slice mem, U8_R top, U8 amount, Opt_farena o) { 
-	return mem_push_aligned_typed(mem.ptr, mem.len, top, amount, o.alignment, o.type_width);
+FI_ Slice fstack_push(Slice mem, U8_R top, U8 amount, Opt_farena o) { Slice res; 
+	mem_push_aligned_typed(mem.ptr, mem.len, top, amount, o.alignment, o.type_width
+		, & res.ptr, & res.len);
+	return res;
 };

 // This is here more for annotation than anything else.
@@ -1,13 +1,13 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #	pragma once
 #	include "dsl.h"
+#	include "asm.h"
 #	include "memory.h"
 #	include "hashing.h"
 #	include "analysis.h"
 #endif

 #pragma region Key Table Linear (KTL)
-
 enum { KT_Slot_value = S_(U8), };
 #define KTL_Slot_(type) Struct_(tmpl(KTL_Slot,type)) { \
 	U8   key;   \
@@ -37,4 +37,19 @@ FI_ void ktl_populate_slice_a2_str8(KTL_Str8* kt, Slice_A2_Str8 values) {
 #define ktl_str8_key(str)      hash64_fnv1a_ret(slice_to_ut(slit8(str)), 0)
 #define ktl_str8_from_arr(arr) (KTL_Str8){arr, Array_len(arr)}

+FI_ Str8_R ktl_str8_find(KTL_Str8 table, U8 key) {
+	U8 i = 0;
+	while (table.len - i >= 4) {
+		KTL_Slot_Str8_R p = table.ptr + i;
+		U4 mask = find_aos_keys_mask_u8x4(& p[0].key, & p[1].key, & p[2].key, & p[3].key, key);
+		if (mask) return & table.ptr[i + C_(U8, count_trailing_zeros_u4(mask))].value;
+		i += 4;
+	}
+	while (i < table.len) {
+		if (table.ptr[i].key == key) return & table.ptr[i].value;
+		++ i;
+	}
+	assert(false);
+	return nullptr;
+}
 #pragma endregion KTL
@@ -6,6 +6,7 @@
 #	include "hashing.h"
 #	include "tables.h"
 #	include "analysis.h"
+#	include "asm.h"
 #endif

 // NOTE(rjf): Includes reverses for uppercase and lowercase hex.
@@ -123,8 +124,7 @@ I_ Str8 str8_from_u4_buf(Slice buf, U4 num, U4 radix, U4 min_digits, U4 digit_gr
 	return result;
 }

-I_ Str8 str8_fmt_ktl_buf(Slice buffer, KTL_Str8 table, Str8 fmt_template)
-{
+I_ Str8 str8_fmt_ktl_buf(Slice buffer, KTL_Str8 table, Str8 fmt_template){
 	slice_assert(buffer);
 	slice_assert(table);
 	slice_assert(fmt_template);
@@ -132,68 +132,41 @@ I_ Str8 str8_fmt_ktl_buf(Slice buffer, KTL_Str8 table, Str8 fmt_template)
 	U8     buffer_remaining = buffer.len;
 	UTF8_R cursor_fmt       = fmt_template.ptr;
 	U8     left_fmt         = fmt_template.len;
+	U1x16  needle_lt        = splat_u4_u1x16(u4_byte_fill('<'));
+	U1x16  needle_gt        = splat_u4_u1x16(u4_byte_fill('>'));
 	while (left_fmt && buffer_remaining)
 	{
 		// Forward until we hit the delimiter '<' or the template's contents are exhausted.
 		U8 copy_offset = 0;
-		if (cursor_fmt[0] == '<')
-		{
-			UTF8_R potential_token_cursor = cursor_fmt + 1; // Skip '<'
-			U8     potential_token_len    = 0;
-			B4     fmt_overflow           = false;
-			while(true) {
-				UTF8_R cursor       = potential_token_cursor + potential_token_len;
-				fmt_overflow        = cursor >= slice_end(fmt_template);
-				B4 found_terminator = potential_token_cursor[potential_token_len] == '>';
-				if (fmt_overflow || found_terminator) { break; }
-				++ potential_token_len;
-			}
-			if (fmt_overflow) { 
-				// Failed to find a subst and we're at end of fmt, just copy segment.
-				copy_offset = 1 + potential_token_len; // '<' + token
-				goto write_to_buffer; 
-			}
-			// Hashing the potential token and cross checking it with our token table
-			U8 key = hash64_fnv1a_ret(slice_ut(u8_(potential_token_cursor), potential_token_len), 0);
-			Str8_R value = nullptr; for slice_iter(table, token) {
-				// We do a linear iteration instead of a hash table lookup because the user should never subst with more than 32-128 unqiue tokens..
-				if (token->key == key) { value = & token->value; break; }
-			}
-			if (value)
-			{
-				// We're going to appending the string, make sure we have enough space in our buffer.
-				// NOTE(Ed): this version doesn't support growing the buffer (No Allocator Interface)
-				copy_offset = min(buffer_remaining, value->len); // Prevent Buffer overflow.
-				assert((buffer_remaining - copy_offset) > 0);
-				mem_copy(u8_(cursor_buffer), u8_(value->ptr), copy_offset);
-				// Sync cursor format to after the processed token
-				cursor_buffer    += copy_offset;
-				buffer_remaining -= copy_offset;
-				cursor_fmt        = potential_token_cursor + 1 + potential_token_len; // '<' + token
-				left_fmt         -= potential_token_len    + 2; // The 2 here are the '<' & '>' delimiters being omitted.
-				continue;
-			}
-			// If not a subsitution, we copy the segment and continue.
-			copy_offset = 1 + potential_token_len; // '<' + token
-			goto write_to_buffer;
+		if (cursor_fmt[0] == '<') {
+			UTF8_R sig     = cursor_fmt + 1;
+			U8     sig_max = slice_end(fmt_template) - sig;
+			U8     sig_len = find_u1_via_u1x16(sig, sig_max, '>', needle_gt);
+			assert(sig_len < sig_max);
+
+			Str8_R value = ktl_str8_find(table, hash64_fnv1a_ret(slice_ut(sig, sig_len), 0));
+			U8     n     = min(buffer_remaining, value->len); 
+			assert((buffer_remaining - n) > 0); mem_copy(u8_(cursor_buffer), u8_(value->ptr), n);
+			cursor_buffer    += n;
+			buffer_remaining -= n;
+			cursor_fmt        = sig + sig_len + 1;
+			left_fmt         -= sig_len + 2;
+			continue;
 		}
-		else do {
-			++ copy_offset;
-		} 
-		while ( (cursor_fmt[copy_offset] != '<' && (cursor_fmt + copy_offset) < slice_end(fmt_template)) );
-	write_to_buffer:
-		assert((buffer_remaining - copy_offset) > 0);
-		copy_offset = min(buffer_remaining, copy_offset); // Prevent buffer overflow.
-		mem_copy(u8_(cursor_buffer), u8_(cursor_fmt), copy_offset);
-		buffer_remaining -= copy_offset;
-		left_fmt         -= copy_offset;
-		cursor_buffer    += copy_offset;
-		cursor_fmt       += copy_offset;
+		U8 n = find_u1_via_u1x16(cursor_fmt, min(left_fmt, buffer_remaining), '<', needle_lt);
+		assert((buffer_remaining - 1) > 0);  n = min(buffer_remaining, n);
+		mem_copy(u8_(cursor_buffer), u8_(cursor_fmt), n);
+		cursor_buffer    += n;
+		cursor_fmt       += n;
+		buffer_remaining -= n;
+		left_fmt         -= n;
 	}
-	return (Str8){C_(UTF8*, buffer.ptr), buffer.len - buffer_remaining};
+	return str8(C_(UTF8*,buffer.ptr), buffer.len - buffer_remaining);
 }

 typedef Struct_(Str8Gen) { UTF8* ptr; U8 cap, len; };
+FI_ Str8Gen str8gen_make(Slice s) { return (Str8Gen){C_(UTF8*,s.ptr), s.len, 0}; }
+
 FI_ Slice str8gen_buf(Str8Gen_R gen) { return (Slice){u8_(gen->ptr) + gen->len, gen->cap - gen->len}; }

 FI_ void str8gen_append_str8(Str8Gen_R gen, Str8 str) { assert(gen != nullptr);
@@ -224,8 +197,7 @@ str16_from_8(FArena* arena, Str8 in) {
    U1* opl = ptr + in.len;
    U8 size = 0;
    UnicodeDecode consume;
-    for(;ptr < opl; ptr += consume.inc)
-    {
+    for(;ptr < opl; ptr += consume.inc) {
      consume = utf8_decode(ptr, opl - ptr);
      size   += utf16_encode(str.ptr + size, consume.codepoint);
    }