Better str8_fmt_ktl_buf (using simd)

2026-09-14 03:39:23 +00:00 · 2026-09-08 20:45:57 -04:00
parent 749a712ceb
commit b9b1ed460a
8 changed files with 387 additions and 296 deletions
@@ -0,0 +1,127 @@
+#ifdef INTELLISENSE_DIRECTIVES
+#	pragma once
+#	include "dsl.h"
+#endif
+
+#define asm_out     /* outputs  */
+#define asm_in      /* inputs   */
+#define asm_clobber /* clobbers */
+
+#define asm_out_r(name)  [name] "=r"(name)
+#define asm_in_r(name)   [name] "r"(name)
+#define asm_out_x(name)  [name] "=&x"(name) /* XMM, earlyclobber */
+#define asm_out_x0(name) [name] "=x"(name)  /* XMM, no earlyclobber */
+#define asm_in_x(name)   [name] "x"(name)
+
+#define x64_r(name)   "%[" #name "]"
+#define x64_m(name)   "(%[" #name "])"
+#define x64_xmm(name) "%x[" #name "]"
+
+typedef U8 U8x2  attribute(vector_size(16));
+typedef U8 U8x4  attribute(vector_size(32));
+typedef U1 U1x16 attribute(vector_size(16));
+
+#define x64_u4_from_byte_hits(dst, src)        "pmovmskb "        x64_r(src) ", " x64_r(dst) "\n"
+#define x64_u4_count_trailing_zeros(dst, src)  "tzcntl "          x64_r(src) ", " x64_r(dst) "\n"
+
+#define x64_u1x16_repeat_dwords(dst, src, n)   "pshufd $" #n ", " x64_r(src) ", " x64_r(dst) "\n"
+#define x64_u1x16_load_mem4(dst, src)          "movd "            x64_r(src) ", " x64_r(dst) "\n"
+#define x64_u1x16_load_mem8(dst, src)          "movq "            x64_m(src) ", " x64_r(dst) "\n"
+#define x64_u1x16_load_mem16(dst, src)         "movdqu "          x64_m(src) ", " x64_r(dst) "\n"
+#define x64_u1x16_match_bytes(dst, src)        "pcmpeqb "         x64_r(src) ", " x64_r(dst) "\n"
+
+#define x64_u8x2_load_u8(dst, src)             "vmovq "                x64_m(src) ", " x64_r(dst) "\n"
+#define x64_u8x2_insert_u8(dst, src, n)        "vpinsrq $" #n ", "     x64_m(src) ", " x64_r(dst) ", " x64_r(dst) "\n"
+#define x64_u8x4_insert_xmm(dst, src, n)       "vinserti128 $" #n ", " x64_r(src) ", " x64_r(dst) ", " x64_r(dst) "\n"
+#define x64_u8x4_broadcast(dst, src)           "vpbroadcastq "         x64_r(src) ", " x64_r(dst) "\n"
+#define x64_u8x4_match(dst, a, b)              "vpcmpeqq "             x64_r(b)   ", " x64_r(a)   ", " x64_r(dst) "\n"
+#define x64_u4_from_qword_hits(dst, src)       "vmovmskpd "            x64_r(src) ", " x64_r(dst) "\n"
+
+I_ U4 count_trailing_zeros_u4(U4 mask) {
+	U4 n;
+	asm volatile(
+		x64_u4_count_trailing_zeros(n, mask)
+		asm_out : asm_out_r(n)
+		asm_in  : asm_in_r(mask)
+	);
+	return n;
+}
+
+I_ U1x16 splat_u4_u1x16(U4 b4) {
+	U1x16 v;
+	asm volatile(
+		x64_u1x16_load_mem4    (v, b4)
+		x64_u1x16_repeat_dwords(v, v, 0)
+		asm_out : asm_out_x0(v)
+		asm_in  : asm_in_r(b4)
+	);
+	return v;
+}
+
+I_ U4 mask_eq8_u1x16(U1_R p, U1x16 needle) {
+	U1x16 chunk;
+	U4    mask;
+	asm volatile(
+		x64_u1x16_load_mem8  (chunk, p)
+		x64_u1x16_match_bytes(chunk, needle)
+		x64_u4_from_byte_hits(mask, chunk)
+		asm_out : asm_out_x(chunk), asm_out_r(mask)
+		asm_in  : asm_in_r(p), asm_in_x(needle)
+		asm_clobber : "memory"
+	);
+	return mask; /* movq zero-fills the high 8 bytes; bits 8–15 stay 0 */
+}
+
+I_ U4 mask_eq16_u1x16(U1_R p, U1x16 needle) {
+	U1x16 chunk;
+	U4    mask;
+	asm volatile(
+		x64_u1x16_load_mem16 (chunk, p)
+		x64_u1x16_match_bytes(chunk, needle)
+		x64_u4_from_byte_hits(mask, chunk)
+		asm_out : asm_out_x(chunk), asm_out_r(mask)
+		asm_in  : asm_in_r(p), asm_in_x(needle)
+		asm_clobber : "memory"
+	);
+	return mask;
+}
+
+I_ U8 find_u1_via_u1x16(U1_R p, U8 len, U1 ch, U1x16 needle) {
+	U8 i = 0;
+	while (len - i >= 16) {
+		U4 mask = mask_eq16_u1x16(p + i, needle);
+		if (mask) return i + C_(U8, count_trailing_zeros_u4(mask));
+		i += 16;
+	}
+	while (len - i >= 8) {
+		U4 mask = mask_eq8_u1x16(p + i, needle);
+		if (mask) return i + C_(U8, count_trailing_zeros_u4(mask));
+		i += 8;
+	}
+	while (i < len) {
+		if (p[i] == ch) return i;
+		++ i;
+	}
+	return len; /* not found */
+}
+
+I_ U4 find_aos_keys_mask_u8x4(U8_R p0, U8_R p1, U8_R p2, U8_R p3, U8 key) {
+	U8x2 lo, hi;
+	U8x4 keys, splat, eq;
+	U4   mask;
+	asm volatile(
+		x64_u8x2_load_u8  (lo, p0)
+		x64_u8x2_insert_u8(lo, p1, 1)
+		x64_u8x2_load_u8  (hi, p2)
+		x64_u8x2_insert_u8(hi, p3, 1)
+		x64_u8x4_insert_xmm(keys, lo, 0)
+		x64_u8x4_insert_xmm(keys, hi, 1)
+		x64_u8x4_broadcast(splat, key)
+		x64_u8x4_match(eq, keys, splat)
+		x64_u4_from_qword_hits(mask, eq)
+		asm_out : asm_out_x(lo), asm_out_x(hi), asm_out_x(keys), asm_out_x(splat), asm_out_x(eq), asm_out_r(mask)
+		asm_in  : asm_in_r(p0), asm_in_r(p1), asm_in_r(p2), asm_in_r(p3), asm_in_r(key)
+		asm_clobber : "memory"
+	);
+	return mask;
+}