From 7e701d1677fbe594e0970824062d8b3564d33d26 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 5 Aug 2024 13:46:24 +0100 Subject: [PATCH] Add `intrinsics.simd_gather` and ``intrinsics.simd_scatter` --- base/intrinsics/intrinsics.odin | 4 +++ core/simd/simd.odin | 5 +++ src/check_builtin.cpp | 54 +++++++++++++++++++++++++++++++++ src/checker_builtin_procs.hpp | 6 ++++ src/llvm_backend_proc.cpp | 46 ++++++++++++++++++++++++++++ 5 files changed, 115 insertions(+) diff --git a/base/intrinsics/intrinsics.odin b/base/intrinsics/intrinsics.odin index fadbda3fb..5566c8c6c 100644 --- a/base/intrinsics/intrinsics.odin +++ b/base/intrinsics/intrinsics.odin @@ -283,6 +283,10 @@ simd_reduce_any :: proc(a: #simd[N]T) -> T where type_is_boolean(T) --- simd_reduce_all :: proc(a: #simd[N]T) -> T where type_is_boolean(T) --- +simd_gather :: proc(ptr: #simd[N]rawptr, val: #simd[N]T, mask: #simd[N]U) -> #simd[N]T where type_is_integer(U) || type_is_boolean(U) --- +simd_scatter :: proc(ptr: #simd[N]rawptr, val: #simd[N]T, mask: #simd[N]U) where type_is_integer(U) || type_is_boolean(U) --- + + simd_shuffle :: proc(a, b: #simd[N]T, indices: ..int) -> #simd[len(indices)]T --- simd_select :: proc(cond: #simd[N]boolean_or_integer, true, false: #simd[N]T) -> #simd[N]T --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 46f6a0112..2fc0bc2c0 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -102,6 +102,11 @@ lanes_le :: intrinsics.simd_lanes_le lanes_gt :: intrinsics.simd_lanes_gt lanes_ge :: intrinsics.simd_lanes_ge + +// Gather and Scatter intrinsics +gather :: intrinsics.simd_gather +scatter :: intrinsics.simd_scatter + // extract :: proc(a: #simd[N]T, idx: uint) -> T extract :: intrinsics.simd_extract // replace :: proc(a: #simd[N]T, idx: uint, elem: T) -> #simd[N]T diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index b4967a56a..99a989b4f 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -663,6 +663,60 @@ gb_internal bool check_builtin_simd_operation(CheckerContext *c, Operand *operan return true; } + case BuiltinProc_simd_gather: + case BuiltinProc_simd_scatter: + { + // gather (ptr: #simd[N]rawptr, values: #simd[N]T, mask: #simd[N]int_or_bool) -> #simd[N]T + // scatter(ptr: #simd[N]rawptr, values: #simd[N]T, mask: #simd[N]int_or_bool) + + Operand ptr = {}; + Operand values = {}; + Operand mask = {}; + check_expr(c, &ptr, ce->args[0]); if (ptr.mode == Addressing_Invalid) return false; + check_expr(c, &values, ce->args[1]); if (values.mode == Addressing_Invalid) return false; + check_expr(c, &mask, ce->args[2]); if (mask.mode == Addressing_Invalid) return false; + if (!is_type_simd_vector(ptr.type)) { error(ptr.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; } + if (!is_type_simd_vector(values.type)) { error(values.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; } + if (!is_type_simd_vector(mask.type)) { error(mask.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; } + + Type *ptr_elem = base_array_type(ptr.type); + if (!is_type_rawptr(ptr_elem)) { + gbString s = type_to_string(ptr.type); + error(ptr.expr, "Expected a simd vector of 'rawptr' for the addresses, got %s", s); + gb_string_free(s); + return false; + } + Type *mask_elem = base_array_type(mask.type); + + if (!is_type_integer(mask_elem) && !is_type_boolean(mask_elem)) { + gbString s = type_to_string(mask.type); + error(mask.expr, "Expected a simd vector of integers or booleans for the mask, got %s", s); + gb_string_free(s); + return false; + } + + i64 ptr_count = get_array_type_count(ptr.type); + i64 values_count = get_array_type_count(values.type); + i64 mask_count = get_array_type_count(mask.type); + if (ptr_count != values_count || + values_count != mask_count || + mask_count != ptr_count) { + gbString s = type_to_string(mask.type); + error(mask.expr, "All simd vectors must be of the same length, got %lld vs %lld vs %lld", cast(long long)ptr_count, cast(long long)values_count, cast(long long)mask_count); + gb_string_free(s); + return false; + } + + if (id == BuiltinProc_simd_gather) { + operand->mode = Addressing_Value; + operand->type = values.type; + } else { + operand->mode = Addressing_NoValue; + operand->type = nullptr; + } + return true; + } + case BuiltinProc_simd_extract: { Operand x = {}; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 7fd71c36a..826c10e10 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -191,6 +191,9 @@ BuiltinProc__simd_begin, BuiltinProc_simd_lanes_rotate_left, BuiltinProc_simd_lanes_rotate_right, + BuiltinProc_simd_gather, + BuiltinProc_simd_scatter, + // Platform specific SIMD intrinsics BuiltinProc_simd_x86__MM_SHUFFLE, @@ -522,6 +525,9 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_lanes_rotate_left"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_lanes_rotate_right"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_gather"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_scatter"), 3, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_x86__MM_SHUFFLE"), 4, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 64db2ad36..5ccbd3399 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1688,6 +1688,52 @@ gb_internal lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAn return res; } + + case BuiltinProc_simd_gather: + case BuiltinProc_simd_scatter: + { + LLVMValueRef ptr = arg0.value; + LLVMValueRef val = arg1.value; + LLVMValueRef mask = arg2.value; + + unsigned count = cast(unsigned)get_array_type_count(arg0.type); + + LLVMTypeRef mask_type = LLVMVectorType(LLVMInt1TypeInContext(p->module->ctx), count); + mask = LLVMBuildTrunc(p->builder, mask, mask_type, ""); + + char const *name = nullptr; + switch (builtin_id) { + case BuiltinProc_simd_gather: name = "llvm.masked.gather"; break; + case BuiltinProc_simd_scatter: name = "llvm.masked.scatter"; break; + } + LLVMTypeRef types[2] = { + lb_type(p->module, arg1.type), + lb_type(p->module, arg0.type) + }; + + auto alignment = cast(unsigned long long)type_align_of(base_array_type(arg1.type)); + LLVMValueRef align = LLVMConstInt(LLVMInt32TypeInContext(p->module->ctx), alignment, false); + + LLVMValueRef args[4] = {}; + switch (builtin_id) { + case BuiltinProc_simd_gather: + args[0] = ptr; + args[1] = align; + args[2] = mask; + args[3] = val; + break; + case BuiltinProc_simd_scatter: + args[0] = val; + args[1] = ptr; + args[2] = align; + args[3] = mask; + break; + } + + res.value = lb_call_intrinsic(p, name, args, gb_count_of(args), types, gb_count_of(types)); + return res; + + } } GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[builtin_id].name));