From b032d5af87ebe8d9dee28698cfa570d3628e58e5 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 17:26:18 +0100 Subject: [PATCH 01/86] Make `#simd` an opaque type --- src/check_decl.cpp | 20 +++++++++++++------- src/check_expr.cpp | 24 ++++++++---------------- src/check_type.cpp | 5 +++++ src/common.cpp | 7 +++++++ src/types.cpp | 2 +- 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/check_decl.cpp b/src/check_decl.cpp index 82ac6c677..d8cad2ce1 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -313,13 +313,19 @@ void check_type_decl(CheckerContext *ctx, Entity *e, Ast *init_expr, Type *def) } named->Named.base = base; - if (is_distinct && is_type_typeid(e->type)) { - error(init_expr, "'distinct' cannot be applied to 'typeid'"); - is_distinct = false; - } - if (is_distinct && is_type_any(e->type)) { - error(init_expr, "'distinct' cannot be applied to 'any'"); - is_distinct = false; + if (is_distinct) { + if (is_type_typeid(e->type)) { + error(init_expr, "'distinct' cannot be applied to 'typeid'"); + is_distinct = false; + } else if (is_type_any(e->type)) { + error(init_expr, "'distinct' cannot be applied to 'any'"); + is_distinct = false; + } else if (is_type_simd_vector(e->type)) { + gbString str = type_to_string(e->type); + error(init_expr, "'distinct' cannot be applied to '%s'", str); + gb_string_free(str); + is_distinct = false; + } } if (!is_distinct) { e->type = bt; diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 7b269e048..a4dfade98 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -1567,9 +1567,16 @@ bool check_unary_op(CheckerContext *c, Operand *o, Token op) { bool check_binary_op(CheckerContext *c, Operand *o, Token op) { Type *main_type = o->type; + + if (is_type_simd_vector(main_type)) { + error(op, "Operator '%.*s' is not supported on #simd vector types, please use the intrinsics.simd_*", LIT(op.string)); + return false; + } + // TODO(bill): Handle errors correctly Type *type = base_type(core_array_type(main_type)); Type *ct = core_type(type); + switch (op.kind) { case Token_Sub: case Token_SubEq: @@ -1638,14 +1645,6 @@ bool check_binary_op(CheckerContext *c, Operand *o, Token op) { error(op, "Operator '%.*s' is only allowed with integers", LIT(op.string)); return false; } - if (is_type_simd_vector(o->type)) { - switch (op.kind) { - case Token_ModMod: - case Token_ModModEq: - error(op, "Operator '%.*s' is only allowed with integers", LIT(op.string)); - return false; - } - } break; case Token_AndNot: @@ -1654,14 +1653,6 @@ bool check_binary_op(CheckerContext *c, Operand *o, Token op) { error(op, "Operator '%.*s' is only allowed with integers and bit sets", LIT(op.string)); return false; } - if (is_type_simd_vector(o->type)) { - switch (op.kind) { - case Token_AndNot: - case Token_AndNotEq: - error(op, "Operator '%.*s' is only allowed with integers", LIT(op.string)); - return false; - } - } break; case Token_CmpAnd: @@ -7738,6 +7729,7 @@ ExprKind check_compound_literal(CheckerContext *c, Operand *o, Ast *node, Type * } if (cl->elems.count > 0 && cl->elems[0]->kind == Ast_FieldValue) { + // TODO(bill): Why was this decision made for simd? if (is_type_simd_vector(t)) { error(cl->elems[0], "'field = value' is not allowed for SIMD vector literals"); } else { diff --git a/src/check_type.cpp b/src/check_type.cpp index 51f472961..193c42cde 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2802,6 +2802,11 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t *type = alloc_type_array(elem, count, generic_type); goto array_end; } + if (count < 1 || !is_power_of_two(count)) { + error(at->elem, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); + *type = alloc_type_array(elem, count, generic_type); + goto array_end; + } *type = alloc_type_simd_vector(count, elem); } else { diff --git a/src/common.cpp b/src/common.cpp index 94248fb62..77caddfe8 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -47,6 +47,13 @@ void debugf(char const *fmt, ...); #include "range_cache.cpp" +bool is_power_of_two(i64 x) { + if (x <= 0) { + return false; + } + return !(x & (x-1)); +} + int isize_cmp(isize x, isize y) { if (x < y) { return -1; diff --git a/src/types.cpp b/src/types.cpp index c79b8e652..d5ba1a531 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -3446,7 +3446,7 @@ i64 type_align_of_internal(Type *t, TypePath *path) { case Type_SimdVector: { // IMPORTANT TODO(bill): Figure out the alignment of vector types - return gb_clamp(next_pow2(type_size_of_internal(t, path)), 1, build_context.max_align); + return gb_clamp(next_pow2(type_size_of_internal(t, path)), 1, build_context.max_align*2); } case Type_Matrix: From 3b54015e80316af8c13fd83f615b64b611508275 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 17:54:05 +0100 Subject: [PATCH 02/86] Mock out simd intrinsics --- src/check_builtin.cpp | 209 +++++++++++++++++++++++++++++++++- src/check_type.cpp | 2 +- src/checker_builtin_procs.hpp | 57 +++++++++- src/types.cpp | 2 +- 4 files changed, 265 insertions(+), 5 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 55dd6b016..939892707 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -246,7 +246,7 @@ bool is_constant_string(CheckerContext *c, String const &builtin_name, Ast *expr } bool check_builtin_objc_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 id, Type *type_hint) { - String builtin_name = builtin_procs[id].name; + String const &builtin_name = builtin_procs[id].name; if (build_context.metrics.os != TargetOs_darwin) { // allow on doc generation (e.g. Metal stuff) @@ -409,6 +409,194 @@ bool check_atomic_memory_order_argument(CheckerContext *c, Ast *expr, String con } + +bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call, i32 id, Type *type_hint) { + ast_node(ce, CallExpr, call); + + String const &builtin_name = builtin_procs[id].name; + switch (id) { + // Any numeric + case BuiltinProc_simd_add: + case BuiltinProc_simd_sub: + case BuiltinProc_simd_mul: + case BuiltinProc_simd_div: + case BuiltinProc_simd_min: + case BuiltinProc_simd_max: + { + Operand x = {}; + Operand y = {}; + check_expr(c, &x, ce->args[0]); + check_expr(c, &y, ce->args[1]); + if (x.mode == Addressing_Invalid) { + return false; + } + if (y.mode == Addressing_Invalid) { + return false; + } + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!is_type_simd_vector(y.type)) { + error(y.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!are_types_identical(x.type, y.type)) { + gbString xs = type_to_string(x.type); + gbString ys = type_to_string(y.type); + error(x.expr, "'%.*s' expected 2 arguments of the same type, got '%s' vs '%s'", LIT(builtin_name), xs, ys); + gb_string_free(ys); + gb_string_free(xs); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem) && !is_type_float(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + + operand->mode = Addressing_Value; + operand->type = x.type; + return true; + } + + // Integer only + case BuiltinProc_simd_rem: + case BuiltinProc_simd_shl: + case BuiltinProc_simd_shr: + case BuiltinProc_simd_shl_masked: + case BuiltinProc_simd_shr_masked: + case BuiltinProc_simd_and: + case BuiltinProc_simd_or: + case BuiltinProc_simd_xor: + { + Operand x = {}; + Operand y = {}; + check_expr(c, &x, ce->args[0]); + check_expr(c, &y, ce->args[1]); + if (x.mode == Addressing_Invalid) { + return false; + } + if (y.mode == Addressing_Invalid) { + return false; + } + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!is_type_simd_vector(y.type)) { + error(y.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!are_types_identical(x.type, y.type)) { + gbString xs = type_to_string(x.type); + gbString ys = type_to_string(y.type); + error(x.expr, "'%.*s' expected 2 arguments of the same type, got '%s' vs '%s'", LIT(builtin_name), xs, ys); + gb_string_free(ys); + gb_string_free(xs); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + + operand->mode = Addressing_Value; + operand->type = x.type; + return true; + } + // Unary + case BuiltinProc_simd_neg: + case BuiltinProc_simd_abs: + { + Operand x = {}; + check_expr(c, &x, ce->args[0]); + if (x.mode == Addressing_Invalid) { + return false; + } + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem) && !is_type_float(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + operand->mode = Addressing_Value; + operand->type = x.type; + return true; + } + + // Return integer masks + case BuiltinProc_simd_eq: + case BuiltinProc_simd_ne: + case BuiltinProc_simd_lt: + case BuiltinProc_simd_le: + case BuiltinProc_simd_gt: + case BuiltinProc_simd_ge: + { + // op(#simd[N]T, #simd[N]T) -> #simd[N]V + // where `V` is an integer, `size_of(T) == size_of(V)` + // `V` will all 0s if false and all 1s if true (e.g. 0x00 and 0xff for false and true, respectively) + + Operand x = {}; + Operand y = {}; + check_expr(c, &x, ce->args[0]); + check_expr(c, &y, ce->args[1]); + if (x.mode == Addressing_Invalid) { + return false; + } + if (y.mode == Addressing_Invalid) { + return false; + } + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem) && !is_type_float(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + Type *vt = base_type(x.type); + GB_ASSERT(vt->kind == Type_SimdVector); + i64 count = vt->SimdVector.count; + + i64 sz = type_size_of(elem); + Type *new_elem = nullptr; + + switch (sz) { + case 1: new_elem = t_u8; break; + case 2: new_elem = t_u16; break; + case 4: new_elem = t_u32; break; + case 8: new_elem = t_u64; break; + case 16: + error(x.expr, "'%.*s' not supported 128-bit integer backed simd vector types", LIT(builtin_name)); + return false; + } + + operand->mode = Addressing_Value; + operand->type = alloc_type_simd_vector(count, new_elem); + return true; + } + default: + GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); + } + + return false; +} + + bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 id, Type *type_hint) { ast_node(ce, CallExpr, call); if (ce->inlining != ProcInlining_none) { @@ -479,7 +667,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 break; } - String builtin_name = builtin_procs[id].name; + String const &builtin_name = builtin_procs[id].name; if (ce->args.count > 0) { @@ -491,6 +679,16 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 } } + if (BuiltinProc__simd_begin < id && id < BuiltinProc__simd_end) { + bool ok = check_builtin_simd_operation(c, operand, call, id, type_hint); + if (!ok) { + operand->type = t_invalid; + } + operand->mode = Addressing_Value; + operand->value = {}; + return ok; + } + switch (id) { default: GB_PANIC("Implement built-in procedure: %.*s", LIT(builtin_name)); @@ -2720,6 +2918,13 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 return false; } + if (count < 1 || !is_power_of_two(count)) { + error(call, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); + operand->mode = Addressing_Type; + operand->type = t_invalid; + return false; + } + operand->mode = Addressing_Type; operand->type = alloc_type_simd_vector(count, elem); break; diff --git a/src/check_type.cpp b/src/check_type.cpp index 193c42cde..1df63e599 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2803,7 +2803,7 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t goto array_end; } if (count < 1 || !is_power_of_two(count)) { - error(at->elem, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); + error(at->count, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); *type = alloc_type_array(elem, count, generic_type); goto array_end; } diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index d407ef7c1..80467ffb1 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -118,6 +118,35 @@ enum BuiltinProcId { BuiltinProc_fixed_point_div_sat, BuiltinProc_expect, + +BuiltinProc__simd_begin, + BuiltinProc_simd_add, + BuiltinProc_simd_sub, + BuiltinProc_simd_mul, + BuiltinProc_simd_div, + BuiltinProc_simd_rem, + BuiltinProc_simd_shl, // Odin logic + BuiltinProc_simd_shr, // Odin logic + BuiltinProc_simd_shl_masked, // C logic + BuiltinProc_simd_shr_masked, // C logic + + BuiltinProc_simd_and, + BuiltinProc_simd_or, + BuiltinProc_simd_xor, + + BuiltinProc_simd_neg, + BuiltinProc_simd_abs, + + BuiltinProc_simd_min, + BuiltinProc_simd_max, + + BuiltinProc_simd_eq, + BuiltinProc_simd_ne, + BuiltinProc_simd_lt, + BuiltinProc_simd_le, + BuiltinProc_simd_gt, + BuiltinProc_simd_ge, +BuiltinProc__simd_end, // Platform specific intrinsics BuiltinProc_syscall, @@ -342,7 +371,33 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("fixed_point_div_sat"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("expect"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - + + {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_add"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_sub"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_mul"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_div"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_rem"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_shl"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_shr"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_shl_masked"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_shr_masked"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_and"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_or"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_xor"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_neg"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_abs"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_min"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_max"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_eq"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_ne"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lt"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_le"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_gt"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_ge"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + + {STR_LIT("syscall"), 1, true, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/types.cpp b/src/types.cpp index d5ba1a531..755f78f1c 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -1932,7 +1932,7 @@ bool is_type_valid_vector_elem(Type *t) { return false; } if (is_type_integer(t)) { - return true; + return !is_type_integer_128bit(t); } if (is_type_float(t)) { return true; From 81dd727f750aad45c6468de64a12119a3141de55 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 18:49:17 +0100 Subject: [PATCH 03/86] Implement backend for simd intrinsics --- src/check_builtin.cpp | 58 ++++++++++- src/llvm_backend_proc.cpp | 201 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+), 4 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 939892707..13eb9f47d 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -464,10 +464,6 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call // Integer only case BuiltinProc_simd_rem: - case BuiltinProc_simd_shl: - case BuiltinProc_simd_shr: - case BuiltinProc_simd_shl_masked: - case BuiltinProc_simd_shr_masked: case BuiltinProc_simd_and: case BuiltinProc_simd_or: case BuiltinProc_simd_xor: @@ -510,6 +506,60 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call operand->type = x.type; return true; } + + case BuiltinProc_simd_shl: + case BuiltinProc_simd_shr: + case BuiltinProc_simd_shl_masked: + case BuiltinProc_simd_shr_masked: + { + Operand x = {}; + Operand y = {}; + check_expr(c, &x, ce->args[0]); + check_expr(c, &y, ce->args[1]); + if (x.mode == Addressing_Invalid) { + return false; + } + if (y.mode == Addressing_Invalid) { + return false; + } + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!is_type_simd_vector(y.type)) { + error(y.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + GB_ASSERT(x.type->kind == Type_SimdVector); + GB_ASSERT(y.type->kind == Type_SimdVector); + Type *xt = x.type; + Type *yt = y.type; + + if (xt->SimdVector.count != yt->SimdVector.count) { + error(x.expr, "'%.*s' mismatched simd vector lengths, got '%lld' vs '%lld'", + LIT(builtin_name), + cast(long long)xt->SimdVector.count, + cast(long long)yt->SimdVector.count); + return false; + } + if (!is_type_integer(base_array_type(x.type))) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + if (!is_type_unsigned(base_array_type(y.type))) { + gbString ys = type_to_string(y.type); + error(y.expr, "'%.*s' expected a #simd type with an unsigned integer element as the shifting operand, got '%s'", LIT(builtin_name), ys); + gb_string_free(ys); + return false; + } + + operand->mode = Addressing_Value; + operand->type = x.type; + return true; + } + // Unary case BuiltinProc_simd_neg: case BuiltinProc_simd_abs: diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 154be2f1f..82ad6daef 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -981,10 +981,211 @@ lbValue lb_emit_call(lbProcedure *p, lbValue value, Array const &args, return result; } +lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, BuiltinProcId id) { + ast_node(ce, CallExpr, expr); + + lbModule *m = p->module; + + lbValue res = {}; + res.type = tv.type; + + lbValue arg0 = lb_build_expr(p, ce->args[0]); + lbValue arg1 = {}; + + Type *elem = base_array_type(arg0.type); + + bool is_float = is_type_float(elem); + bool is_signed = !is_type_unsigned(elem); + + LLVMOpcode op_code = cast(LLVMOpcode)0; + + switch (id) { + case BuiltinProc_simd_add: + case BuiltinProc_simd_sub: + case BuiltinProc_simd_mul: + case BuiltinProc_simd_div: + case BuiltinProc_simd_rem: + arg1 = lb_build_expr(p, ce->args[1]); + if (is_float) { + switch (id) { + case BuiltinProc_simd_add: op_code = LLVMFAdd; break; + case BuiltinProc_simd_sub: op_code = LLVMFSub; break; + case BuiltinProc_simd_mul: op_code = LLVMFMul; break; + case BuiltinProc_simd_div: op_code = LLVMFDiv; break; + } + } else { + switch (id) { + case BuiltinProc_simd_add: op_code = LLVMAdd; break; + case BuiltinProc_simd_sub: op_code = LLVMSub; break; + case BuiltinProc_simd_mul: op_code = LLVMMul; break; + case BuiltinProc_simd_div: + if (is_signed) { + op_code = LLVMSDiv; + } else { + op_code = LLVMUDiv; + } + break; + case BuiltinProc_simd_rem: + if (is_signed) { + op_code = LLVMSRem; + } else { + op_code = LLVMURem; + } + break; + } + } + if (op_code) { + res.value = LLVMBuildBinOp(p->builder, op_code, arg0.value, arg1.value, ""); + return res; + } + break; + case BuiltinProc_simd_shl: // Odin logic + case BuiltinProc_simd_shr: // Odin logic + case BuiltinProc_simd_shl_masked: // C logic + case BuiltinProc_simd_shr_masked: // C logic + arg1 = lb_build_expr(p, ce->args[1]); + { + i64 sz = type_size_of(elem); + GB_ASSERT(arg0.type->kind == Type_SimdVector); + + i64 count = arg0.type->SimdVector.count; + Type *elem1 = base_array_type(arg1.type); + + bool is_masked = false; + switch (id) { + case BuiltinProc_simd_shl: op_code = LLVMShl; is_masked = false; break; + case BuiltinProc_simd_shr: op_code = is_signed ? LLVMAShr : LLVMLShr; is_masked = false; break; + case BuiltinProc_simd_shl_masked: op_code = LLVMShl; is_masked = true; break; + case BuiltinProc_simd_shr_masked: op_code = is_signed ? LLVMAShr : LLVMLShr; is_masked = true; break; + } + if (op_code) { + LLVMValueRef bit_value = lb_const_int(m, elem1, sz*8 - 1).value; + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, count); + for (i64 i = 0; i < count; i++) { + values[i] = bit_value; + } + LLVMValueRef bits = LLVMConstVector(values, cast(unsigned)count); + if (is_masked) { + // C logic + LLVMValueRef shift = LLVMBuildAnd(p->builder, arg1.value, bits, ""); + res.value = LLVMBuildBinOp(p->builder, op_code, arg0.value, shift, ""); + } else { + // Odin logic + LLVMValueRef zero = lb_const_nil(m, arg1.type).value; + LLVMValueRef mask = LLVMBuildICmp(p->builder, LLVMIntULE, arg1.value, bits, ""); + LLVMValueRef shift = LLVMBuildBinOp(p->builder, op_code, arg0.value, arg1.value, ""); + res.value = LLVMBuildSelect(p->builder, mask, shift, zero, ""); + } + + return res; + } + } + break; + case BuiltinProc_simd_and: + case BuiltinProc_simd_or: + case BuiltinProc_simd_xor: + arg1 = lb_build_expr(p, ce->args[1]); + switch (id) { + case BuiltinProc_simd_and: op_code = LLVMAnd; break; + case BuiltinProc_simd_or: op_code = LLVMOr; break; + case BuiltinProc_simd_xor: op_code = LLVMXor; break; + } + if (op_code) { + res.value = LLVMBuildBinOp(p->builder, op_code, arg0.value, arg1.value, ""); + return res; + } + break; + case BuiltinProc_simd_neg: + if (is_float) { + res.value = LLVMBuildFNeg(p->builder, arg0.value, ""); + } else { + res.value = LLVMBuildNeg(p->builder, arg0.value, ""); + } + return res; + case BuiltinProc_simd_abs: + if (is_float) { + LLVMValueRef pos = arg0.value; + LLVMValueRef neg = LLVMBuildFNeg(p->builder, pos, ""); + LLVMValueRef cond = LLVMBuildFCmp(p->builder, LLVMRealOGT, pos, neg, ""); + res.value = LLVMBuildSelect(p->builder, cond, pos, neg, ""); + } else { + LLVMValueRef pos = arg0.value; + LLVMValueRef neg = LLVMBuildNeg(p->builder, pos, ""); + LLVMValueRef cond = LLVMBuildICmp(p->builder, is_signed ? LLVMIntSGT : LLVMIntUGT, pos, neg, ""); + res.value = LLVMBuildSelect(p->builder, cond, pos, neg, ""); + } + return res; + case BuiltinProc_simd_min: + if (is_float) { + LLVMValueRef cond = LLVMBuildFCmp(p->builder, LLVMRealOLT, arg0.value, arg1.value, ""); + res.value = LLVMBuildSelect(p->builder, cond, arg0.value, arg1.value, ""); + } else { + LLVMValueRef cond = LLVMBuildICmp(p->builder, is_signed ? LLVMIntSLT : LLVMIntULT, arg0.value, arg1.value, ""); + res.value = LLVMBuildSelect(p->builder, cond, arg0.value, arg1.value, ""); + } + return res; + case BuiltinProc_simd_max: + arg1 = lb_build_expr(p, ce->args[1]); + if (is_float) { + LLVMValueRef cond = LLVMBuildFCmp(p->builder, LLVMRealOGT, arg0.value, arg1.value, ""); + res.value = LLVMBuildSelect(p->builder, cond, arg0.value, arg1.value, ""); + } else { + LLVMValueRef cond = LLVMBuildICmp(p->builder, is_signed ? LLVMIntSGT : LLVMIntUGT, arg0.value, arg1.value, ""); + res.value = LLVMBuildSelect(p->builder, cond, arg0.value, arg1.value, ""); + } + return res; + case BuiltinProc_simd_eq: + case BuiltinProc_simd_ne: + case BuiltinProc_simd_lt: + case BuiltinProc_simd_le: + case BuiltinProc_simd_gt: + case BuiltinProc_simd_ge: + arg1 = lb_build_expr(p, ce->args[1]); + if (is_float) { + LLVMRealPredicate pred = cast(LLVMRealPredicate)0; + switch (id) { + case BuiltinProc_simd_eq: pred = LLVMRealOEQ; break; + case BuiltinProc_simd_ne: pred = LLVMRealONE; break; + case BuiltinProc_simd_lt: pred = LLVMRealOLT; break; + case BuiltinProc_simd_le: pred = LLVMRealOLE; break; + case BuiltinProc_simd_gt: pred = LLVMRealOGT; break; + case BuiltinProc_simd_ge: pred = LLVMRealOGE; break; + } + if (pred) { + res.value = LLVMBuildFCmp(p->builder, pred, arg0.value, arg1.value, ""); + res.value = LLVMBuildSExtOrBitCast(p->builder, res.value, lb_type(m, tv.type), ""); + return res; + } + } else { + LLVMIntPredicate pred = cast(LLVMIntPredicate)0; + switch (id) { + case BuiltinProc_simd_eq: pred = LLVMIntEQ; break; + case BuiltinProc_simd_ne: pred = LLVMIntNE; break; + case BuiltinProc_simd_lt: pred = is_signed ? LLVMIntSLT :LLVMIntULT; break; + case BuiltinProc_simd_le: pred = is_signed ? LLVMIntSLE :LLVMIntULE; break; + case BuiltinProc_simd_gt: pred = is_signed ? LLVMIntSGT :LLVMIntUGT; break; + case BuiltinProc_simd_ge: pred = is_signed ? LLVMIntSGE :LLVMIntUGE; break; + } + if (pred) { + res.value = LLVMBuildICmp(p->builder, pred, arg0.value, arg1.value, ""); + res.value = LLVMBuildSExtOrBitCast(p->builder, res.value, lb_type(m, tv.type), ""); + return res; + } + } + break; + } + GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[id].name)); + return {}; +} + lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, BuiltinProcId id) { ast_node(ce, CallExpr, expr); + if (BuiltinProc__simd_begin < id && id < BuiltinProc__simd_end) { + return lb_build_builtin_simd_proc(p, expr, tv, id); + } + switch (id) { case BuiltinProc_DIRECTIVE: { ast_node(bd, BasicDirective, ce->proc); From f21e9ee71281ec8665a3009cd1a350ffde3b7046 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 18:59:47 +0100 Subject: [PATCH 04/86] Allow basic casting of simd vectors --- src/check_expr.cpp | 12 ++++++++++++ src/llvm_backend_expr.cpp | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index a4dfade98..9a7cd5b8b 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -2688,6 +2688,18 @@ bool check_is_castable_to(CheckerContext *c, Operand *operand, Type *y) { return true; } + if (is_type_simd_vector(src) && is_type_simd_vector(dst)) { + if (src->SimdVector.count != dst->SimdVector.count) { + return false; + } + Type *elem_src = base_array_type(src); + Type *elem_dst = base_array_type(dst); + Operand x = {}; + x.type = elem_src; + x.mode = Addressing_Value; + return check_is_castable_to(c, &x, elem_dst); + } + return false; } diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 133df4d41..f4b5702bb 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -1820,6 +1820,38 @@ lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { return res; } + if (is_type_simd_vector(src) && is_type_simd_vector(dst)) { + Type *src_elem = core_array_type(src); + Type *dst_elem = core_array_type(dst); + + GB_ASSERT(src->SimdVector.count == dst->SimdVector.count); + + lbValue res = {}; + res.type = t; + if (are_types_identical(src_elem, dst_elem)) { + res.value = value.value; + } else if (is_type_float(src_elem) && is_type_integer(dst_elem)) { + if (is_type_unsigned(dst_elem)) { + res.value = LLVMBuildFPToUI(p->builder, value.value, lb_type(m, t), ""); + } else { + res.value = LLVMBuildFPToSI(p->builder, value.value, lb_type(m, t), ""); + } + } else if (is_type_integer(src_elem) && is_type_float(dst_elem)) { + if (is_type_unsigned(src_elem)) { + res.value = LLVMBuildUIToFP(p->builder, value.value, lb_type(m, t), ""); + } else { + res.value = LLVMBuildSIToFP(p->builder, value.value, lb_type(m, t), ""); + } + } else if (is_type_integer(src_elem) && is_type_integer(dst_elem)) { + res.value = LLVMBuildIntCast2(p->builder, value.value, lb_type(m, t), !is_type_unsigned(src_elem), ""); + } else if (is_type_float(src_elem) && is_type_float(dst_elem)) { + res.value = LLVMBuildFPCast(p->builder, value.value, lb_type(m, t), ""); + } else { + GB_PANIC("Unhandled simd vector conversion: %s -> %s", type_to_string(src), type_to_string(dst)); + } + return res; + } + // Pointer <-> uintptr if (is_type_pointer(src) && is_type_uintptr(dst)) { lbValue res = {}; From 5c72974167405a37bd397788e3224d773efd9c46 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 19:04:25 +0100 Subject: [PATCH 05/86] Simplify transmute for #simd --- src/llvm_backend_utility.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index 037171637..bfd21bedb 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -201,6 +201,11 @@ lbValue lb_emit_transmute(lbProcedure *p, lbValue value, Type *t) { return res; } + if (is_type_simd_vector(src) && is_type_simd_vector(dst)) { + res.value = LLVMBuildBitCast(p->builder, value.value, lb_type(p->module, t), ""); + return res; + } + if (lb_is_type_aggregate(src) || lb_is_type_aggregate(dst)) { lbValue s = lb_address_from_load_or_generate_local(p, value); lbValue d = lb_emit_transmute(p, s, alloc_type_pointer(t)); From 4c4480104de9d6ba520215afb6330c95b0e56b93 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 20:27:14 +0100 Subject: [PATCH 06/86] Add `simd_extract` and `simd_insert` --- src/check_builtin.cpp | 120 +++++++++++++++++++++++++--------- src/checker_builtin_procs.hpp | 6 ++ src/llvm_backend_proc.cpp | 11 ++++ 3 files changed, 105 insertions(+), 32 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 13eb9f47d..ab4cc210c 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -425,14 +425,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call { Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); - check_expr(c, &y, ce->args[1]); - if (x.mode == Addressing_Invalid) { - return false; - } - if (y.mode == Addressing_Invalid) { - return false; - } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &y, x.type); if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -470,14 +465,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call { Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); - check_expr(c, &y, ce->args[1]); - if (x.mode == Addressing_Invalid) { - return false; - } - if (y.mode == Addressing_Invalid) { - return false; - } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &y, x.type); if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -514,14 +504,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call { Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); - check_expr(c, &y, ce->args[1]); - if (x.mode == Addressing_Invalid) { - return false; - } - if (y.mode == Addressing_Invalid) { - return false; - } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &y, x.type); if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -599,14 +584,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); - check_expr(c, &y, ce->args[1]); - if (x.mode == Addressing_Invalid) { - return false; - } - if (y.mode == Addressing_Invalid) { - return false; - } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &y, x.type); if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -639,6 +619,81 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call operand->type = alloc_type_simd_vector(count, new_elem); return true; } + + case BuiltinProc_simd_extract: + { + Operand x = {}; + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem) && !is_type_float(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + i64 max_count = x.type->SimdVector.count; + i64 value = -1; + if (!check_index_value(c, x.type, false, ce->args[1], max_count, &value)) { + return false; + } + if (max_count < 0) { + error(ce->args[1], "'%.*s' expected a constant integer index, got '%lld'", LIT(builtin_name), cast(long long)value); + return false; + } + + operand->mode = Addressing_Value; + operand->type = elem; + return true; + } + break; + case BuiltinProc_simd_insert: + { + Operand x = {}; + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem) && !is_type_float(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + i64 max_count = x.type->SimdVector.count; + i64 value = -1; + if (!check_index_value(c, x.type, false, ce->args[1], max_count, &value)) { + return false; + } + if (max_count < 0) { + error(ce->args[1], "'%.*s' expected a constant integer index, got '%lld'", LIT(builtin_name), cast(long long)value); + return false; + } + + Operand y = {}; + check_expr_with_type_hint(c, &y, ce->args[2], elem); if (y.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &y, elem); + if (!are_types_identical(y.type, elem)) { + gbString et = type_to_string(elem); + gbString yt = type_to_string(y.type); + error(y.expr, "'%.*s' expected a type of '%s' to insert, got '%s'", LIT(builtin_name), et, yt); + gb_string_free(yt); + gb_string_free(et); + return false; + } + + operand->mode = Addressing_Value; + operand->type = x.type; + return true; + } + break; default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); } @@ -736,6 +791,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 } operand->mode = Addressing_Value; operand->value = {}; + operand->expr = call; return ok; } diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 80467ffb1..604e9dc8c 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -146,6 +146,9 @@ BuiltinProc__simd_begin, BuiltinProc_simd_le, BuiltinProc_simd_gt, BuiltinProc_simd_ge, + + BuiltinProc_simd_extract, + BuiltinProc_simd_insert, BuiltinProc__simd_end, // Platform specific intrinsics @@ -395,6 +398,9 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_le"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_gt"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_ge"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + + {STR_LIT("simd_extract"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_insert"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 82ad6daef..4af5d9440 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -991,6 +991,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const lbValue arg0 = lb_build_expr(p, ce->args[0]); lbValue arg1 = {}; + lbValue arg2 = {}; Type *elem = base_array_type(arg0.type); @@ -1173,6 +1174,16 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const } } break; + + case BuiltinProc_simd_extract: + arg1 = lb_build_expr(p, ce->args[1]); + res.value = LLVMBuildExtractElement(p->builder, arg0.value, arg1.value, ""); + return res; + case BuiltinProc_simd_insert: + arg1 = lb_build_expr(p, ce->args[1]); + arg2 = lb_build_expr(p, ce->args[2]); + res.value = LLVMBuildInsertElement(p->builder, arg0.value, arg2.value, arg1.value, ""); + return res; } GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[id].name)); return {}; From 53f0c6ef1a73ab7afe21ed15d3d88a266af6a03c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 20:31:31 +0100 Subject: [PATCH 07/86] Add ranges for simd compounds literals --- src/check_expr.cpp | 192 ++++++++++++++++++------------------- src/llvm_backend_const.cpp | 85 +++++++++++++--- 2 files changed, 163 insertions(+), 114 deletions(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 9a7cd5b8b..3dbecb1c0 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -7741,112 +7741,106 @@ ExprKind check_compound_literal(CheckerContext *c, Operand *o, Ast *node, Type * } if (cl->elems.count > 0 && cl->elems[0]->kind == Ast_FieldValue) { - // TODO(bill): Why was this decision made for simd? - if (is_type_simd_vector(t)) { - error(cl->elems[0], "'field = value' is not allowed for SIMD vector literals"); - } else { - RangeCache rc = range_cache_make(heap_allocator()); - defer (range_cache_destroy(&rc)); + RangeCache rc = range_cache_make(heap_allocator()); + defer (range_cache_destroy(&rc)); - for_array(i, cl->elems) { - Ast *elem = cl->elems[i]; - if (elem->kind != Ast_FieldValue) { - error(elem, "Mixture of 'field = value' and value elements in a literal is not allowed"); + for_array(i, cl->elems) { + Ast *elem = cl->elems[i]; + if (elem->kind != Ast_FieldValue) { + error(elem, "Mixture of 'field = value' and value elements in a literal is not allowed"); + continue; + } + ast_node(fv, FieldValue, elem); + + if (is_ast_range(fv->field)) { + Token op = fv->field->BinaryExpr.op; + + Operand x = {}; + Operand y = {}; + bool ok = check_range(c, fv->field, &x, &y, nullptr); + if (!ok) { continue; } - ast_node(fv, FieldValue, elem); - - if (is_ast_range(fv->field)) { - Token op = fv->field->BinaryExpr.op; - - Operand x = {}; - Operand y = {}; - bool ok = check_range(c, fv->field, &x, &y, nullptr); - if (!ok) { - continue; - } - if (x.mode != Addressing_Constant || !is_type_integer(core_type(x.type))) { - error(x.expr, "Expected a constant integer as an array field"); - continue; - } - - if (y.mode != Addressing_Constant || !is_type_integer(core_type(y.type))) { - error(y.expr, "Expected a constant integer as an array field"); - continue; - } - - i64 lo = exact_value_to_i64(x.value); - i64 hi = exact_value_to_i64(y.value); - i64 max_index = hi; - if (op.kind == Token_RangeHalf) { // ..< (exclusive) - hi -= 1; - } else { // .. (inclusive) - max_index += 1; - } - - bool new_range = range_cache_add_range(&rc, lo, hi); - if (!new_range) { - error(elem, "Overlapping field range index %lld %.*s %lld for %.*s", lo, LIT(op.string), hi, LIT(context_name)); - continue; - } - - - if (max_type_count >= 0 && (lo < 0 || lo >= max_type_count)) { - error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", lo, max_type_count, LIT(context_name)); - continue; - } - if (max_type_count >= 0 && (hi < 0 || hi >= max_type_count)) { - error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", hi, max_type_count, LIT(context_name)); - continue; - } - - if (max < hi) { - max = max_index; - } - - Operand operand = {}; - check_expr_with_type_hint(c, &operand, fv->value, elem_type); - check_assignment(c, &operand, elem_type, context_name); - - is_constant = is_constant && operand.mode == Addressing_Constant; - } else { - Operand op_index = {}; - check_expr(c, &op_index, fv->field); - - if (op_index.mode != Addressing_Constant || !is_type_integer(core_type(op_index.type))) { - error(elem, "Expected a constant integer as an array field"); - continue; - } - // add_type_and_value(c->info, op_index.expr, op_index.mode, op_index.type, op_index.value); - - i64 index = exact_value_to_i64(op_index.value); - - if (max_type_count >= 0 && (index < 0 || index >= max_type_count)) { - error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", index, max_type_count, LIT(context_name)); - continue; - } - - bool new_index = range_cache_add_index(&rc, index); - if (!new_index) { - error(elem, "Duplicate field index %lld for %.*s", index, LIT(context_name)); - continue; - } - - if (max < index+1) { - max = index+1; - } - - Operand operand = {}; - check_expr_with_type_hint(c, &operand, fv->value, elem_type); - check_assignment(c, &operand, elem_type, context_name); - - is_constant = is_constant && operand.mode == Addressing_Constant; + if (x.mode != Addressing_Constant || !is_type_integer(core_type(x.type))) { + error(x.expr, "Expected a constant integer as an array field"); + continue; } - } - cl->max_count = max; + if (y.mode != Addressing_Constant || !is_type_integer(core_type(y.type))) { + error(y.expr, "Expected a constant integer as an array field"); + continue; + } + + i64 lo = exact_value_to_i64(x.value); + i64 hi = exact_value_to_i64(y.value); + i64 max_index = hi; + if (op.kind == Token_RangeHalf) { // ..< (exclusive) + hi -= 1; + } else { // .. (inclusive) + max_index += 1; + } + + bool new_range = range_cache_add_range(&rc, lo, hi); + if (!new_range) { + error(elem, "Overlapping field range index %lld %.*s %lld for %.*s", lo, LIT(op.string), hi, LIT(context_name)); + continue; + } + + + if (max_type_count >= 0 && (lo < 0 || lo >= max_type_count)) { + error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", lo, max_type_count, LIT(context_name)); + continue; + } + if (max_type_count >= 0 && (hi < 0 || hi >= max_type_count)) { + error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", hi, max_type_count, LIT(context_name)); + continue; + } + + if (max < hi) { + max = max_index; + } + + Operand operand = {}; + check_expr_with_type_hint(c, &operand, fv->value, elem_type); + check_assignment(c, &operand, elem_type, context_name); + + is_constant = is_constant && operand.mode == Addressing_Constant; + } else { + Operand op_index = {}; + check_expr(c, &op_index, fv->field); + + if (op_index.mode != Addressing_Constant || !is_type_integer(core_type(op_index.type))) { + error(elem, "Expected a constant integer as an array field"); + continue; + } + // add_type_and_value(c->info, op_index.expr, op_index.mode, op_index.type, op_index.value); + + i64 index = exact_value_to_i64(op_index.value); + + if (max_type_count >= 0 && (index < 0 || index >= max_type_count)) { + error(elem, "Index %lld is out of bounds (0..<%lld) for %.*s", index, max_type_count, LIT(context_name)); + continue; + } + + bool new_index = range_cache_add_index(&rc, index); + if (!new_index) { + error(elem, "Duplicate field index %lld for %.*s", index, LIT(context_name)); + continue; + } + + if (max < index+1) { + max = index+1; + } + + Operand operand = {}; + check_expr_with_type_hint(c, &operand, fv->value, elem_type); + check_assignment(c, &operand, elem_type, context_name); + + is_constant = is_constant && operand.mode == Addressing_Constant; + } } + cl->max_count = max; } else { isize index = 0; for (; index < cl->elems.count; index++) { diff --git a/src/llvm_backend_const.cpp b/src/llvm_backend_const.cpp index 8f17a1cfb..3a3067dbc 100644 --- a/src/llvm_backend_const.cpp +++ b/src/llvm_backend_const.cpp @@ -819,26 +819,81 @@ lbValue lb_const_value(lbModule *m, Type *type, ExactValue value, bool allow_loc return lb_const_nil(m, original_type); } GB_ASSERT(elem_type_can_be_constant(elem_type)); - isize total_elem_count = cast(isize)type->SimdVector.count; LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, total_elem_count); - for (isize i = 0; i < elem_count; i++) { - TypeAndValue tav = cl->elems[i]->tav; - GB_ASSERT(tav.mode != Addressing_Invalid); - values[i] = lb_const_value(m, elem_type, tav.value, allow_local).value; - } - LLVMTypeRef et = lb_type(m, elem_type); + if (cl->elems[0]->kind == Ast_FieldValue) { + // TODO(bill): This is O(N*M) and will be quite slow; it should probably be sorted before hand + isize value_index = 0; + for (i64 i = 0; i < total_elem_count; i++) { + bool found = false; - for (isize i = elem_count; i < type->SimdVector.count; i++) { - values[i] = LLVMConstNull(et); - } - for (isize i = 0; i < total_elem_count; i++) { - values[i] = llvm_const_cast(values[i], et); - } + for (isize j = 0; j < elem_count; j++) { + Ast *elem = cl->elems[j]; + ast_node(fv, FieldValue, elem); + if (is_ast_range(fv->field)) { + ast_node(ie, BinaryExpr, fv->field); + TypeAndValue lo_tav = ie->left->tav; + TypeAndValue hi_tav = ie->right->tav; + GB_ASSERT(lo_tav.mode == Addressing_Constant); + GB_ASSERT(hi_tav.mode == Addressing_Constant); - res.value = LLVMConstVector(values, cast(unsigned)total_elem_count); - return res; + TokenKind op = ie->op.kind; + i64 lo = exact_value_to_i64(lo_tav.value); + i64 hi = exact_value_to_i64(hi_tav.value); + if (op != Token_RangeHalf) { + hi += 1; + } + if (lo == i) { + TypeAndValue tav = fv->value->tav; + LLVMValueRef val = lb_const_value(m, elem_type, tav.value, allow_local).value; + for (i64 k = lo; k < hi; k++) { + values[value_index++] = val; + } + + found = true; + i += (hi-lo-1); + break; + } + } else { + TypeAndValue index_tav = fv->field->tav; + GB_ASSERT(index_tav.mode == Addressing_Constant); + i64 index = exact_value_to_i64(index_tav.value); + if (index == i) { + TypeAndValue tav = fv->value->tav; + LLVMValueRef val = lb_const_value(m, elem_type, tav.value, allow_local).value; + values[value_index++] = val; + found = true; + break; + } + } + } + + if (!found) { + values[value_index++] = LLVMConstNull(lb_type(m, elem_type)); + } + } + + res.value = LLVMConstVector(values, cast(unsigned)total_elem_count); + return res; + } else { + for (isize i = 0; i < elem_count; i++) { + TypeAndValue tav = cl->elems[i]->tav; + GB_ASSERT(tav.mode != Addressing_Invalid); + values[i] = lb_const_value(m, elem_type, tav.value, allow_local).value; + } + LLVMTypeRef et = lb_type(m, elem_type); + + for (isize i = elem_count; i < total_elem_count; i++) { + values[i] = LLVMConstNull(et); + } + for (isize i = 0; i < total_elem_count; i++) { + values[i] = llvm_const_cast(values[i], et); + } + + res.value = LLVMConstVector(values, cast(unsigned)total_elem_count); + return res; + } } else if (is_type_struct(type)) { ast_node(cl, CompoundLit, value.value_compound); From 0203bb657ed09046dddc958be5322b1cd8c0a589 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 20:39:22 +0100 Subject: [PATCH 08/86] Allow for non-constant simd vector compound types --- src/check_expr.cpp | 2 +- src/llvm_backend_expr.cpp | 96 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 3dbecb1c0..9fd6acefd 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -7885,7 +7885,7 @@ ExprKind check_compound_literal(CheckerContext *c, Operand *o, Ast *node, Type * if (t->kind == Type_SimdVector) { if (!is_constant) { - error(node, "Expected all constant elements for a simd vector"); + // error(node, "Expected all constant elements for a simd vector"); } } diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index f4b5702bb..1b10cd776 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -4641,6 +4641,102 @@ lbAddr lb_build_addr(lbProcedure *p, Ast *expr) { break; } + case Type_SimdVector: { + if (cl->elems.count > 0) { + lbValue vector_value = lb_const_value(p->module, type, exact_value_compound(expr)); + defer (lb_addr_store(p, v, vector_value)); + + auto temp_data = array_make(temporary_allocator(), 0, cl->elems.count); + + // NOTE(bill): Separate value, store into their own chunks + for_array(i, cl->elems) { + Ast *elem = cl->elems[i]; + if (elem->kind == Ast_FieldValue) { + ast_node(fv, FieldValue, elem); + if (lb_is_elem_const(fv->value, et)) { + continue; + } + if (is_ast_range(fv->field)) { + ast_node(ie, BinaryExpr, fv->field); + TypeAndValue lo_tav = ie->left->tav; + TypeAndValue hi_tav = ie->right->tav; + GB_ASSERT(lo_tav.mode == Addressing_Constant); + GB_ASSERT(hi_tav.mode == Addressing_Constant); + + TokenKind op = ie->op.kind; + i64 lo = exact_value_to_i64(lo_tav.value); + i64 hi = exact_value_to_i64(hi_tav.value); + if (op != Token_RangeHalf) { + hi += 1; + } + + lbValue value = lb_build_expr(p, fv->value); + + for (i64 k = lo; k < hi; k++) { + lbCompoundLitElemTempData data = {}; + data.value = value; + data.elem_index = cast(i32)k; + array_add(&temp_data, data); + } + + } else { + auto tav = fv->field->tav; + GB_ASSERT(tav.mode == Addressing_Constant); + i64 index = exact_value_to_i64(tav.value); + + lbValue value = lb_build_expr(p, fv->value); + lbCompoundLitElemTempData data = {}; + data.value = lb_emit_conv(p, value, et); + data.expr = fv->value; + data.elem_index = cast(i32)index; + array_add(&temp_data, data); + } + + } else { + if (lb_is_elem_const(elem, et)) { + continue; + } + lbCompoundLitElemTempData data = {}; + data.expr = elem; + data.elem_index = cast(i32)i; + array_add(&temp_data, data); + } + } + + + for_array(i, temp_data) { + lbValue field_expr = temp_data[i].value; + Ast *expr = temp_data[i].expr; + + auto prev_hint = lb_set_copy_elision_hint(p, lb_addr(temp_data[i].gep), expr); + + if (field_expr.value == nullptr) { + field_expr = lb_build_expr(p, expr); + } + Type *t = field_expr.type; + GB_ASSERT(t->kind != Type_Tuple); + lbValue ev = lb_emit_conv(p, field_expr, et); + + if (!p->copy_elision_hint.used) { + temp_data[i].value = ev; + } + + lb_reset_copy_elision_hint(p, prev_hint); + } + + + // TODO(bill): reduce the need for individual `insertelement` if a `shufflevector` + // might be a better option + + for_array(i, temp_data) { + if (temp_data[i].value.value != nullptr) { + LLVMValueRef index = lb_const_int(p->module, t_u32, temp_data[i].elem_index).value; + vector_value.value = LLVMBuildInsertElement(p->builder, vector_value.value, temp_data[i].value.value, index, ""); + } + } + } + break; + } } return v; From b168bf9460491a101f3a7d41c28500a45898ecbf Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 21:00:00 +0100 Subject: [PATCH 09/86] Rename `simd_insert` to `simd_replace` --- src/check_builtin.cpp | 2 +- src/checker_builtin_procs.hpp | 4 ++-- src/llvm_backend_proc.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index ab4cc210c..64b2ebfce 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -651,7 +651,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } break; - case BuiltinProc_simd_insert: + case BuiltinProc_simd_replace: { Operand x = {}; check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 604e9dc8c..f5d4111bc 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -148,7 +148,7 @@ BuiltinProc__simd_begin, BuiltinProc_simd_ge, BuiltinProc_simd_extract, - BuiltinProc_simd_insert, + BuiltinProc_simd_replace, BuiltinProc__simd_end, // Platform specific intrinsics @@ -400,7 +400,7 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_ge"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_extract"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_insert"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_replace"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 4af5d9440..cfb69c654 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1179,7 +1179,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const arg1 = lb_build_expr(p, ce->args[1]); res.value = LLVMBuildExtractElement(p->builder, arg0.value, arg1.value, ""); return res; - case BuiltinProc_simd_insert: + case BuiltinProc_simd_replace: arg1 = lb_build_expr(p, ce->args[1]); arg2 = lb_build_expr(p, ce->args[2]); res.value = LLVMBuildInsertElement(p->builder, arg0.value, arg2.value, arg1.value, ""); From 1549d01bf76e8c5e13626e57b1f976330a9cdd50 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 21:17:21 +0100 Subject: [PATCH 10/86] Restrict `swizzle` to a power of two for #simd --- src/check_builtin.cpp | 35 +++++++++++++++++++++++++++++++++++ src/check_expr.cpp | 6 +++++- src/check_type.cpp | 6 ++++-- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 64b2ebfce..69e584827 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -694,6 +694,36 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } break; + + // case BuiltinProc_simd_rotate_left: + // { + // Operand x = {}; + // check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + + // if (!is_type_simd_vector(x.type)) { + // error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + // return false; + // } + // Type *elem = base_array_type(x.type); + // if (!is_type_integer(elem) && !is_type_float(elem)) { + // gbString xs = type_to_string(x.type); + // error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + // gb_string_free(xs); + // return false; + // } + + // Operand offset = {}; + // check_expr_with_type_hint(c, &offset, ce->args[1]); if (x.mode == Addressing_Invalid) { return false; } + // convert_to_typed(c, &offset, t_int); + // if (offset.mode != Addressing_Constant) { + // error(offset.expr, "'%.*s' expected a constant integer for the offset", LIT(builtin_name)); + // return false; + // } + + // operand->mode = Addressing_Value; + // operand->type = x.type; + // return true + // } default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); } @@ -1749,6 +1779,11 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 operand->mode = Addressing_Value; } + if (is_type_simd_vector(type) && !is_power_of_two(arg_count)) { + error(call, "'swizzle' with a #simd vector must have a power of two arguments, got %lld", cast(long long)arg_count); + return false; + } + operand->type = determine_swizzle_array_type(original_type, type_hint, arg_count); break; } diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 9fd6acefd..a30f83e7e 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -4119,7 +4119,11 @@ ExactValue get_constant_field(CheckerContext *c, Operand const *operand, Selecti Type *determine_swizzle_array_type(Type *original_type, Type *type_hint, isize new_count) { Type *array_type = base_type(type_deref(original_type)); - GB_ASSERT(array_type->kind == Type_Array); + GB_ASSERT(array_type->kind == Type_Array || array_type->kind == Type_SimdVector); + if (array_type->kind == Type_SimdVector) { + Type *elem_type = array_type->SimdVector.elem; + return alloc_type_simd_vector(new_count, elem_type); + } Type *elem_type = array_type->Array.elem; Type *swizzle_array_type = nullptr; diff --git a/src/check_type.cpp b/src/check_type.cpp index 1df63e599..088853810 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2795,14 +2795,16 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t if (name == "soa") { *type = make_soa_struct_fixed(ctx, e, at->elem, elem, count, generic_type); } else if (name == "simd") { - if (!is_type_valid_vector_elem(elem)) { + if (!is_type_valid_vector_elem(elem) && !is_type_polymorphic(elem)) { gbString str = type_to_string(elem); error(at->elem, "Invalid element type for 'intrinsics.simd_vector', expected an integer or float with no specific endianness, got '%s'", str); gb_string_free(str); *type = alloc_type_array(elem, count, generic_type); goto array_end; } - if (count < 1 || !is_power_of_two(count)) { + if (is_type_polymorphic(elem)) { + count = 1; + } else if (count < 1 || !is_power_of_two(count)) { error(at->count, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); *type = alloc_type_array(elem, count, generic_type); goto array_end; From 63cc8a80a045d48960d85640d11f39237c2f8ca4 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 21:29:45 +0100 Subject: [PATCH 11/86] Correct parapoly for #simd --- src/check_expr.cpp | 13 +++++++++++++ src/check_type.cpp | 4 ++-- src/types.cpp | 9 ++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index a30f83e7e..fcd7818bc 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -1328,6 +1328,19 @@ bool is_polymorphic_type_assignable(CheckerContext *c, Type *poly, Type *source, } } return false; + + case Type_SimdVector: + if (source->kind == Type_SimdVector) { + if (poly->SimdVector.generic_count != nullptr) { + if (!polymorphic_assign_index(&poly->SimdVector.generic_count, &poly->SimdVector.count, source->SimdVector.count)) { + return false; + } + } + if (poly->SimdVector.count == source->SimdVector.count) { + return is_polymorphic_type_assignable(c, poly->SimdVector.elem, source->SimdVector.elem, true, modify_type); + } + } + return false; } return false; } diff --git a/src/check_type.cpp b/src/check_type.cpp index 088853810..354ab6e94 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2803,14 +2803,14 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t goto array_end; } if (is_type_polymorphic(elem)) { - count = 1; + // Ignore } else if (count < 1 || !is_power_of_two(count)) { error(at->count, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); *type = alloc_type_array(elem, count, generic_type); goto array_end; } - *type = alloc_type_simd_vector(count, elem); + *type = alloc_type_simd_vector(count, elem, generic_type); } else { error(at->tag, "Invalid tag applied to array, got #%.*s", LIT(name)); *type = alloc_type_array(elem, count, generic_type); diff --git a/src/types.cpp b/src/types.cpp index 755f78f1c..2d5709b19 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -261,6 +261,7 @@ struct TypeProc { TYPE_KIND(SimdVector, struct { \ i64 count; \ Type *elem; \ + Type *generic_count; \ }) \ TYPE_KIND(RelativePointer, struct { \ Type *pointer_type; \ @@ -1085,10 +1086,11 @@ Type *alloc_type_bit_set() { -Type *alloc_type_simd_vector(i64 count, Type *elem) { +Type *alloc_type_simd_vector(i64 count, Type *elem, Type *generic_count=nullptr) { Type *t = alloc_type(Type_SimdVector); t->SimdVector.count = count; t->SimdVector.elem = elem; + t->SimdVector.generic_count = generic_count; return t; } @@ -2078,6 +2080,11 @@ bool is_type_polymorphic(Type *t, bool or_specialized=false) { return true; } return is_type_polymorphic(t->Array.elem, or_specialized); + case Type_SimdVector: + if (t->SimdVector.generic_count != nullptr) { + return true; + } + return is_type_polymorphic(t->SimdVector.elem, or_specialized); case Type_DynamicArray: return is_type_polymorphic(t->DynamicArray.elem, or_specialized); case Type_Slice: From 8ac12886ed90298f3de1e4685153b90bb67fd6db Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 21:30:10 +0100 Subject: [PATCH 12/86] Add `core:simd` --- core/simd/simd.odin | 58 ++++++++++++++++++++++++++++++++++++++ examples/all/all_main.odin | 1 + 2 files changed, 59 insertions(+) create mode 100644 core/simd/simd.odin diff --git a/core/simd/simd.odin b/core/simd/simd.odin new file mode 100644 index 000000000..87386f91f --- /dev/null +++ b/core/simd/simd.odin @@ -0,0 +1,58 @@ +package simd + +import "core:intrinsics" + +add :: intrinsics.simd_add +sub :: intrinsics.simd_sub +mul :: intrinsics.simd_mul +div :: intrinsics.simd_div +rem :: intrinsics.simd_rem + +// Keeps Odin's Behaviour +// (x << y) if y <= mask else 0 +shl :: intrinsics.simd_shl +shr :: intrinsics.simd_shr + +// Similar to C's Behaviour +// x << (y & mask) +shl_masked :: intrinsics.simd_shl_masked +shr_masked :: intrinsics.simd_shr_masked + +and :: intrinsics.simd_and +or :: intrinsics.simd_or +xor :: intrinsics.simd_xor +neg :: intrinsics.simd_neg +abs :: intrinsics.simd_abs +min :: intrinsics.simd_min +max :: intrinsics.simd_max +eq :: intrinsics.simd_eq +ne :: intrinsics.simd_ne +lt :: intrinsics.simd_lt +le :: intrinsics.simd_le +gt :: intrinsics.simd_gt +ge :: intrinsics.simd_ge +extract :: intrinsics.simd_extract +replace :: intrinsics.simd_replace + +splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { + return T{0.. ^[LANES]E { + return (^[LANES]E)(v) +} +to_array :: #force_inline proc "contextless" (v: #simd[$LANES]$E) -> [LANES]E { + return transmute([LANES]E)(v) +} +from_array :: #force_inline proc "contextless" (v: $A/[$LANES]$E) -> #simd[LANES]E where LANES & (LANES-1) == 0 { + return transmute(#simd[LANES]E)v +} + +from_slice :: proc($T: typeid/#simd[$LANES]$E, slice: []E) -> T where LANES & (LANES-1) == 0 { + assert(len(slice) >= LANES, "slice length must be a least the number of lanes") + array: [LANES]E + #no_bounds_check for i in 0.. Date: Wed, 25 May 2022 22:04:47 +0100 Subject: [PATCH 13/86] Add `intrinsics.simd_reduce_*` --- core/simd/simd.odin | 8 +++ src/check_builtin.cpp | 50 +++++++++++++++++ src/checker_builtin_procs.hpp | 16 ++++++ src/llvm_backend_proc.cpp | 102 +++++++++++++++++++++++++++++++--- 4 files changed, 167 insertions(+), 9 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 87386f91f..ad14855bd 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -34,6 +34,14 @@ ge :: intrinsics.simd_ge extract :: intrinsics.simd_extract replace :: intrinsics.simd_replace +reduce_add_ordered :: intrinsics.simd_reduce_add_ordered +reduce_mul_ordered :: intrinsics.simd_reduce_mul_ordered +reduce_min :: intrinsics.simd_reduce_min +reduce_max :: intrinsics.simd_reduce_max +reduce_and :: intrinsics.simd_reduce_and +reduce_or :: intrinsics.simd_reduce_or +reduce_xor :: intrinsics.simd_reduce_xor + splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { return T{0..args[0]); if (x.mode == Addressing_Invalid) { return false; } + + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem) && !is_type_float(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + + operand->mode = Addressing_Value; + operand->type = base_array_type(x.type); + return true; + } + + case BuiltinProc_simd_reduce_and: + case BuiltinProc_simd_reduce_or: + case BuiltinProc_simd_reduce_xor: + { + Operand x = {}; + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + + operand->mode = Addressing_Value; + operand->type = base_array_type(x.type); + return true; + } + + // case BuiltinProc_simd_rotate_left: // { // Operand x = {}; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index f5d4111bc..98cc9f284 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -149,6 +149,14 @@ BuiltinProc__simd_begin, BuiltinProc_simd_extract, BuiltinProc_simd_replace, + + BuiltinProc_simd_reduce_add_ordered, + BuiltinProc_simd_reduce_mul_ordered, + BuiltinProc_simd_reduce_min, + BuiltinProc_simd_reduce_max, + BuiltinProc_simd_reduce_and, + BuiltinProc_simd_reduce_or, + BuiltinProc_simd_reduce_xor, BuiltinProc__simd_end, // Platform specific intrinsics @@ -401,6 +409,14 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_extract"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_replace"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + + {STR_LIT("simd_reduce_add_ordered"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_reduce_mul_ordered"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_reduce_min"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_reduce_max"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_reduce_and"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_reduce_or"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_reduce_xor"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index cfb69c654..c09265e7a 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -981,7 +981,7 @@ lbValue lb_emit_call(lbProcedure *p, lbValue value, Array const &args, return result; } -lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, BuiltinProcId id) { +lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, BuiltinProcId builtin_id) { ast_node(ce, CallExpr, expr); lbModule *m = p->module; @@ -1000,7 +1000,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const LLVMOpcode op_code = cast(LLVMOpcode)0; - switch (id) { + switch (builtin_id) { case BuiltinProc_simd_add: case BuiltinProc_simd_sub: case BuiltinProc_simd_mul: @@ -1008,14 +1008,14 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_rem: arg1 = lb_build_expr(p, ce->args[1]); if (is_float) { - switch (id) { + switch (builtin_id) { case BuiltinProc_simd_add: op_code = LLVMFAdd; break; case BuiltinProc_simd_sub: op_code = LLVMFSub; break; case BuiltinProc_simd_mul: op_code = LLVMFMul; break; case BuiltinProc_simd_div: op_code = LLVMFDiv; break; } } else { - switch (id) { + switch (builtin_id) { case BuiltinProc_simd_add: op_code = LLVMAdd; break; case BuiltinProc_simd_sub: op_code = LLVMSub; break; case BuiltinProc_simd_mul: op_code = LLVMMul; break; @@ -1053,7 +1053,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const Type *elem1 = base_array_type(arg1.type); bool is_masked = false; - switch (id) { + switch (builtin_id) { case BuiltinProc_simd_shl: op_code = LLVMShl; is_masked = false; break; case BuiltinProc_simd_shr: op_code = is_signed ? LLVMAShr : LLVMLShr; is_masked = false; break; case BuiltinProc_simd_shl_masked: op_code = LLVMShl; is_masked = true; break; @@ -1086,7 +1086,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_or: case BuiltinProc_simd_xor: arg1 = lb_build_expr(p, ce->args[1]); - switch (id) { + switch (builtin_id) { case BuiltinProc_simd_and: op_code = LLVMAnd; break; case BuiltinProc_simd_or: op_code = LLVMOr; break; case BuiltinProc_simd_xor: op_code = LLVMXor; break; @@ -1144,7 +1144,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const arg1 = lb_build_expr(p, ce->args[1]); if (is_float) { LLVMRealPredicate pred = cast(LLVMRealPredicate)0; - switch (id) { + switch (builtin_id) { case BuiltinProc_simd_eq: pred = LLVMRealOEQ; break; case BuiltinProc_simd_ne: pred = LLVMRealONE; break; case BuiltinProc_simd_lt: pred = LLVMRealOLT; break; @@ -1159,7 +1159,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const } } else { LLVMIntPredicate pred = cast(LLVMIntPredicate)0; - switch (id) { + switch (builtin_id) { case BuiltinProc_simd_eq: pred = LLVMIntEQ; break; case BuiltinProc_simd_ne: pred = LLVMIntNE; break; case BuiltinProc_simd_lt: pred = is_signed ? LLVMIntSLT :LLVMIntULT; break; @@ -1184,8 +1184,92 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const arg2 = lb_build_expr(p, ce->args[2]); res.value = LLVMBuildInsertElement(p->builder, arg0.value, arg2.value, arg1.value, ""); return res; + + case BuiltinProc_simd_reduce_add_ordered: + case BuiltinProc_simd_reduce_mul_ordered: + { + LLVMTypeRef llvm_elem = lb_type(m, elem); + LLVMValueRef args[2] = {}; + isize args_count = 0; + + char const *name = nullptr; + switch (builtin_id) { + case BuiltinProc_simd_reduce_add_ordered: + if (is_float) { + name = "llvm.vector.reduce.fadd"; + args[args_count++] = LLVMConstReal(llvm_elem, 0.0); + } else { + name = "llvm.vector.reduce.add"; + } + break; + case BuiltinProc_simd_reduce_mul_ordered: + if (is_float) { + name = "llvm.vector.reduce.fmul"; + args[args_count++] = LLVMConstReal(llvm_elem, 1.0); + } else { + name = "llvm.vector.reduce.mul"; + } + break; + } + args[args_count++] = arg0.value; + + + LLVMTypeRef types[1] = {lb_type(p->module, arg0.type)}; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + lbValue res = {}; + res.value = LLVMBuildCall(p->builder, ip, args, cast(unsigned)args_count, ""); + res.type = tv.type; + return res; + } + case BuiltinProc_simd_reduce_min: + case BuiltinProc_simd_reduce_max: + case BuiltinProc_simd_reduce_and: + case BuiltinProc_simd_reduce_or: + case BuiltinProc_simd_reduce_xor: + { + char const *name = nullptr; + switch (builtin_id) { + case BuiltinProc_simd_reduce_min: + if (is_float) { + name = "llvm.vector.reduce.fmin"; + } else if (is_signed) { + name = "llvm.vector.reduce.smin"; + } else { + name = "llvm.vector.reduce.umin"; + } + break; + case BuiltinProc_simd_reduce_max: + if (is_float) { + name = "llvm.vector.reduce.fmax"; + } else if (is_signed) { + name = "llvm.vector.reduce.smax"; + } else { + name = "llvm.vector.reduce.umax"; + } + break; + case BuiltinProc_simd_reduce_and: name = "llvm.vector.reduce.and"; break; + case BuiltinProc_simd_reduce_or: name = "llvm.vector.reduce.or"; break; + case BuiltinProc_simd_reduce_xor: name = "llvm.vector.reduce.xor"; break; + } + LLVMTypeRef types[1] = {lb_type(p->module, arg0.type)}; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + LLVMValueRef args[1] = {}; + args[0] = arg0.value; + + lbValue res = {}; + res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); + res.type = tv.type; + return res; + } } - GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[id].name)); + GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[builtin_id].name)); + return {}; } From 63d6c08d9035fb3b344dc17b7667b24928a1edf7 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 22:09:38 +0100 Subject: [PATCH 14/86] Add `raw_simd_data` --- core/mem/raw.odin | 1 + core/runtime/core_builtin.odin | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/mem/raw.odin b/core/mem/raw.odin index 0a0780dfd..2bce2d7aa 100644 --- a/core/mem/raw.odin +++ b/core/mem/raw.odin @@ -21,6 +21,7 @@ make_any :: proc "contextless" (data: rawptr, id: typeid) -> any { } raw_array_data :: runtime.raw_array_data +raw_simd_data :: runtime.raw_simd_data raw_string_data :: runtime.raw_string_data raw_slice_data :: runtime.raw_slice_data raw_dynamic_array_data :: runtime.raw_dynamic_array_data diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 4ddc3928a..7cb5287c0 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -604,6 +604,10 @@ raw_array_data :: proc "contextless" (a: $P/^($T/[$N]$E)) -> [^]E { return ([^]E)(a) } @builtin +raw_simd_data :: proc "contextless" (a: $P/^($T/#simd[$N]$E)) -> [^]E { + return ([^]E)(a) +} +@builtin raw_slice_data :: proc "contextless" (s: $S/[]$E) -> [^]E { ptr := (transmute(Raw_Slice)s).data return ([^]E)(ptr) @@ -619,7 +623,7 @@ raw_string_data :: proc "contextless" (s: $S/string) -> [^]u8 { } @builtin -raw_data :: proc{raw_array_data, raw_slice_data, raw_dynamic_array_data, raw_string_data} +raw_data :: proc{raw_array_data, raw_slice_data, raw_dynamic_array_data, raw_string_data, raw_simd_data} From 808ea30b48b35d1556afbddcd49839ea9014d76e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 22:16:44 +0100 Subject: [PATCH 15/86] Allow booleans for #simd --- src/check_builtin.cpp | 79 +++++++++++++++++++++++++------------------ src/check_type.cpp | 2 +- src/types.cpp | 3 ++ 3 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index a499937b2..e4fd504b3 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -447,7 +447,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call Type *elem = base_array_type(x.type); if (!is_type_integer(elem) && !is_type_float(elem)) { gbString xs = type_to_string(x.type); - error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating point element, got '%s'", LIT(builtin_name), xs); gb_string_free(xs); return false; } @@ -485,11 +485,21 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return false; } Type *elem = base_array_type(x.type); - if (!is_type_integer(elem)) { - gbString xs = type_to_string(x.type); - error(x.expr, "'%.*s' expected a #simd type with an integer element, got '%s'", LIT(builtin_name), xs); - gb_string_free(xs); - return false; + + if (id == BuiltinProc_simd_rem) { + if (!is_type_integer(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + } else { + if (!is_type_integer(elem) && !is_type_boolean(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or boolean element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } } operand->mode = Addressing_Value; @@ -497,10 +507,10 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } - case BuiltinProc_simd_shl: - case BuiltinProc_simd_shr: - case BuiltinProc_simd_shl_masked: - case BuiltinProc_simd_shr_masked: + case BuiltinProc_simd_shl: // Odin-like + case BuiltinProc_simd_shr: // Odin-like + case BuiltinProc_simd_shl_masked: // C-like + case BuiltinProc_simd_shr_masked: // C-like { Operand x = {}; Operand y = {}; @@ -561,7 +571,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call Type *elem = base_array_type(x.type); if (!is_type_integer(elem) && !is_type_float(elem)) { gbString xs = type_to_string(x.type); - error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating point element, got '%s'", LIT(builtin_name), xs); gb_string_free(xs); return false; } @@ -592,12 +602,27 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return false; } Type *elem = base_array_type(x.type); - if (!is_type_integer(elem) && !is_type_float(elem)) { - gbString xs = type_to_string(x.type); - error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); - gb_string_free(xs); - return false; + switch (id) { + case BuiltinProc_simd_eq: + case BuiltinProc_simd_ne: + if (!is_type_integer(elem) && !is_type_float(elem) && !is_type_boolean(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer, floating point, or boolean element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + break; + default: + if (!is_type_integer(elem) && !is_type_float(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating point element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + break; } + + Type *vt = base_type(x.type); GB_ASSERT(vt->kind == Type_SimdVector); i64 count = vt->SimdVector.count; @@ -630,12 +655,6 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return false; } Type *elem = base_array_type(x.type); - if (!is_type_integer(elem) && !is_type_float(elem)) { - gbString xs = type_to_string(x.type); - error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); - gb_string_free(xs); - return false; - } i64 max_count = x.type->SimdVector.count; i64 value = -1; if (!check_index_value(c, x.type, false, ce->args[1], max_count, &value)) { @@ -661,12 +680,6 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return false; } Type *elem = base_array_type(x.type); - if (!is_type_integer(elem) && !is_type_float(elem)) { - gbString xs = type_to_string(x.type); - error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); - gb_string_free(xs); - return false; - } i64 max_count = x.type->SimdVector.count; i64 value = -1; if (!check_index_value(c, x.type, false, ce->args[1], max_count, &value)) { @@ -710,7 +723,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call Type *elem = base_array_type(x.type); if (!is_type_integer(elem) && !is_type_float(elem)) { gbString xs = type_to_string(x.type); - error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating point element, got '%s'", LIT(builtin_name), xs); gb_string_free(xs); return false; } @@ -732,9 +745,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return false; } Type *elem = base_array_type(x.type); - if (!is_type_integer(elem)) { + if (!is_type_integer(elem) && !is_type_boolean(elem)) { gbString xs = type_to_string(x.type); - error(x.expr, "'%.*s' expected a #simd type with an integer element, got '%s'", LIT(builtin_name), xs); + error(x.expr, "'%.*s' expected a #simd type with an integer or boolean element, got '%s'", LIT(builtin_name), xs); gb_string_free(xs); return false; } @@ -757,7 +770,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call // Type *elem = base_array_type(x.type); // if (!is_type_integer(elem) && !is_type_float(elem)) { // gbString xs = type_to_string(x.type); - // error(x.expr, "'%.*s' expected a #simd type with an integer or floating-point element, got '%s'", LIT(builtin_name), xs); + // error(x.expr, "'%.*s' expected a #simd type with an integer or floating point element, got '%s'", LIT(builtin_name), xs); // gb_string_free(xs); // return false; // } @@ -3102,7 +3115,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 Type *elem = y.type; if (!is_type_valid_vector_elem(elem)) { gbString str = type_to_string(elem); - error(call, "Invalid element type for 'intrinsics.simd_vector', expected an integer or float with no specific endianness, got '%s'", str); + error(call, "Invalid element type for 'intrinsics.simd_vector', expected an integer, float, or boolean with no specific endianness, got '%s'", str); gb_string_free(str); operand->mode = Addressing_Type; operand->type = t_invalid; diff --git a/src/check_type.cpp b/src/check_type.cpp index 354ab6e94..540413e32 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2797,7 +2797,7 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t } else if (name == "simd") { if (!is_type_valid_vector_elem(elem) && !is_type_polymorphic(elem)) { gbString str = type_to_string(elem); - error(at->elem, "Invalid element type for 'intrinsics.simd_vector', expected an integer or float with no specific endianness, got '%s'", str); + error(at->elem, "Invalid element type for 'intrinsics.simd_vector', expected an integer, float, or boolean with no specific endianness, got '%s'", str); gb_string_free(str); *type = alloc_type_array(elem, count, generic_type); goto array_end; diff --git a/src/types.cpp b/src/types.cpp index 2d5709b19..4fca25e52 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -1939,6 +1939,9 @@ bool is_type_valid_vector_elem(Type *t) { if (is_type_float(t)) { return true; } + if (is_type_boolean(t)) { + return true; + } } return false; } From 140c00aa0cdeac6d1149db3845cc9f3433140cf9 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 23:01:33 +0100 Subject: [PATCH 16/86] `intrinsics.simd_shuffle` --- core/simd/simd.odin | 24 +++++++++++++ src/check_builtin.cpp | 64 +++++++++++++++++++++++++++++++++++ src/check_type.cpp | 10 ++++-- src/checker_builtin_procs.hpp | 4 +++ src/llvm_backend_proc.cpp | 45 +++++++++++++++++++----- 5 files changed, 137 insertions(+), 10 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index ad14855bd..08839fd23 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -1,7 +1,28 @@ package simd +import "core:builtin" import "core:intrinsics" +// boolx16 :: #simd[16]bool +// b8x16 :: #simd[16]b8 +// b16x8 :: #simd[8]b16 +// b32x4 :: #simd[4]b32 +// b64x2 :: #simd[2]b64 + +// u8x16 :: #simd[16]u8 +// i8x16 :: #simd[16]i8 +// u16x8 :: #simd[8]u16 +// i16x8 :: #simd[8]i16 +// u32x4 :: #simd[4]u32 +// i32x4 :: #simd[4]i32 +// u64x2 :: #simd[2]u64 +// i64x2 :: #simd[2]i64 + +// f16x8 :: #simd[8]f16 +// f32x4 :: #simd[4]f32 +// f64x2 :: #simd[2]f64 + + add :: intrinsics.simd_add sub :: intrinsics.simd_sub mul :: intrinsics.simd_mul @@ -42,6 +63,9 @@ reduce_and :: intrinsics.simd_reduce_and reduce_or :: intrinsics.simd_reduce_or reduce_xor :: intrinsics.simd_reduce_xor +swizzle :: builtin.swizzle +shuffle :: intrinsics.simd_shuffle + splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { return T{0..args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &y, x.type); + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!is_type_simd_vector(y.type)) { + error(y.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!are_types_identical(x.type, y.type)) { + gbString xs = type_to_string(x.type); + gbString ys = type_to_string(y.type); + error(x.expr, "'%.*s' expected 2 arguments of the same type, got '%s' vs '%s'", LIT(builtin_name), xs, ys); + gb_string_free(ys); + gb_string_free(xs); + return false; + } + Type *elem = base_array_type(x.type); + + check_expr(c, &z, ce->args[2]); if (z.mode == Addressing_Invalid) { return false; } + Type *z_elem = base_array_type(z.type); + if (!is_type_simd_vector(z.type) || !are_types_identical(z_elem, t_u32)) { + gbString zstr = type_to_string(z.type); + error(z.expr, "'%.*s' expected a simd vector type with an element of type 'u32', got '%s'", LIT(builtin_name), zstr); + gb_string_free(zstr); + return false; + } + + i64 x_count = x.type->SimdVector.count; + i64 z_count = z.type->SimdVector.count; + + if (!is_power_of_two(z_count)) { + gbString zstr = type_to_string(z.type); + error(z.expr, "'%.*s' expected a simd vector type with a power of two length, got '%s'", LIT(builtin_name), zstr); + gb_string_free(zstr); + return false; + } + if (z_count > x_count) { + gbString zstr = type_to_string(z.type); + error(z.expr, "'%.*s' expected a simd vector type excepts the sum of the two input vectors, got '%s'", LIT(builtin_name), zstr); + gb_string_free(zstr); + return false; + } + + + operand->mode = Addressing_Value; + operand->type = alloc_type_simd_vector(z_count, elem); + return true; + } + + // case BuiltinProc_simd_rotate_left: // { // Operand x = {}; @@ -3131,6 +3189,12 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 operand->mode = Addressing_Type; operand->type = alloc_type_simd_vector(count, elem); + if (is_arch_wasm()) { + if (type_size_of(operand->type) != 16) { + error(x.expr, "wasm based targets are limited to 128-bit types"); + } + } + break; } diff --git a/src/check_type.cpp b/src/check_type.cpp index 540413e32..74fa235d5 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2802,15 +2802,21 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t *type = alloc_type_array(elem, count, generic_type); goto array_end; } + if (is_type_polymorphic(elem)) { // Ignore } else if (count < 1 || !is_power_of_two(count)) { error(at->count, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); *type = alloc_type_array(elem, count, generic_type); goto array_end; - } - + } else *type = alloc_type_simd_vector(count, elem, generic_type); + + if (is_arch_wasm()) { + if (type_size_of(*type) != 16) { + error(at->count, "wasm based targets are limited to 128-bit types"); + } + } } else { error(at->tag, "Invalid tag applied to array, got #%.*s", LIT(name)); *type = alloc_type_array(elem, count, generic_type); diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 98cc9f284..722bbec84 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -157,6 +157,8 @@ BuiltinProc__simd_begin, BuiltinProc_simd_reduce_and, BuiltinProc_simd_reduce_or, BuiltinProc_simd_reduce_xor, + + BuiltinProc_simd_shuffle, BuiltinProc__simd_end, // Platform specific intrinsics @@ -417,6 +419,8 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_reduce_and"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_reduce_or"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_reduce_xor"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + + {STR_LIT("simd_shuffle"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index c09265e7a..1b5d15d9b 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -981,6 +981,24 @@ lbValue lb_emit_call(lbProcedure *p, lbValue value, Array const &args, return result; } +LLVMValueRef llvm_splat_float(i64 count, LLVMTypeRef type, f64 value) { + LLVMValueRef v = LLVMConstReal(type, value); + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, count); + for (i64 i = 0; i < count; i++) { + values[i] = v; + } + return LLVMConstVector(values, cast(unsigned)count); +} +LLVMValueRef llvm_splat_int(i64 count, LLVMTypeRef type, i64 value, bool is_signed=false) { + LLVMValueRef v = LLVMConstInt(type, value, is_signed); + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, count); + for (i64 i = 0; i < count; i++) { + values[i] = v; + } + return LLVMConstVector(values, cast(unsigned)count); +} + + lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, BuiltinProcId builtin_id) { ast_node(ce, CallExpr, expr); @@ -1060,12 +1078,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_shr_masked: op_code = is_signed ? LLVMAShr : LLVMLShr; is_masked = true; break; } if (op_code) { - LLVMValueRef bit_value = lb_const_int(m, elem1, sz*8 - 1).value; - LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, count); - for (i64 i = 0; i < count; i++) { - values[i] = bit_value; - } - LLVMValueRef bits = LLVMConstVector(values, cast(unsigned)count); + LLVMValueRef bits = llvm_splat_int(count, lb_type(m, elem1), sz*8 - 1); if (is_masked) { // C logic LLVMValueRef shift = LLVMBuildAnd(p->builder, arg1.value, bits, ""); @@ -1077,7 +1090,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const LLVMValueRef shift = LLVMBuildBinOp(p->builder, op_code, arg0.value, arg1.value, ""); res.value = LLVMBuildSelect(p->builder, mask, shift, zero, ""); } - return res; } } @@ -1264,7 +1276,24 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const lbValue res = {}; res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); - res.type = tv.type; + return res; + } + + case BuiltinProc_simd_shuffle: + { + arg1 = lb_build_expr(p, ce->args[1]); + arg2 = lb_build_expr(p, ce->args[2]); + + Type *vt = arg0.type; + GB_ASSERT(vt->kind == Type_SimdVector); + + LLVMValueRef mask = arg2.value; + + i64 max_count = vt->SimdVector.count*2; + LLVMValueRef max_mask = llvm_splat_int(max_count, lb_type(m, arg2.type->SimdVector.elem), max_count-1); + mask = LLVMBuildAnd(p->builder, mask, max_mask, ""); + + res.value = LLVMBuildShuffleVector(p->builder, arg0.value, arg1.value, mask, ""); return res; } } From 09f936b04db2be7d30f695fe050ba57ac6d6da3d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 23:24:32 +0100 Subject: [PATCH 17/86] Correct casting between integer and boolean #simd --- src/llvm_backend_expr.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 1b10cd776..426becc1c 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -1842,10 +1842,13 @@ lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { } else { res.value = LLVMBuildSIToFP(p->builder, value.value, lb_type(m, t), ""); } - } else if (is_type_integer(src_elem) && is_type_integer(dst_elem)) { + } else if ((is_type_integer(src_elem) || is_type_boolean(src_elem)) && is_type_integer(dst_elem)) { res.value = LLVMBuildIntCast2(p->builder, value.value, lb_type(m, t), !is_type_unsigned(src_elem), ""); } else if (is_type_float(src_elem) && is_type_float(dst_elem)) { res.value = LLVMBuildFPCast(p->builder, value.value, lb_type(m, t), ""); + } else if (is_type_integer(src_elem) && is_type_boolean(dst_elem)) { + LLVMValueRef i1vector = LLVMBuildICmp(p->builder, LLVMIntNE, value.value, LLVMConstNull(LLVMTypeOf(value.value)), ""); + res.value = LLVMBuildIntCast2(p->builder, i1vector, lb_type(m, t), !is_type_unsigned(src_elem), ""); } else { GB_PANIC("Unhandled simd vector conversion: %s -> %s", type_to_string(src), type_to_string(dst)); } From 57e69ea3922ce56b3c959e60aeeaa032ab81ff8d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 23:24:42 +0100 Subject: [PATCH 18/86] Add comments --- core/simd/simd.odin | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 08839fd23..b5207e154 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -42,16 +42,25 @@ shr_masked :: intrinsics.simd_shr_masked and :: intrinsics.simd_and or :: intrinsics.simd_or xor :: intrinsics.simd_xor + neg :: intrinsics.simd_neg + abs :: intrinsics.simd_abs min :: intrinsics.simd_min max :: intrinsics.simd_max -eq :: intrinsics.simd_eq -ne :: intrinsics.simd_ne -lt :: intrinsics.simd_lt -le :: intrinsics.simd_le -gt :: intrinsics.simd_gt -ge :: intrinsics.simd_ge + +// Return an unsigned integer of the same size as the input type +// NOT A BOOLEAN +// element-wise: +// false => 0x00...00 +// true => 0xff...ff +eq :: intrinsics.simd_eq +ne :: intrinsics.simd_ne +lt :: intrinsics.simd_lt +le :: intrinsics.simd_le +gt :: intrinsics.simd_gt +ge :: intrinsics.simd_ge + extract :: intrinsics.simd_extract replace :: intrinsics.simd_replace From 7002c94a63c58aa0ac5a5d74b1fffd1511aeb699 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 23:34:41 +0100 Subject: [PATCH 19/86] Add `intrinsics.simd_select` --- core/simd/simd.odin | 1 + src/check_builtin.cpp | 51 +++++++++++++++++++++++++++++++++++ src/checker_builtin_procs.hpp | 2 ++ src/llvm_backend_proc.cpp | 12 +++++++++ 4 files changed, 66 insertions(+) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index b5207e154..1819f3951 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -74,6 +74,7 @@ reduce_xor :: intrinsics.simd_reduce_xor swizzle :: builtin.swizzle shuffle :: intrinsics.simd_shuffle +select :: intrinsics.simd_select splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { return T{0..args[0]); if (cond.mode == Addressing_Invalid) { return false; } + + if (!is_type_simd_vector(cond.type)) { + error(cond.expr, "'%.*s' expected a simd vector boolean type", LIT(builtin_name)); + return false; + } + if (!is_type_boolean(base_array_type(cond.type))) { + error(cond.expr, "'%.*s' expected a simd vector boolean type", LIT(builtin_name)); + return false; + } + + Operand x = {}; + Operand y = {}; + check_expr(c, &x, ce->args[1]); if (x.mode == Addressing_Invalid) { return false; } + check_expr_with_type_hint(c, &y, ce->args[2], x.type); if (y.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &y, x.type); + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!is_type_simd_vector(y.type)) { + error(y.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!are_types_identical(x.type, y.type)) { + gbString xs = type_to_string(x.type); + gbString ys = type_to_string(y.type); + error(x.expr, "'%.*s' expected 2 results of the same type, got '%s' vs '%s'", LIT(builtin_name), xs, ys); + gb_string_free(ys); + gb_string_free(xs); + return false; + } + + if (cond.type->SimdVector.count != x.type->SimdVector.count) { + error(x.expr, "'%.*s' expected condition vector to match the length of the result lengths, got '%lld' vs '%lld'", + LIT(builtin_name), + cast(long long)cond.type->SimdVector.count, + cast(long long)x.type->SimdVector.count); + return false; + } + + + operand->mode = Addressing_Value; + operand->type = x.type; + return true; + } + + // case BuiltinProc_simd_rotate_left: // { diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 722bbec84..2fb355e91 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -159,6 +159,7 @@ BuiltinProc__simd_begin, BuiltinProc_simd_reduce_xor, BuiltinProc_simd_shuffle, + BuiltinProc_simd_select, BuiltinProc__simd_end, // Platform specific intrinsics @@ -421,6 +422,7 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_reduce_xor"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_shuffle"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_select"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 1b5d15d9b..7a86427d4 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1296,6 +1296,18 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const res.value = LLVMBuildShuffleVector(p->builder, arg0.value, arg1.value, mask, ""); return res; } + + case BuiltinProc_simd_select: + { + LLVMValueRef cond = arg0.value; + LLVMValueRef x = lb_build_expr(p, ce->args[1]).value; + LLVMValueRef y = lb_build_expr(p, ce->args[2]).value; + + cond = LLVMBuildICmp(p->builder, LLVMIntNE, cond, LLVMConstNull(LLVMTypeOf(cond)), ""); + res.value = LLVMBuildSelect(p->builder, cond, x, y, ""); + return res; + } + } GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[builtin_id].name)); From 12d19d21c4a77bce5ff4acd8f0184e72709fb364 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 23:40:59 +0100 Subject: [PATCH 20/86] Document simd stuff in intrinsics.odin --- core/intrinsics/intrinsics.odin | 54 +++++++++++++++++++++++++++++++++ core/simd/simd.odin | 7 +++++ 2 files changed, 61 insertions(+) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index d71522936..a24f1d868 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -186,6 +186,60 @@ type_hasher_proc :: proc($T: typeid) -> (hasher: proc "contextless" (data: rawpt constant_utf16_cstring :: proc($literal: string) -> [^]u16 --- +// SIMD related +simd_add :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_sub :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_mul :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_div :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_rem :: proc(a, b: #simd[N]T) -> #simd[N]T --- + +// Keeps Odin's Behaviour +// (x << y) if y <= mask else 0 +simd_shl :: proc(a: #simd[N]T, b: #simd[N]Unsigned_Integer) -> #simd[N]T --- +simd_shr :: proc(a: #simd[N]T, b: #simd[N]Unsigned_Integer) -> #simd[N]T --- + +// Similar to C's Behaviour +// x << (y & mask) +simd_shl_masked :: proc(a: #simd[N]T, b: #simd[N]Unsigned_Integer) -> #simd[N]T --- +simd_shr_masked :: proc(a: #simd[N]T, b: #simd[N]Unsigned_Integer) -> #simd[N]T --- + +simd_and :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_or :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_xor :: proc(a, b: #simd[N]T) -> #simd[N]T --- + +simd_neg :: proc(a: #simd[N]T) -> #simd[N]T --- + +simd_abs :: proc(a: #simd[N]T) -> #simd[N]T --- +simd_min :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_max :: proc(a, b: #simd[N]T) -> #simd[N]T --- + +// Return an unsigned integer of the same size as the input type +// NOT A BOOLEAN +// element-wise: +// false => 0x00...00 +// true => 0xff...ff +simd_eq :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_ne :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_lt :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_le :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_gt :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_ge :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- + +simd_extract :: proc(a: #simd[N]T, idx: uint) -> T --- +simd_replace :: proc(a: #simd[N]T, idx: uint, elem: T) -> #simd[N]T --- + +simd_reduce_add_ordered :: proc(a: #simd[N]T) -> T --- +simd_reduce_mul_ordered :: proc(a: #simd[N]T) -> T --- +simd_reduce_min :: proc(a: #simd[N]T) -> T --- +simd_reduce_max :: proc(a: #simd[N]T) -> T --- +simd_reduce_and :: proc(a: #simd[N]T) -> T --- +simd_reduce_or :: proc(a: #simd[N]T) -> T --- +simd_reduce_xor :: proc(a: #simd[N]T) -> T --- + +simd_shuffle :: proc(a, b: #simd[N]T, indices: #simd[max 2*N]u32) -> #simd[len(indices)]T --- +simd_select :: proc(cond: #simd[N]any_boolean, true, false: #simd[N]T) -> #simd[N]T --- + + // WASM targets only wasm_memory_grow :: proc(index, delta: uintptr) -> int --- wasm_memory_size :: proc(index: uintptr) -> int --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 1819f3951..c5473a92c 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -61,7 +61,9 @@ le :: intrinsics.simd_le gt :: intrinsics.simd_gt ge :: intrinsics.simd_ge +// extract :: proc(a: #simd[N]T, idx: uint) -> T extract :: intrinsics.simd_extract +// replace :: proc(a: #simd[N]T, idx: uint, elem: T) -> #simd[N]T replace :: intrinsics.simd_replace reduce_add_ordered :: intrinsics.simd_reduce_add_ordered @@ -72,8 +74,13 @@ reduce_and :: intrinsics.simd_reduce_and reduce_or :: intrinsics.simd_reduce_or reduce_xor :: intrinsics.simd_reduce_xor +// swizzle :: proc(a: #simd[N]T, indices: ..int) -> #simd[len(indices)]T swizzle :: builtin.swizzle + +// shuffle :: proc(a, b: #simd[N]T, indices: #simd[max 2*N]u32) -> #simd[len(indices)]T shuffle :: intrinsics.simd_shuffle + +// select :: proc(cond: #simd[N]any_boolean, true, false: #simd[N]T) -> #simd[N]T select :: intrinsics.simd_select splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { From 8e57511ffa58e9198ceab9e0e458ce6f132d37d1 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 23:42:25 +0100 Subject: [PATCH 21/86] Minor clean up --- core/simd/simd.odin | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index c5473a92c..ab0d9937b 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -23,31 +23,31 @@ import "core:intrinsics" // f64x2 :: #simd[2]f64 -add :: intrinsics.simd_add -sub :: intrinsics.simd_sub -mul :: intrinsics.simd_mul -div :: intrinsics.simd_div -rem :: intrinsics.simd_rem +add :: intrinsics.simd_add +sub :: intrinsics.simd_sub +mul :: intrinsics.simd_mul +div :: intrinsics.simd_div +rem :: intrinsics.simd_rem // Keeps Odin's Behaviour // (x << y) if y <= mask else 0 -shl :: intrinsics.simd_shl -shr :: intrinsics.simd_shr +shl :: intrinsics.simd_shl +shr :: intrinsics.simd_shr // Similar to C's Behaviour // x << (y & mask) shl_masked :: intrinsics.simd_shl_masked shr_masked :: intrinsics.simd_shr_masked -and :: intrinsics.simd_and -or :: intrinsics.simd_or -xor :: intrinsics.simd_xor +and :: intrinsics.simd_and +or :: intrinsics.simd_or +xor :: intrinsics.simd_xor -neg :: intrinsics.simd_neg +neg :: intrinsics.simd_neg -abs :: intrinsics.simd_abs -min :: intrinsics.simd_min -max :: intrinsics.simd_max +abs :: intrinsics.simd_abs +min :: intrinsics.simd_min +max :: intrinsics.simd_max // Return an unsigned integer of the same size as the input type // NOT A BOOLEAN From c2f5cbdeb48e49d25dc75c1fcc02ce688dc85e26 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 25 May 2022 23:49:23 +0100 Subject: [PATCH 22/86] Allow integer vectors in select --- core/intrinsics/intrinsics.odin | 2 +- core/simd/simd.odin | 2 +- src/check_builtin.cpp | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index a24f1d868..8f2ebce13 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -237,7 +237,7 @@ simd_reduce_or :: proc(a: #simd[N]T) -> T --- simd_reduce_xor :: proc(a: #simd[N]T) -> T --- simd_shuffle :: proc(a, b: #simd[N]T, indices: #simd[max 2*N]u32) -> #simd[len(indices)]T --- -simd_select :: proc(cond: #simd[N]any_boolean, true, false: #simd[N]T) -> #simd[N]T --- +simd_select :: proc(cond: #simd[N]boolean_or_integer, true, false: #simd[N]T) -> #simd[N]T --- // WASM targets only diff --git a/core/simd/simd.odin b/core/simd/simd.odin index ab0d9937b..b26719d56 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -80,7 +80,7 @@ swizzle :: builtin.swizzle // shuffle :: proc(a, b: #simd[N]T, indices: #simd[max 2*N]u32) -> #simd[len(indices)]T shuffle :: intrinsics.simd_shuffle -// select :: proc(cond: #simd[N]any_boolean, true, false: #simd[N]T) -> #simd[N]T +// select :: proc(cond: #simd[N]boolean_or_integer, true, false: #simd[N]T) -> #simd[N]T select :: intrinsics.simd_select splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index eaf71fdab..add719280 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -824,8 +824,11 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call error(cond.expr, "'%.*s' expected a simd vector boolean type", LIT(builtin_name)); return false; } - if (!is_type_boolean(base_array_type(cond.type))) { - error(cond.expr, "'%.*s' expected a simd vector boolean type", LIT(builtin_name)); + Type *cond_elem = base_array_type(cond.type); + if (!is_type_boolean(cond_elem) && !is_type_integer(cond_elem)) { + gbString cond_str = type_to_string(cond.type); + error(cond.expr, "'%.*s' expected a simd vector boolean or integer type, got '%s'", LIT(builtin_name), cond_str); + gb_string_free(cond_str); return false; } From cde6a2f7a5e5ea0676f9732342f10169baa64c52 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 00:36:24 +0100 Subject: [PATCH 23/86] Make `simd_shuffle` act closer to `swizzle` --- core/intrinsics/intrinsics.odin | 2 +- core/simd/simd.odin | 4 +- src/check_builtin.cpp | 95 +++++++++++++++------------------ src/checker_builtin_procs.hpp | 2 +- src/llvm_backend_proc.cpp | 14 +++-- 5 files changed, 59 insertions(+), 58 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 8f2ebce13..097496d47 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -236,7 +236,7 @@ simd_reduce_and :: proc(a: #simd[N]T) -> T --- simd_reduce_or :: proc(a: #simd[N]T) -> T --- simd_reduce_xor :: proc(a: #simd[N]T) -> T --- -simd_shuffle :: proc(a, b: #simd[N]T, indices: #simd[max 2*N]u32) -> #simd[len(indices)]T --- +simd_shuffle :: proc(a, b: #simd[N]T, indices: ..int) -> #simd[len(indices)]T --- simd_select :: proc(cond: #simd[N]boolean_or_integer, true, false: #simd[N]T) -> #simd[N]T --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index b26719d56..e81f82341 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -93,11 +93,11 @@ to_array_ptr :: #force_inline proc "contextless" (v: ^#simd[$LANES]$E) -> ^[LANE to_array :: #force_inline proc "contextless" (v: #simd[$LANES]$E) -> [LANES]E { return transmute([LANES]E)(v) } -from_array :: #force_inline proc "contextless" (v: $A/[$LANES]$E) -> #simd[LANES]E where LANES & (LANES-1) == 0 { +from_array :: #force_inline proc "contextless" (v: $A/[$LANES]$E) -> #simd[LANES]E { return transmute(#simd[LANES]E)v } -from_slice :: proc($T: typeid/#simd[$LANES]$E, slice: []E) -> T where LANES & (LANES-1) == 0 { +from_slice :: proc($T: typeid/#simd[$LANES]$E, slice: []E) -> T { assert(len(slice) >= LANES, "slice length must be a least the number of lanes") array: [LANES]E #no_bounds_check for i in 0..args[0]); if (x.mode == Addressing_Invalid) { return false; } check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } convert_to_typed(c, &y, x.type); @@ -784,34 +783,53 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call } Type *elem = base_array_type(x.type); - check_expr(c, &z, ce->args[2]); if (z.mode == Addressing_Invalid) { return false; } - Type *z_elem = base_array_type(z.type); - if (!is_type_simd_vector(z.type) || !are_types_identical(z_elem, t_u32)) { - gbString zstr = type_to_string(z.type); - error(z.expr, "'%.*s' expected a simd vector type with an element of type 'u32', got '%s'", LIT(builtin_name), zstr); - gb_string_free(zstr); + i64 max_count = x.type->SimdVector.count + y.type->SimdVector.count; + + i64 arg_count = 0; + for_array(i, ce->args) { + if (i < 2) { + continue; + } + Ast *arg = ce->args[i]; + Operand op = {}; + check_expr(c, &op, arg); + if (op.mode == Addressing_Invalid) { + return false; + } + Type *arg_type = base_type(op.type); + if (!is_type_integer(arg_type) || op.mode != Addressing_Constant) { + error(op.expr, "Indices to '%.*s' must be constant integers", LIT(builtin_name)); + return false; + } + + if (big_int_is_neg(&op.value.value_integer)) { + error(op.expr, "Negative '%.*s' index", LIT(builtin_name)); + return false; + } + + BigInt mc = {}; + big_int_from_i64(&mc, max_count); + if (big_int_cmp(&mc, &op.value.value_integer) <= 0) { + error(op.expr, "'%.*s' index exceeds length", LIT(builtin_name)); + return false; + } + + arg_count++; + } + + if (arg_count > max_count) { + error(call, "Too many '%.*s' indices, %td > %td", LIT(builtin_name), arg_count, max_count); return false; } - i64 x_count = x.type->SimdVector.count; - i64 z_count = z.type->SimdVector.count; - if (!is_power_of_two(z_count)) { - gbString zstr = type_to_string(z.type); - error(z.expr, "'%.*s' expected a simd vector type with a power of two length, got '%s'", LIT(builtin_name), zstr); - gb_string_free(zstr); + if (!is_power_of_two(arg_count)) { + error(call, "'%.*s' must have a power of two index arguments, got %lld", LIT(builtin_name), cast(long long)arg_count); return false; } - if (z_count > x_count) { - gbString zstr = type_to_string(z.type); - error(z.expr, "'%.*s' expected a simd vector type excepts the sum of the two input vectors, got '%s'", LIT(builtin_name), zstr); - gb_string_free(zstr); - return false; - } - operand->mode = Addressing_Value; - operand->type = alloc_type_simd_vector(z_count, elem); + operand->type = alloc_type_simd_vector(arg_count, elem); return true; } @@ -869,36 +887,6 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call } - - // case BuiltinProc_simd_rotate_left: - // { - // Operand x = {}; - // check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } - - // if (!is_type_simd_vector(x.type)) { - // error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); - // return false; - // } - // Type *elem = base_array_type(x.type); - // if (!is_type_integer(elem) && !is_type_float(elem)) { - // gbString xs = type_to_string(x.type); - // error(x.expr, "'%.*s' expected a #simd type with an integer or floating point element, got '%s'", LIT(builtin_name), xs); - // gb_string_free(xs); - // return false; - // } - - // Operand offset = {}; - // check_expr_with_type_hint(c, &offset, ce->args[1]); if (x.mode == Addressing_Invalid) { return false; } - // convert_to_typed(c, &offset, t_int); - // if (offset.mode != Addressing_Constant) { - // error(offset.expr, "'%.*s' expected a constant integer for the offset", LIT(builtin_name)); - // return false; - // } - - // operand->mode = Addressing_Value; - // operand->type = x.type; - // return true - // } default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); } @@ -1540,6 +1528,11 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 bt->Struct.soa_kind == StructSoa_Dynamic) { mode = Addressing_Value; } + } else if (is_type_simd_vector(op_type)) { + Type *bt = base_type(op_type); + mode = Addressing_Constant; + value = exact_value_i64(bt->SimdVector.count); + type = t_untyped_integer; } if (operand->mode == Addressing_Type && mode != Addressing_Constant) { mode = Addressing_Invalid; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 2fb355e91..660208eb0 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -421,7 +421,7 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_reduce_or"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_reduce_xor"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_shuffle"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_shuffle"), 2, true, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_select"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 7a86427d4..2bffa111c 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1282,15 +1282,23 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_shuffle: { arg1 = lb_build_expr(p, ce->args[1]); - arg2 = lb_build_expr(p, ce->args[2]); Type *vt = arg0.type; GB_ASSERT(vt->kind == Type_SimdVector); - LLVMValueRef mask = arg2.value; + i64 mask_count = ce->args.count-2; i64 max_count = vt->SimdVector.count*2; - LLVMValueRef max_mask = llvm_splat_int(max_count, lb_type(m, arg2.type->SimdVector.elem), max_count-1); + + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, mask_count); + for (isize i = 0; i < max_count; i++) { + lbValue idx = lb_build_expr(p, ce->args[i+2]); + GB_ASSERT(LLVMIsConstant(idx.value)); + values[i] = idx.value; + } + LLVMValueRef mask = LLVMConstVector(values, cast(unsigned)mask_count); + + LLVMValueRef max_mask = llvm_splat_int(mask_count, lb_type(m, t_u32), max_count-1); mask = LLVMBuildAnd(p->builder, mask, max_mask, ""); res.value = LLVMBuildShuffleVector(p->builder, arg0.value, arg1.value, mask, ""); From 83d880a94a783e2c78c4ab0c1c2f1740c8ceac72 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 00:37:48 +0100 Subject: [PATCH 24/86] Remove unneeded mask --- src/llvm_backend_proc.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 2bffa111c..1ec71bc21 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1287,21 +1287,18 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const GB_ASSERT(vt->kind == Type_SimdVector); - i64 mask_count = ce->args.count-2; + i64 indices_count = ce->args.count-2; i64 max_count = vt->SimdVector.count*2; - LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, mask_count); + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, indices_count); for (isize i = 0; i < max_count; i++) { lbValue idx = lb_build_expr(p, ce->args[i+2]); GB_ASSERT(LLVMIsConstant(idx.value)); values[i] = idx.value; } - LLVMValueRef mask = LLVMConstVector(values, cast(unsigned)mask_count); + LLVMValueRef indices = LLVMConstVector(values, cast(unsigned)indices_count); - LLVMValueRef max_mask = llvm_splat_int(mask_count, lb_type(m, t_u32), max_count-1); - mask = LLVMBuildAnd(p->builder, mask, max_mask, ""); - - res.value = LLVMBuildShuffleVector(p->builder, arg0.value, arg1.value, mask, ""); + res.value = LLVMBuildShuffleVector(p->builder, arg0.value, arg1.value, indices, ""); return res; } From b95ca80f85efc3f7be292d9c751a742e43e41b2e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 00:39:34 +0100 Subject: [PATCH 25/86] Fix `simd_shuffle` --- src/llvm_backend_proc.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 1ec71bc21..05477d84b 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1289,9 +1289,10 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const i64 indices_count = ce->args.count-2; i64 max_count = vt->SimdVector.count*2; + GB_ASSERT(indices_count <= max_count); LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, indices_count); - for (isize i = 0; i < max_count; i++) { + for (isize i = 0; i < indices_count; i++) { lbValue idx = lb_build_expr(p, ce->args[i+2]); GB_ASSERT(LLVMIsConstant(idx.value)); values[i] = idx.value; From 06337129d8de636ad00e8ea64218d48c67514611 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 10:38:51 +0100 Subject: [PATCH 26/86] Remove `intrinsics.odin.simd_vector` in favour of `#simd[N]T` --- core/intrinsics/intrinsics.odin | 1 - src/check_builtin.cpp | 53 --------------------------------- src/checker_builtin_procs.hpp | 2 -- 3 files changed, 56 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 097496d47..6a1ffe5a0 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -6,7 +6,6 @@ package intrinsics is_package_imported :: proc(package_name: string) -> bool --- // Types -simd_vector :: proc($N: int, $T: typeid) -> type/#simd[N]T soa_struct :: proc($N: int, $T: typeid) -> type/#soa[N]T // Volatile diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 334202230..40933fcaa 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -3192,59 +3192,6 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 break; } - case BuiltinProc_simd_vector: { - Operand x = {}; - Operand y = {}; - x = *operand; - if (!is_type_integer(x.type) || x.mode != Addressing_Constant) { - error(call, "Expected a constant integer for 'intrinsics.simd_vector'"); - operand->mode = Addressing_Type; - operand->type = t_invalid; - return false; - } - if (big_int_is_neg(&x.value.value_integer)) { - error(call, "Negative vector element length"); - operand->mode = Addressing_Type; - operand->type = t_invalid; - return false; - } - i64 count = big_int_to_i64(&x.value.value_integer); - - check_expr_or_type(c, &y, ce->args[1]); - if (y.mode != Addressing_Type) { - error(call, "Expected a type 'intrinsics.simd_vector'"); - operand->mode = Addressing_Type; - operand->type = t_invalid; - return false; - } - Type *elem = y.type; - if (!is_type_valid_vector_elem(elem)) { - gbString str = type_to_string(elem); - error(call, "Invalid element type for 'intrinsics.simd_vector', expected an integer, float, or boolean with no specific endianness, got '%s'", str); - gb_string_free(str); - operand->mode = Addressing_Type; - operand->type = t_invalid; - return false; - } - - if (count < 1 || !is_power_of_two(count)) { - error(call, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); - operand->mode = Addressing_Type; - operand->type = t_invalid; - return false; - } - - operand->mode = Addressing_Type; - operand->type = alloc_type_simd_vector(count, elem); - if (is_arch_wasm()) { - if (type_size_of(operand->type) != 16) { - error(x.expr, "wasm based targets are limited to 128-bit types"); - } - } - - break; - } - case BuiltinProc_is_package_imported: { bool value = false; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 660208eb0..0fff70f01 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -45,7 +45,6 @@ enum BuiltinProcId { // "Intrinsics" BuiltinProc_is_package_imported, - BuiltinProc_simd_vector, BuiltinProc_soa_struct, BuiltinProc_alloca, @@ -311,7 +310,6 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { // "Intrinsics" {STR_LIT("is_package_imported"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_vector"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, // Type {STR_LIT("soa_struct"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, // Type {STR_LIT("alloca"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, From 0fd43c1a0b697ea919efdeef42427694f32692bf Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 11:02:02 +0100 Subject: [PATCH 27/86] Add simd.{sqrt, ceil, floor, trunc, nearest} --- core/intrinsics/intrinsics.odin | 6 ++++++ core/simd/simd.odin | 7 +++++++ src/check_builtin.cpp | 26 ++++++++++++++++++++++++++ src/check_type.cpp | 7 +++++-- src/checker_builtin_procs.hpp | 12 ++++++++++++ src/llvm_backend_proc.cpp | 30 +++++++++++++++++++++++++++--- src/types.cpp | 3 +++ 7 files changed, 86 insertions(+), 5 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 6a1ffe5a0..13a185da0 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -239,6 +239,12 @@ simd_shuffle :: proc(a, b: #simd[N]T, indices: ..int) -> #simd[len(indices)]T -- simd_select :: proc(cond: #simd[N]boolean_or_integer, true, false: #simd[N]T) -> #simd[N]T --- +simd_sqrt :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +simd_ceil :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +simd_floor :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +simd_trunc :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +simd_nearest :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- + // WASM targets only wasm_memory_grow :: proc(index, delta: uintptr) -> int --- wasm_memory_size :: proc(index: uintptr) -> int --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index e81f82341..9c37c380c 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -83,6 +83,13 @@ shuffle :: intrinsics.simd_shuffle // select :: proc(cond: #simd[N]boolean_or_integer, true, false: #simd[N]T) -> #simd[N]T select :: intrinsics.simd_select + +sqrt :: intrinsics.simd_sqrt +ceil :: intrinsics.simd_ceil +floor :: intrinsics.simd_floor +trunc :: intrinsics.simd_trunc +nearest :: intrinsics.simd_nearest + splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { return T{0..args[0]); if (x.mode == Addressing_Invalid) { return false; } + + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector boolean type", LIT(builtin_name)); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_float(elem)) { + gbString x_str = type_to_string(x.type); + error(x.expr, "'%.*s' expected a simd vector floating point type, got '%s'", LIT(builtin_name), x_str); + gb_string_free(x_str); + return false; + } + + operand->mode = Addressing_Value; + operand->type = x.type; + return true; + } + default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); diff --git a/src/check_type.cpp b/src/check_type.cpp index 74fa235d5..de58db054 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2797,7 +2797,7 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t } else if (name == "simd") { if (!is_type_valid_vector_elem(elem) && !is_type_polymorphic(elem)) { gbString str = type_to_string(elem); - error(at->elem, "Invalid element type for 'intrinsics.simd_vector', expected an integer, float, or boolean with no specific endianness, got '%s'", str); + error(at->elem, "Invalid element type for #simd, expected an integer, float, or boolean with no specific endianness, got '%s'", str); gb_string_free(str); *type = alloc_type_array(elem, count, generic_type); goto array_end; @@ -2806,7 +2806,7 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t if (is_type_polymorphic(elem)) { // Ignore } else if (count < 1 || !is_power_of_two(count)) { - error(at->count, "Invalid length for 'intrinsics.simd_vector', expected a power of two length, got '%lld'", cast(long long)count); + error(at->count, "Invalid length for #simd, expected a power of two length, got '%lld'", cast(long long)count); *type = alloc_type_array(elem, count, generic_type); goto array_end; } else @@ -2817,6 +2817,9 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t error(at->count, "wasm based targets are limited to 128-bit types"); } } + if (count > SIMD_ELEMENT_COUNT_MAX) { + error(at->count, "#simd support a maximum element count of %d, got %lld", SIMD_ELEMENT_COUNT_MAX, cast(long long)count); + } } else { error(at->tag, "Invalid tag applied to array, got #%.*s", LIT(name)); *type = alloc_type_array(elem, count, generic_type); diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 0fff70f01..adb4e4624 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -159,6 +159,12 @@ BuiltinProc__simd_begin, BuiltinProc_simd_shuffle, BuiltinProc_simd_select, + + BuiltinProc_simd_sqrt, + BuiltinProc_simd_ceil, + BuiltinProc_simd_floor, + BuiltinProc_simd_trunc, + BuiltinProc_simd_nearest, BuiltinProc__simd_end, // Platform specific intrinsics @@ -421,6 +427,12 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_shuffle"), 2, true, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_select"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + + {STR_LIT("simd_sqrt") , 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_ceil") , 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_floor"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_trunc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_nearest"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 05477d84b..88129ba5d 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1231,9 +1231,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); - lbValue res = {}; res.value = LLVMBuildCall(p->builder, ip, args, cast(unsigned)args_count, ""); - res.type = tv.type; return res; } case BuiltinProc_simd_reduce_min: @@ -1274,7 +1272,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const LLVMValueRef args[1] = {}; args[0] = arg0.value; - lbValue res = {}; res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); return res; } @@ -1314,6 +1311,33 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } + case BuiltinProc_simd_sqrt: + case BuiltinProc_simd_ceil: + case BuiltinProc_simd_floor: + case BuiltinProc_simd_trunc: + case BuiltinProc_simd_nearest: + { + char const *name = nullptr; + switch (builtin_id) { + case BuiltinProc_simd_sqrt: name = "llvm.sqrt"; break; + case BuiltinProc_simd_ceil: name = "llvm.ceil"; break; + case BuiltinProc_simd_floor: name = "llvm.floor"; break; + case BuiltinProc_simd_trunc: name = "llvm.trunc"; break; + case BuiltinProc_simd_nearest: name = "llvm.nearbyint"; break; + } + + LLVMTypeRef types[1] = {lb_type(p->module, arg0.type)}; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + LLVMValueRef args[1] = {}; + args[0] = arg0.value; + + res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); + return res; + } + } GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[builtin_id].name)); diff --git a/src/types.cpp b/src/types.cpp index 4fca25e52..fccea2937 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -363,6 +363,9 @@ enum : int { MATRIX_ELEMENT_COUNT_MIN = 1, MATRIX_ELEMENT_COUNT_MAX = 16, MATRIX_ELEMENT_MAX_SIZE = MATRIX_ELEMENT_COUNT_MAX * (2 * 8), // complex128 + + SIMD_ELEMENT_COUNT_MIN = 1, + SIMD_ELEMENT_COUNT_MAX = 64, }; From 7ec0236fbf55939ef46662a732b00908730f826b Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 11:14:22 +0100 Subject: [PATCH 28/86] Add `simd_reverse` --- core/intrinsics/intrinsics.odin | 2 ++ core/simd/simd.odin | 2 ++ src/check_builtin.cpp | 13 +++++++++++++ src/checker_builtin_procs.hpp | 4 ++++ src/llvm_backend_proc.cpp | 16 ++++++++++++++++ src/types.cpp | 2 ++ 6 files changed, 39 insertions(+) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 13a185da0..a14488210 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -245,6 +245,8 @@ simd_floor :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- simd_trunc :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- simd_nearest :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +simd_reverse :: proc(a: #simd[N]T) -> #simd[N]T --- + // WASM targets only wasm_memory_grow :: proc(index, delta: uintptr) -> int --- wasm_memory_size :: proc(index: uintptr) -> int --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 9c37c380c..060e32323 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -90,6 +90,8 @@ floor :: intrinsics.simd_floor trunc :: intrinsics.simd_trunc nearest :: intrinsics.simd_nearest +reverse :: intrinsics.simd_reverse + splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { return T{0..args[0]); if (x.mode == Addressing_Invalid) { return false; } + + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + operand->type = x.type; + operand->mode = Addressing_Value; + return true; + } default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index adb4e4624..22ee3d141 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -165,6 +165,8 @@ BuiltinProc__simd_begin, BuiltinProc_simd_floor, BuiltinProc_simd_trunc, BuiltinProc_simd_nearest, + + BuiltinProc_simd_reverse, BuiltinProc__simd_end, // Platform specific intrinsics @@ -433,6 +435,8 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_floor"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_trunc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_nearest"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + + {STR_LIT("simd_reverse"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 88129ba5d..42f5a60fa 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1338,6 +1338,22 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } + case BuiltinProc_simd_reverse: + { + i64 count = get_array_type_count(arg0.type); + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, count); + LLVMTypeRef llvm_u32 = lb_type(m, t_u32); + for (i64 i = 0; i < count; i++) { + values[i] = LLVMConstInt(llvm_u32, count-1-i, false); + } + LLVMValueRef mask = LLVMConstVector(values, cast(unsigned)count); + + LLVMValueRef v = arg0.value; + res.value = LLVMBuildShuffleVector(p->builder, v, v, mask, ""); + return res; + } + + } GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[builtin_id].name)); diff --git a/src/types.cpp b/src/types.cpp index fccea2937..6f61015d3 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -1598,6 +1598,8 @@ i64 get_array_type_count(Type *t) { return bt->Array.count; } else if (bt->kind == Type_EnumeratedArray) { return bt->EnumeratedArray.count; + } else if (bt->kind == Type_SimdVector) { + return bt->SimdVector.count; } GB_ASSERT(is_type_array_like(t)); return -1; From 35502816c7d53b0b5fd422537d32efb5c34b1811 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 11:24:10 +0100 Subject: [PATCH 29/86] Add `simd_add_sat` `simd_sub_sat` --- core/intrinsics/intrinsics.odin | 9 +++++++-- core/simd/simd.odin | 4 ++++ src/check_builtin.cpp | 11 +++++++++-- src/checker_builtin_procs.hpp | 7 +++++++ src/llvm_backend_proc.cpp | 24 ++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 4 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index a14488210..7bbbc1efd 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -202,6 +202,9 @@ simd_shr :: proc(a: #simd[N]T, b: #simd[N]Unsigned_Integer) -> #simd[N]T --- simd_shl_masked :: proc(a: #simd[N]T, b: #simd[N]Unsigned_Integer) -> #simd[N]T --- simd_shr_masked :: proc(a: #simd[N]T, b: #simd[N]Unsigned_Integer) -> #simd[N]T --- +simd_add_sat :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_sub_sat :: proc(a, b: #simd[N]T) -> #simd[N]T --- + simd_and :: proc(a, b: #simd[N]T) -> #simd[N]T --- simd_or :: proc(a, b: #simd[N]T) -> #simd[N]T --- simd_xor :: proc(a, b: #simd[N]T) -> #simd[N]T --- @@ -238,13 +241,15 @@ simd_reduce_xor :: proc(a: #simd[N]T) -> T --- simd_shuffle :: proc(a, b: #simd[N]T, indices: ..int) -> #simd[len(indices)]T --- simd_select :: proc(cond: #simd[N]boolean_or_integer, true, false: #simd[N]T) -> #simd[N]T --- - -simd_sqrt :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +// Lane-wise operations +simd_sqrt :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- // IEEE sqrt simd_ceil :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- simd_floor :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- simd_trunc :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +// rounding to the nearest integral value; if two values are equally near, rounds to the even one simd_nearest :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +// equivalent a swizzle with descending indices, e.g. reserve(a, 3, 2, 1, 0) simd_reverse :: proc(a: #simd[N]T) -> #simd[N]T --- // WASM targets only diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 060e32323..2ce487d13 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -39,6 +39,10 @@ shr :: intrinsics.simd_shr shl_masked :: intrinsics.simd_shl_masked shr_masked :: intrinsics.simd_shr_masked +// Saturation Arithmetic +add_sat :: intrinsics.simd_add_sat +sub_sat :: intrinsics.simd_sub_sat + and :: intrinsics.simd_and or :: intrinsics.simd_or xor :: intrinsics.simd_xor diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index edf84b152..682667972 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -458,6 +458,8 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call } // Integer only + case BuiltinProc_simd_add_sat: + case BuiltinProc_simd_sub_sat: case BuiltinProc_simd_rem: case BuiltinProc_simd_and: case BuiltinProc_simd_or: @@ -486,20 +488,25 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call } Type *elem = base_array_type(x.type); - if (id == BuiltinProc_simd_rem) { + switch (id) { + case BuiltinProc_simd_add_sat: + case BuiltinProc_simd_sub_sat: + case BuiltinProc_simd_rem: if (!is_type_integer(elem)) { gbString xs = type_to_string(x.type); error(x.expr, "'%.*s' expected a #simd type with an integer element, got '%s'", LIT(builtin_name), xs); gb_string_free(xs); return false; } - } else { + break; + default: if (!is_type_integer(elem) && !is_type_boolean(elem)) { gbString xs = type_to_string(x.type); error(x.expr, "'%.*s' expected a #simd type with an integer or boolean element, got '%s'", LIT(builtin_name), xs); gb_string_free(xs); return false; } + break; } operand->mode = Addressing_Value; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 22ee3d141..07c425cfa 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -129,6 +129,9 @@ BuiltinProc__simd_begin, BuiltinProc_simd_shl_masked, // C logic BuiltinProc_simd_shr_masked, // C logic + BuiltinProc_simd_add_sat, // saturation arithmetic + BuiltinProc_simd_sub_sat, // saturation arithmetic + BuiltinProc_simd_and, BuiltinProc_simd_or, BuiltinProc_simd_xor, @@ -402,6 +405,10 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_shr"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_shl_masked"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_shr_masked"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + + {STR_LIT("simd_add_sat"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_sub_sat"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_and"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_or"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_xor"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 42f5a60fa..dbdc51078 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1353,6 +1353,30 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } + case BuiltinProc_simd_add_sat: + case BuiltinProc_simd_sub_sat: + { + arg1 = lb_build_expr(p, ce->args[1]); + + char const *name = nullptr; + switch (builtin_id) { + case BuiltinProc_simd_add_sat: name = is_signed ? "llvm.sadd.sat" : "llvm.uadd.sat"; break; + case BuiltinProc_simd_sub_sat: name = is_signed ? "llvm.ssub.sat" : "llvm.usub.sat"; break; + } + + LLVMTypeRef types[1] = {lb_type(p->module, arg0.type)}; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + LLVMValueRef args[2] = {}; + args[0] = arg0.value; + args[1] = arg1.value; + + res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); + return res; + } + } GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[builtin_id].name)); From e331b0647e99a07b5d0f70cbac948ab30b30b5c7 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 11:48:04 +0100 Subject: [PATCH 30/86] Add `simd_rotate_left` simd_rotate_right` --- core/intrinsics/intrinsics.odin | 4 ++++ core/simd/simd.odin | 3 +++ src/check_builtin.cpp | 23 ++++++++++++++++++++++ src/checker_builtin_procs.hpp | 4 ++++ src/llvm_backend_proc.cpp | 35 +++++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 7bbbc1efd..f8e138ef6 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -252,6 +252,10 @@ simd_nearest :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- // equivalent a swizzle with descending indices, e.g. reserve(a, 3, 2, 1, 0) simd_reverse :: proc(a: #simd[N]T) -> #simd[N]T --- +simd_rotate_left :: proc(a: #simd[N]T, $offset: int) -> #simd[N]T --- +simd_rotate_right :: proc(a: #simd[N]T, $offset: int) -> #simd[N]T --- + + // WASM targets only wasm_memory_grow :: proc(index, delta: uintptr) -> int --- wasm_memory_size :: proc(index: uintptr) -> int --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 2ce487d13..5d2b5edab 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -96,6 +96,9 @@ nearest :: intrinsics.simd_nearest reverse :: intrinsics.simd_reverse +rotate_left :: intrinsics.simd_rotate_left +rotate_right :: intrinsics.simd_rotate_right + splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { return T{0..args[0]); if (x.mode == Addressing_Invalid) { return false; } + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + Operand offset = {}; + check_expr(c, &offset, ce->args[1]); if (offset.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &offset, t_i64); + if (!is_type_integer(offset.type) || offset.mode != Addressing_Constant) { + error(offset.expr, "'%.*s' expected a constant integer offset"); + return false; + } + check_assignment(c, &offset, t_i64, builtin_name); + + operand->type = x.type; + operand->mode = Addressing_Value; + return true; + } + default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); } diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 07c425cfa..13fe2822c 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -170,6 +170,8 @@ BuiltinProc__simd_begin, BuiltinProc_simd_nearest, BuiltinProc_simd_reverse, + BuiltinProc_simd_rotate_left, + BuiltinProc_simd_rotate_right, BuiltinProc__simd_end, // Platform specific intrinsics @@ -444,6 +446,8 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_nearest"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_reverse"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_rotate_left"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_rotate_right"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index dbdc51078..c433334d1 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1353,6 +1353,41 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } + case BuiltinProc_simd_rotate_left: + case BuiltinProc_simd_rotate_right: + { + + i64 count = get_array_type_count(arg0.type); + GB_ASSERT(is_power_of_two(count)); + BigInt bi_count = {}; + big_int_from_i64(&bi_count, count); + + TypeAndValue const &tv = ce->args[1]->tav; + ExactValue val = exact_value_to_integer(tv.value); + GB_ASSERT(val.kind == ExactValue_Integer); + BigInt *bi = &val.value_integer; + if (builtin_id == BuiltinProc_simd_rotate_right) { + big_int_neg(bi, bi); + } + big_int_rem(bi, bi, &bi_count); + big_int_dealloc(&bi_count); + + i64 left = big_int_to_i64(bi); + + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, count); + LLVMTypeRef llvm_u32 = lb_type(m, t_u32); + for (i64 i = 0; i < count; i++) { + u64 idx = cast(u64)(i+left) & cast(u64)(count-1); + values[i] = LLVMConstInt(llvm_u32, idx, false); + } + LLVMValueRef mask = LLVMConstVector(values, cast(unsigned)count); + + LLVMValueRef v = arg0.value; + res.value = LLVMBuildShuffleVector(p->builder, v, v, mask, ""); + return res; + } + + case BuiltinProc_simd_add_sat: case BuiltinProc_simd_sub_sat: { From f3f6c12a7cc6dca745ae40744a5220878ff9261e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 11:58:55 +0100 Subject: [PATCH 31/86] Add `simd_clamp` --- core/intrinsics/intrinsics.odin | 6 ++-- core/simd/simd.odin | 8 ++++-- src/check_builtin.cpp | 51 +++++++++++++++++++++++++++++++++ src/checker_builtin_procs.hpp | 5 ++++ src/llvm_backend_proc.cpp | 22 ++++++++++++++ 5 files changed, 87 insertions(+), 5 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index f8e138ef6..484cd945c 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -212,8 +212,10 @@ simd_xor :: proc(a, b: #simd[N]T) -> #simd[N]T --- simd_neg :: proc(a: #simd[N]T) -> #simd[N]T --- simd_abs :: proc(a: #simd[N]T) -> #simd[N]T --- -simd_min :: proc(a, b: #simd[N]T) -> #simd[N]T --- -simd_max :: proc(a, b: #simd[N]T) -> #simd[N]T --- + +simd_min :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_max :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_clamp :: proc(v, min, max: #simd[N]T) -> #simd[N]T --- // Return an unsigned integer of the same size as the input type // NOT A BOOLEAN diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 5d2b5edab..1da0bd3e5 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -49,9 +49,11 @@ xor :: intrinsics.simd_xor neg :: intrinsics.simd_neg -abs :: intrinsics.simd_abs -min :: intrinsics.simd_min -max :: intrinsics.simd_max +abs :: intrinsics.simd_abs + +min :: intrinsics.simd_min +max :: intrinsics.simd_max +clamp :: intrinsics.simd_clamp // Return an unsigned integer of the same size as the input type // NOT A BOOLEAN diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 54bede3a8..45c9c93c5 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -956,6 +956,57 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } + case BuiltinProc_simd_clamp: + { + Operand x = {}; + Operand y = {}; + Operand z = {}; + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } + check_expr_with_type_hint(c, &z, ce->args[2], x.type); if (z.mode == Addressing_Invalid) { return false; } + convert_to_typed(c, &y, x.type); + convert_to_typed(c, &z, x.type); + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!is_type_simd_vector(y.type)) { + error(y.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!is_type_simd_vector(z.type)) { + error(z.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + if (!are_types_identical(x.type, y.type)) { + gbString xs = type_to_string(x.type); + gbString ys = type_to_string(y.type); + error(x.expr, "'%.*s' expected 2 arguments of the same type, got '%s' vs '%s'", LIT(builtin_name), xs, ys); + gb_string_free(ys); + gb_string_free(xs); + return false; + } + if (!are_types_identical(x.type, z.type)) { + gbString xs = type_to_string(x.type); + gbString zs = type_to_string(z.type); + error(x.expr, "'%.*s' expected 2 arguments of the same type, got '%s' vs '%s'", LIT(builtin_name), xs, zs); + gb_string_free(zs); + gb_string_free(xs); + return false; + } + Type *elem = base_array_type(x.type); + if (!is_type_integer(elem) && !is_type_float(elem)) { + gbString xs = type_to_string(x.type); + error(x.expr, "'%.*s' expected a #simd type with an integer or floating point element, got '%s'", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + + operand->mode = Addressing_Value; + operand->type = x.type; + return true; + } + default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); } diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 13fe2822c..eea4d0a8b 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -141,6 +141,7 @@ BuiltinProc__simd_begin, BuiltinProc_simd_min, BuiltinProc_simd_max, + BuiltinProc_simd_clamp, BuiltinProc_simd_eq, BuiltinProc_simd_ne, @@ -415,9 +416,13 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_or"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_xor"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_neg"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_abs"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_min"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_max"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_clamp"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_eq"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_ne"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_lt"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index c433334d1..97bb02ba3 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1412,6 +1412,28 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } + case BuiltinProc_simd_clamp: + { + arg1 = lb_build_expr(p, ce->args[1]); + arg2 = lb_build_expr(p, ce->args[2]); + + LLVMValueRef v = arg0.value; + LLVMValueRef min = arg1.value; + LLVMValueRef max = arg2.value; + + if (is_float) { + v = LLVMBuildSelect(p->builder, LLVMBuildFCmp(p->builder, LLVMRealOLT, v, min, ""), min, v, ""); + res.value = LLVMBuildSelect(p->builder, LLVMBuildFCmp(p->builder, LLVMRealOGT, v, max, ""), max, v, ""); + } else if (is_signed) { + v = LLVMBuildSelect(p->builder, LLVMBuildICmp(p->builder, LLVMIntSLT, v, min, ""), min, v, ""); + res.value = LLVMBuildSelect(p->builder, LLVMBuildICmp(p->builder, LLVMIntSGT, v, max, ""), max, v, ""); + } else { + v = LLVMBuildSelect(p->builder, LLVMBuildICmp(p->builder, LLVMIntULT, v, min, ""), min, v, ""); + res.value = LLVMBuildSelect(p->builder, LLVMBuildICmp(p->builder, LLVMIntUGT, v, max, ""), max, v, ""); + } + return res; + } + } GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[builtin_id].name)); From 66b5a35ec352b74e66ad866640669d920e9d0849 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 13:45:47 +0100 Subject: [PATCH 32/86] Add `simd_to_bits`; correct fix typo causing issue with parapoly --- src/check_builtin.cpp | 27 +++++++++++++++++++++++++++ src/check_decl.cpp | 8 ++++---- src/check_type.cpp | 7 ++++--- src/checker_builtin_procs.hpp | 4 ++++ src/llvm_backend_proc.cpp | 5 +++++ src/parser.cpp | 14 +++++++++++++- 6 files changed, 57 insertions(+), 8 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 45c9c93c5..c432d6080 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -1007,6 +1007,33 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } + case BuiltinProc_simd_to_bits: + { + Operand x = {}; + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + + if (!is_type_simd_vector(x.type)) { + error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); + return false; + } + Type *elem = base_array_type(x.type); + i64 count = get_array_type_count(x.type); + i64 sz = type_size_of(elem); + Type *bit_elem = nullptr; + switch (sz) { + case 1: bit_elem = t_u8; break; + case 2: bit_elem = t_u16; break; + case 4: bit_elem = t_u32; break; + case 8: bit_elem = t_u64; break; + } + GB_ASSERT(bit_elem != nullptr); + + operand->type = alloc_type_simd_vector(count, bit_elem); + operand->mode = Addressing_Value; + return true; + } + + default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); } diff --git a/src/check_decl.cpp b/src/check_decl.cpp index d8cad2ce1..62a1e2555 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -1315,20 +1315,20 @@ void check_proc_group_decl(CheckerContext *ctx, Entity *&pg_entity, DeclInfo *d) if (!both_have_where_clauses) switch (kind) { case ProcOverload_Identical: - error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); + error(p->token, "Overloaded procedure '%.*s' has the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); is_invalid = true; break; // case ProcOverload_CallingConvention: - // error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); + // error(p->token, "Overloaded procedure '%.*s' has the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); // is_invalid = true; // break; case ProcOverload_ParamVariadic: - error(p->token, "Overloaded procedure '%.*s' as the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); + error(p->token, "Overloaded procedure '%.*s' has the same type as another procedure in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); is_invalid = true; break; case ProcOverload_ResultCount: case ProcOverload_ResultTypes: - error(p->token, "Overloaded procedure '%.*s' as the same parameters but different results in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); + error(p->token, "Overloaded procedure '%.*s' has the same parameters but different results in the procedure group '%.*s'", LIT(name), LIT(proc_group_name)); is_invalid = true; break; case ProcOverload_Polymorphic: diff --git a/src/check_type.cpp b/src/check_type.cpp index de58db054..f84c15a19 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -1234,7 +1234,7 @@ bool check_type_specialization_to(CheckerContext *ctx, Type *specialization, Typ } -Type *determine_type_from_polymorphic(CheckerContext *ctx, Type *poly_type, Operand operand) { +Type *determine_type_from_polymorphic(CheckerContext *ctx, Type *poly_type, Operand const &operand) { bool modify_type = !ctx->no_polymorphic_errors; bool show_error = modify_type && !ctx->hide_polymorphic_errors; if (!is_operand_value(operand)) { @@ -2803,13 +2803,14 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t goto array_end; } - if (is_type_polymorphic(elem)) { + if (generic_type != nullptr) { // Ignore } else if (count < 1 || !is_power_of_two(count)) { error(at->count, "Invalid length for #simd, expected a power of two length, got '%lld'", cast(long long)count); *type = alloc_type_array(elem, count, generic_type); goto array_end; - } else + } + *type = alloc_type_simd_vector(count, elem, generic_type); if (is_arch_wasm()) { diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index eea4d0a8b..350213de2 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -170,6 +170,8 @@ BuiltinProc__simd_begin, BuiltinProc_simd_trunc, BuiltinProc_simd_nearest, + BuiltinProc_simd_to_bits, + BuiltinProc_simd_reverse, BuiltinProc_simd_rotate_left, BuiltinProc_simd_rotate_right, @@ -450,6 +452,8 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_trunc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_nearest"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_to_bits"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_reverse"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_rotate_left"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_rotate_right"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 97bb02ba3..a56aa862a 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1434,6 +1434,11 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } + case BuiltinProc_simd_to_bits: + { + res.value = LLVMBuildBitCast(p->builder, arg0.value, lb_type(m, tv.type), ""); + return res; + } } GB_PANIC("Unhandled simd intrinsic: '%.*s'", LIT(builtin_procs[builtin_id].name)); diff --git a/src/parser.cpp b/src/parser.cpp index d19e249e5..5280fd4b0 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -360,6 +360,7 @@ Ast *clone_ast(Ast *node) { case Ast_ArrayType: n->ArrayType.count = clone_ast(n->ArrayType.count); n->ArrayType.elem = clone_ast(n->ArrayType.elem); + n->ArrayType.tag = clone_ast(n->ArrayType.tag); break; case Ast_DynamicArrayType: n->DynamicArrayType.elem = clone_ast(n->DynamicArrayType.elem); @@ -2127,7 +2128,18 @@ Ast *parse_operand(AstFile *f, bool lhs) { Token name = expect_token(f, Token_Ident); if (name.string == "type") { return ast_helper_type(f, token, parse_type(f)); - } else if (name.string == "soa" || name.string == "simd") { + } else if ( name.string == "simd") { + Ast *tag = ast_basic_directive(f, token, name); + Ast *original_type = parse_type(f); + Ast *type = unparen_expr(original_type); + switch (type->kind) { + case Ast_ArrayType: type->ArrayType.tag = tag; break; + default: + syntax_error(type, "Expected a fixed array type after #%.*s, got %.*s", LIT(name.string), LIT(ast_strings[type->kind])); + break; + } + return original_type; + } else if (name.string == "soa") { Ast *tag = ast_basic_directive(f, token, name); Ast *original_type = parse_type(f); Ast *type = unparen_expr(original_type); From 59e9df2609ef3e22b4ea0eca158b736c3eba4a15 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 13:49:27 +0100 Subject: [PATCH 33/86] `simd.bit_not`; `simd.copysign` --- core/simd/simd.odin | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 1da0bd3e5..6df30fda7 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -27,7 +27,7 @@ add :: intrinsics.simd_add sub :: intrinsics.simd_sub mul :: intrinsics.simd_mul div :: intrinsics.simd_div -rem :: intrinsics.simd_rem +rem :: intrinsics.simd_rem // integers only // Keeps Odin's Behaviour // (x << y) if y <= mask else 0 @@ -96,6 +96,8 @@ floor :: intrinsics.simd_floor trunc :: intrinsics.simd_trunc nearest :: intrinsics.simd_nearest +to_bits :: intrinsics.simd_to_bits + reverse :: intrinsics.simd_reverse rotate_left :: intrinsics.simd_rotate_left @@ -123,3 +125,15 @@ from_slice :: proc($T: typeid/#simd[$LANES]$E, slice: []E) -> T { } return transmute(T)array } + +bit_not :: #force_inline proc "contextless" (v: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_integer(E) { + ones := splat(type_of(v), ~E(0)) + return xor(v, ones) +} + +copysign :: #force_inline proc "contextless" (v, sign: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_float(E) { + neg_zero := to_bits(splat(T, E(-0.0))) + sign_bit := and(to_bits(sign), neg_zero) + magnitude := and(to_bits(v), bit_not(neg_zero)) + return transmute(T)or(sign_bit, magnitude) +} \ No newline at end of file From c2610cb75ed81d687a4e6962283f498e94773706 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 13:56:35 +0100 Subject: [PATCH 34/86] Keep -vet happy --- core/intrinsics/intrinsics.odin | 2 ++ core/simd/simd.odin | 2 +- examples/all/all_main.odin | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 484cd945c..a30bb109f 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -251,6 +251,8 @@ simd_trunc :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- // rounding to the nearest integral value; if two values are equally near, rounds to the even one simd_nearest :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- +simd_to_bits :: proc(v: #simd[N]T) -> #simd[N]Integer where size_of(T) == size_of(Integer), type_is_unsigned(Integer) --- + // equivalent a swizzle with descending indices, e.g. reserve(a, 3, 2, 1, 0) simd_reverse :: proc(a: #simd[N]T) -> #simd[N]T --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 6df30fda7..79d26c845 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -136,4 +136,4 @@ copysign :: #force_inline proc "contextless" (v, sign: $T/#simd[$LANES]$E) -> T sign_bit := and(to_bits(sign), neg_zero) magnitude := and(to_bits(v), bit_not(neg_zero)) return transmute(T)or(sign_bit, magnitude) -} \ No newline at end of file +} diff --git a/examples/all/all_main.odin b/examples/all/all_main.odin index 94a841a26..1ab242305 100644 --- a/examples/all/all_main.odin +++ b/examples/all/all_main.odin @@ -193,6 +193,7 @@ _ :: slashpath _ :: filepath _ :: reflect _ :: runtime +_ :: simd _ :: slice _ :: sort _ :: strconv From f308f37ba112ca361715e470d513749236da026d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 14:51:50 +0100 Subject: [PATCH 35/86] Remove need for `simd.splat` --- core/simd/simd.odin | 9 ++--- src/check_expr.cpp | 8 +++++ src/llvm_backend_const.cpp | 26 +++++++++++--- src/llvm_backend_expr.cpp | 72 ++++++++++++++++++++++++-------------- 4 files changed, 76 insertions(+), 39 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 79d26c845..ef5fdc70b 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -103,10 +103,6 @@ reverse :: intrinsics.simd_reverse rotate_left :: intrinsics.simd_rotate_left rotate_right :: intrinsics.simd_rotate_right -splat :: #force_inline proc "contextless" ($T: typeid/#simd[$LANES]$E, value: E) -> T { - return T{0.. ^[LANES]E { return (^[LANES]E)(v) } @@ -127,12 +123,11 @@ from_slice :: proc($T: typeid/#simd[$LANES]$E, slice: []E) -> T { } bit_not :: #force_inline proc "contextless" (v: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_integer(E) { - ones := splat(type_of(v), ~E(0)) - return xor(v, ones) + return xor(v, T(~E(0))) } copysign :: #force_inline proc "contextless" (v, sign: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_float(E) { - neg_zero := to_bits(splat(T, E(-0.0))) + neg_zero := to_bits(T(-0.0)) sign_bit := and(to_bits(sign), neg_zero) magnitude := and(to_bits(v), bit_not(neg_zero)) return transmute(T)or(sign_bit, magnitude) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index fcd7818bc..2a3b5bf02 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -2713,6 +2713,14 @@ bool check_is_castable_to(CheckerContext *c, Operand *operand, Type *y) { return check_is_castable_to(c, &x, elem_dst); } + if (is_type_simd_vector(dst)) { + Type *elem = base_array_type(dst); + if (check_is_castable_to(c, operand, elem)) { + return true; + } + } + + return false; } diff --git a/src/llvm_backend_const.cpp b/src/llvm_backend_const.cpp index 3a3067dbc..bd76400de 100644 --- a/src/llvm_backend_const.cpp +++ b/src/llvm_backend_const.cpp @@ -495,9 +495,9 @@ lbValue lb_const_value(lbModule *m, Type *type, ExactValue value, bool allow_loc res.value = data; return res; } else if (is_type_array(type) && - value.kind != ExactValue_Invalid && - value.kind != ExactValue_String && - value.kind != ExactValue_Compound) { + value.kind != ExactValue_Invalid && + value.kind != ExactValue_String && + value.kind != ExactValue_Compound) { i64 count = type->Array.count; Type *elem = type->Array.elem; @@ -513,8 +513,8 @@ lbValue lb_const_value(lbModule *m, Type *type, ExactValue value, bool allow_loc res.value = llvm_const_array(lb_type(m, elem), elems, cast(unsigned)count); return res; } else if (is_type_matrix(type) && - value.kind != ExactValue_Invalid && - value.kind != ExactValue_Compound) { + value.kind != ExactValue_Invalid && + value.kind != ExactValue_Compound) { i64 row = type->Matrix.row_count; i64 column = type->Matrix.column_count; GB_ASSERT(row == column); @@ -537,6 +537,22 @@ lbValue lb_const_value(lbModule *m, Type *type, ExactValue value, bool allow_loc res.value = LLVMConstArray(lb_type(m, elem), elems, cast(unsigned)total_elem_count); return res; + } else if (is_type_simd_vector(type) && + value.kind != ExactValue_Invalid && + value.kind != ExactValue_Compound) { + i64 count = type->SimdVector.count; + Type *elem = type->SimdVector.elem; + + lbValue single_elem = lb_const_value(m, elem, value, allow_local); + single_elem.value = llvm_const_cast(single_elem.value, lb_type(m, elem)); + + LLVMValueRef *elems = gb_alloc_array(permanent_allocator(), LLVMValueRef, count); + for (i64 i = 0; i < count; i++) { + elems[i] = single_elem.value; + } + + res.value = LLVMConstVector(elems, cast(unsigned)count); + return res; } switch (value.kind) { diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 426becc1c..10c337650 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -1820,41 +1820,59 @@ lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { return res; } - if (is_type_simd_vector(src) && is_type_simd_vector(dst)) { - Type *src_elem = core_array_type(src); - Type *dst_elem = core_array_type(dst); + if (is_type_simd_vector(dst)) { + Type *et = base_array_type(dst); + if (is_type_simd_vector(src)) { + Type *src_elem = core_array_type(src); + Type *dst_elem = core_array_type(dst); - GB_ASSERT(src->SimdVector.count == dst->SimdVector.count); + GB_ASSERT(src->SimdVector.count == dst->SimdVector.count); - lbValue res = {}; - res.type = t; - if (are_types_identical(src_elem, dst_elem)) { - res.value = value.value; - } else if (is_type_float(src_elem) && is_type_integer(dst_elem)) { - if (is_type_unsigned(dst_elem)) { - res.value = LLVMBuildFPToUI(p->builder, value.value, lb_type(m, t), ""); + lbValue res = {}; + res.type = t; + if (are_types_identical(src_elem, dst_elem)) { + res.value = value.value; + } else if (is_type_float(src_elem) && is_type_integer(dst_elem)) { + if (is_type_unsigned(dst_elem)) { + res.value = LLVMBuildFPToUI(p->builder, value.value, lb_type(m, t), ""); + } else { + res.value = LLVMBuildFPToSI(p->builder, value.value, lb_type(m, t), ""); + } + } else if (is_type_integer(src_elem) && is_type_float(dst_elem)) { + if (is_type_unsigned(src_elem)) { + res.value = LLVMBuildUIToFP(p->builder, value.value, lb_type(m, t), ""); + } else { + res.value = LLVMBuildSIToFP(p->builder, value.value, lb_type(m, t), ""); + } + } else if ((is_type_integer(src_elem) || is_type_boolean(src_elem)) && is_type_integer(dst_elem)) { + res.value = LLVMBuildIntCast2(p->builder, value.value, lb_type(m, t), !is_type_unsigned(src_elem), ""); + } else if (is_type_float(src_elem) && is_type_float(dst_elem)) { + res.value = LLVMBuildFPCast(p->builder, value.value, lb_type(m, t), ""); + } else if (is_type_integer(src_elem) && is_type_boolean(dst_elem)) { + LLVMValueRef i1vector = LLVMBuildICmp(p->builder, LLVMIntNE, value.value, LLVMConstNull(LLVMTypeOf(value.value)), ""); + res.value = LLVMBuildIntCast2(p->builder, i1vector, lb_type(m, t), !is_type_unsigned(src_elem), ""); } else { - res.value = LLVMBuildFPToSI(p->builder, value.value, lb_type(m, t), ""); + GB_PANIC("Unhandled simd vector conversion: %s -> %s", type_to_string(src), type_to_string(dst)); } - } else if (is_type_integer(src_elem) && is_type_float(dst_elem)) { - if (is_type_unsigned(src_elem)) { - res.value = LLVMBuildUIToFP(p->builder, value.value, lb_type(m, t), ""); - } else { - res.value = LLVMBuildSIToFP(p->builder, value.value, lb_type(m, t), ""); - } - } else if ((is_type_integer(src_elem) || is_type_boolean(src_elem)) && is_type_integer(dst_elem)) { - res.value = LLVMBuildIntCast2(p->builder, value.value, lb_type(m, t), !is_type_unsigned(src_elem), ""); - } else if (is_type_float(src_elem) && is_type_float(dst_elem)) { - res.value = LLVMBuildFPCast(p->builder, value.value, lb_type(m, t), ""); - } else if (is_type_integer(src_elem) && is_type_boolean(dst_elem)) { - LLVMValueRef i1vector = LLVMBuildICmp(p->builder, LLVMIntNE, value.value, LLVMConstNull(LLVMTypeOf(value.value)), ""); - res.value = LLVMBuildIntCast2(p->builder, i1vector, lb_type(m, t), !is_type_unsigned(src_elem), ""); + return res; } else { - GB_PANIC("Unhandled simd vector conversion: %s -> %s", type_to_string(src), type_to_string(dst)); + i64 count = get_array_type_count(dst); + LLVMTypeRef vt = lb_type(m, t); + LLVMTypeRef llvm_u32 = lb_type(m, t_u32); + LLVMValueRef elem = lb_emit_conv(p, value, et).value; + LLVMValueRef vector = LLVMConstNull(vt); + for (i64 i = 0; i < count; i++) { + LLVMValueRef idx = LLVMConstInt(llvm_u32, i, false); + vector = LLVMBuildInsertElement(p->builder, vector, elem, idx, ""); + } + lbValue res = {}; + res.type = t; + res.value = vector; + return res; } - return res; } + // Pointer <-> uintptr if (is_type_pointer(src) && is_type_uintptr(dst)) { lbValue res = {}; From 208226dba29d46514c8c2b7a8fcd023f1ebf7bd8 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 14:55:10 +0100 Subject: [PATCH 36/86] Improve `#simd` literal support --- src/check_builtin.cpp | 76 +++++++++++++++++++++---------------------- src/check_expr.cpp | 9 +++++ 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index c432d6080..34b7d14e9 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -425,9 +425,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call { Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } - check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) return false; + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -467,9 +467,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call { Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } - check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) return false; + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -521,9 +521,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call { Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } - check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) return false; + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -601,9 +601,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } - check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) return false; + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -655,7 +655,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_extract: { Operand x = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); @@ -680,7 +680,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_replace: { Operand x = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); @@ -698,8 +698,8 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call } Operand y = {}; - check_expr_with_type_hint(c, &y, ce->args[2], elem); if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, elem); + check_expr_with_type_hint(c, &y, ce->args[2], elem); if (y.mode == Addressing_Invalid) return false; + convert_to_typed(c, &y, elem); if (y.mode == Addressing_Invalid) return false; if (!are_types_identical(y.type, elem)) { gbString et = type_to_string(elem); gbString yt = type_to_string(y.type); @@ -721,7 +721,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_reduce_max: { Operand x = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); @@ -745,7 +745,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_reduce_xor: { Operand x = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); @@ -769,9 +769,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call { Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } - check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) return false; + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -843,7 +843,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_select: { Operand cond = {}; - check_expr(c, &cond, ce->args[0]); if (cond.mode == Addressing_Invalid) { return false; } + check_expr(c, &cond, ce->args[0]); if (cond.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(cond.type)) { error(cond.expr, "'%.*s' expected a simd vector boolean type", LIT(builtin_name)); @@ -859,9 +859,9 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call Operand x = {}; Operand y = {}; - check_expr(c, &x, ce->args[1]); if (x.mode == Addressing_Invalid) { return false; } - check_expr_with_type_hint(c, &y, ce->args[2], x.type); if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + check_expr(c, &x, ce->args[1]); if (x.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &y, ce->args[2], x.type); if (y.mode == Addressing_Invalid) return false; + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; @@ -900,7 +900,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_nearest: { Operand x = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector boolean type", LIT(builtin_name)); @@ -922,7 +922,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_reverse: { Operand x = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); @@ -937,13 +937,13 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_rotate_right: { Operand x = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); return false; } Operand offset = {}; - check_expr(c, &offset, ce->args[1]); if (offset.mode == Addressing_Invalid) { return false; } + check_expr(c, &offset, ce->args[1]); if (offset.mode == Addressing_Invalid) return false; convert_to_typed(c, &offset, t_i64); if (!is_type_integer(offset.type) || offset.mode != Addressing_Constant) { error(offset.expr, "'%.*s' expected a constant integer offset"); @@ -961,10 +961,10 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call Operand x = {}; Operand y = {}; Operand z = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } - check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) { return false; } - check_expr_with_type_hint(c, &z, ce->args[2], x.type); if (z.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &y, ce->args[1], x.type); if (y.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &z, ce->args[2], x.type); if (z.mode == Addressing_Invalid) return false; + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; convert_to_typed(c, &z, x.type); if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); @@ -1010,7 +1010,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_to_bits: { Operand x = {}; - check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) { return false; } + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; if (!is_type_simd_vector(x.type)) { error(x.expr, "'%.*s' expected a simd vector type", LIT(builtin_name)); @@ -2933,7 +2933,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 if (i == j) continue; Operand *b = ops[j]; convert_to_typed(c, a, b->type); - if (a->mode == Addressing_Invalid) { return false; } + if (a->mode == Addressing_Invalid) return false; } } @@ -3616,7 +3616,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; convert_to_typed(c, &x, y.type); if (is_type_untyped(x.type)) { gbString xts = type_to_string(x.type); @@ -4232,7 +4232,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 if (x.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; if (x.mode == Addressing_Invalid) { return false; } @@ -4289,7 +4289,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 if (y.mode == Addressing_Invalid) { return false; } - convert_to_typed(c, &y, x.type); + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; convert_to_typed(c, &x, y.type); if (!are_types_identical(x.type, y.type)) { gbString xts = type_to_string(x.type); diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 2a3b5bf02..b7568aa70 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -777,6 +777,14 @@ i64 check_distance_between_types(CheckerContext *c, Operand *operand, Type *type return distance + 6; } } + + if (is_type_simd_vector(dst)) { + Type *dst_elem = base_array_type(dst); + i64 distance = check_distance_between_types(c, operand, dst_elem); + if (distance >= 0) { + return distance + 6; + } + } if (is_type_matrix(dst)) { Type *dst_elem = base_array_type(dst); @@ -786,6 +794,7 @@ i64 check_distance_between_types(CheckerContext *c, Operand *operand, Type *type } } + if (is_type_any(dst)) { if (!is_type_polymorphic(src)) { if (operand->mode == Addressing_Context && operand->type == t_context) { From d0e8a735bae52eaa7d1d852953da721071571020 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 17:09:46 +0100 Subject: [PATCH 37/86] Add arithmetic operator support for simd vectors; Add `intrinsics.simd_and_not` --- core/simd/simd.odin | 16 +++++++++++--- src/check_builtin.cpp | 1 + src/check_expr.cpp | 7 ++---- src/checker_builtin_procs.hpp | 3 +++ src/llvm_backend_expr.cpp | 40 ++++++++++++++++++++++++++++++++++- src/llvm_backend_proc.cpp | 4 ++++ src/types.cpp | 3 +++ 7 files changed, 65 insertions(+), 9 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index ef5fdc70b..9673b5cc9 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -43,9 +43,10 @@ shr_masked :: intrinsics.simd_shr_masked add_sat :: intrinsics.simd_add_sat sub_sat :: intrinsics.simd_sub_sat -and :: intrinsics.simd_and -or :: intrinsics.simd_or -xor :: intrinsics.simd_xor +and :: intrinsics.simd_and +or :: intrinsics.simd_or +xor :: intrinsics.simd_xor +and_not :: intrinsics.simd_and_not neg :: intrinsics.simd_neg @@ -132,3 +133,12 @@ copysign :: #force_inline proc "contextless" (v, sign: $T/#simd[$LANES]$E) -> T magnitude := and(to_bits(v), bit_not(neg_zero)) return transmute(T)or(sign_bit, magnitude) } + +signum :: #force_inline proc "contextless" (v: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_float(E) { + is_nan := ne(v, v) + return select(is_nan, v, copysign(T(1), v)) +} + +recip :: #force_inline proc "contextless" (v: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_float(E) { + return div(T(1), v) +} diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 34b7d14e9..bfaa91cf8 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -464,6 +464,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call case BuiltinProc_simd_and: case BuiltinProc_simd_or: case BuiltinProc_simd_xor: + case BuiltinProc_simd_and_not: { Operand x = {}; Operand y = {}; diff --git a/src/check_expr.cpp b/src/check_expr.cpp index b7568aa70..7fe9b8acf 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -1590,11 +1590,6 @@ bool check_unary_op(CheckerContext *c, Operand *o, Token op) { bool check_binary_op(CheckerContext *c, Operand *o, Token op) { Type *main_type = o->type; - if (is_type_simd_vector(main_type)) { - error(op, "Operator '%.*s' is not supported on #simd vector types, please use the intrinsics.simd_*", LIT(op.string)); - return false; - } - // TODO(bill): Handle errors correctly Type *type = base_type(core_array_type(main_type)); Type *ct = core_type(type); @@ -2500,6 +2495,8 @@ void check_shift(CheckerContext *c, Operand *x, Operand *y, Ast *node, Type *typ gb_string_free(err_str); } + // TODO(bill): Should we support shifts for fixed arrays and #simd vectors? + if (!is_type_integer(x->type)) { gbString err_str = expr_to_string(y->expr); error(node, "Shift operand '%s' must be an integer", err_str); diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 350213de2..eb4bc1498 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -135,6 +135,7 @@ BuiltinProc__simd_begin, BuiltinProc_simd_and, BuiltinProc_simd_or, BuiltinProc_simd_xor, + BuiltinProc_simd_and_not, BuiltinProc_simd_neg, BuiltinProc_simd_abs, @@ -417,6 +418,8 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_and"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_or"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_xor"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_and_not"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_neg"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_abs"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 10c337650..55b76b93a 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -258,7 +258,13 @@ lbValue lb_emit_unary_arith(lbProcedure *p, TokenKind op, lbValue x, Type *type) LLVMBuildStore(p->builder, v2, LLVMBuildStructGEP(p->builder, addr.addr.value, 2, "")); LLVMBuildStore(p->builder, v3, LLVMBuildStructGEP(p->builder, addr.addr.value, 3, "")); return lb_addr_load(p, addr); - + } else if (is_type_simd_vector(x.type)) { + Type *elem = base_array_type(x.type); + if (is_type_float(elem)) { + res.value = LLVMBuildFNeg(p->builder, x.value, ""); + } else { + res.value = LLVMBuildNeg(p->builder, x.value, ""); + } } else { GB_PANIC("Unhandled type %s", type_to_string(x.type)); } @@ -2559,6 +2565,38 @@ lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left, lbValue ri case Token_NotEq: pred = LLVMIntNE; break; } res.value = LLVMBuildICmp(p->builder, pred, left.value, right.value, ""); + } else if (is_type_simd_vector(a)) { + LLVMValueRef mask = nullptr; + Type *elem = base_array_type(a); + if (is_type_float(elem)) { + LLVMRealPredicate pred = {}; + switch (op_kind) { + case Token_CmpEq: pred = LLVMRealOEQ; break; + case Token_NotEq: pred = LLVMRealONE; break; + } + mask = LLVMBuildFCmp(p->builder, pred, left.value, right.value, ""); + } else { + LLVMIntPredicate pred = {}; + switch (op_kind) { + case Token_CmpEq: pred = LLVMIntEQ; break; + case Token_NotEq: pred = LLVMIntNE; break; + } + mask = LLVMBuildICmp(p->builder, pred, left.value, right.value, ""); + } + GB_ASSERT_MSG(mask != nullptr, "Unhandled comparison kind %s (%s) %.*s %s (%s)", type_to_string(left.type), type_to_string(base_type(left.type)), LIT(token_strings[op_kind]), type_to_string(right.type), type_to_string(base_type(right.type))); + + // TODO(bill): is this a good approach to dealing with comparisons of vectors? + char const *name = "llvm.vector.reduce.umax"; + LLVMTypeRef types[1] = {LLVMTypeOf(mask)}; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + LLVMValueRef args[1] = {}; + args[0] = mask; + res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); + return res; + } else { GB_PANIC("Unhandled comparison kind %s (%s) %.*s %s (%s)", type_to_string(left.type), type_to_string(base_type(left.type)), LIT(token_strings[op_kind]), type_to_string(right.type), type_to_string(base_type(right.type))); } diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index a56aa862a..99c023311 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1097,11 +1097,15 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_and: case BuiltinProc_simd_or: case BuiltinProc_simd_xor: + case BuiltinProc_simd_and_not: arg1 = lb_build_expr(p, ce->args[1]); switch (builtin_id) { case BuiltinProc_simd_and: op_code = LLVMAnd; break; case BuiltinProc_simd_or: op_code = LLVMOr; break; case BuiltinProc_simd_xor: op_code = LLVMXor; break; + case BuiltinProc_simd_and_not: + res.value = LLVMBuildAnd(p->builder, arg0.value, LLVMBuildNot(p->builder, arg1.value, ""), ""); + return res; } if (op_code) { res.value = LLVMBuildBinOp(p->builder, op_code, arg0.value, arg1.value, ""); diff --git a/src/types.cpp b/src/types.cpp index 6f61015d3..ad83e0568 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -2306,6 +2306,9 @@ bool is_type_comparable(Type *t) { } } return true; + + case Type_SimdVector: + return true; } return false; } From 7092273a8f5a2ae8c60ece297e7114a29e0f3652 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 17:36:13 +0100 Subject: [PATCH 38/86] Rename `simd_eq` etc to `simd_lanes_eq` --- core/intrinsics/intrinsics.odin | 19 ++++++++--------- core/simd/simd.odin | 12 +++++------ src/check_builtin.cpp | 16 +++++++-------- src/checker_builtin_procs.hpp | 24 +++++++++++----------- src/llvm_backend_proc.cpp | 36 ++++++++++++++++----------------- 5 files changed, 54 insertions(+), 53 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index a30bb109f..7c211d8fc 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -205,9 +205,10 @@ simd_shr_masked :: proc(a: #simd[N]T, b: #simd[N]Unsigned_Integer) -> #simd[N]T simd_add_sat :: proc(a, b: #simd[N]T) -> #simd[N]T --- simd_sub_sat :: proc(a, b: #simd[N]T) -> #simd[N]T --- -simd_and :: proc(a, b: #simd[N]T) -> #simd[N]T --- -simd_or :: proc(a, b: #simd[N]T) -> #simd[N]T --- -simd_xor :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_and :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_or :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_xor :: proc(a, b: #simd[N]T) -> #simd[N]T --- +simd_and_not :: proc(a, b: #simd[N]T) -> #simd[N]T --- simd_neg :: proc(a: #simd[N]T) -> #simd[N]T --- @@ -222,12 +223,12 @@ simd_clamp :: proc(v, min, max: #simd[N]T) -> #simd[N]T --- // element-wise: // false => 0x00...00 // true => 0xff...ff -simd_eq :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- -simd_ne :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- -simd_lt :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- -simd_le :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- -simd_gt :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- -simd_ge :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_lanes_eq :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_lanes_ne :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_lanes_lt :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_lanes_le :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_lanes_gt :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- +simd_lanes_ge :: proc(a, b: #simd[N]T) -> #simd[N]Integer --- simd_extract :: proc(a: #simd[N]T, idx: uint) -> T --- simd_replace :: proc(a: #simd[N]T, idx: uint, elem: T) -> #simd[N]T --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 9673b5cc9..c54d2a480 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -61,12 +61,12 @@ clamp :: intrinsics.simd_clamp // element-wise: // false => 0x00...00 // true => 0xff...ff -eq :: intrinsics.simd_eq -ne :: intrinsics.simd_ne -lt :: intrinsics.simd_lt -le :: intrinsics.simd_le -gt :: intrinsics.simd_gt -ge :: intrinsics.simd_ge +lanes_eq :: intrinsics.simd_lanes_eq +lanes_ne :: intrinsics.simd_lanes_ne +lanes_lt :: intrinsics.simd_lanes_lt +lanes_le :: intrinsics.simd_lanes_le +lanes_gt :: intrinsics.simd_lanes_gt +lanes_ge :: intrinsics.simd_lanes_ge // extract :: proc(a: #simd[N]T, idx: uint) -> T extract :: intrinsics.simd_extract diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index bfaa91cf8..c63c67d90 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -589,12 +589,12 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call } // Return integer masks - case BuiltinProc_simd_eq: - case BuiltinProc_simd_ne: - case BuiltinProc_simd_lt: - case BuiltinProc_simd_le: - case BuiltinProc_simd_gt: - case BuiltinProc_simd_ge: + case BuiltinProc_simd_lanes_eq: + case BuiltinProc_simd_lanes_ne: + case BuiltinProc_simd_lanes_lt: + case BuiltinProc_simd_lanes_le: + case BuiltinProc_simd_lanes_gt: + case BuiltinProc_simd_lanes_ge: { // op(#simd[N]T, #simd[N]T) -> #simd[N]V // where `V` is an integer, `size_of(T) == size_of(V)` @@ -611,8 +611,8 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call } Type *elem = base_array_type(x.type); switch (id) { - case BuiltinProc_simd_eq: - case BuiltinProc_simd_ne: + case BuiltinProc_simd_lanes_eq: + case BuiltinProc_simd_lanes_ne: if (!is_type_integer(elem) && !is_type_float(elem) && !is_type_boolean(elem)) { gbString xs = type_to_string(x.type); error(x.expr, "'%.*s' expected a #simd type with an integer, floating point, or boolean element, got '%s'", LIT(builtin_name), xs); diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index eb4bc1498..1b2c105f1 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -144,12 +144,12 @@ BuiltinProc__simd_begin, BuiltinProc_simd_max, BuiltinProc_simd_clamp, - BuiltinProc_simd_eq, - BuiltinProc_simd_ne, - BuiltinProc_simd_lt, - BuiltinProc_simd_le, - BuiltinProc_simd_gt, - BuiltinProc_simd_ge, + BuiltinProc_simd_lanes_eq, + BuiltinProc_simd_lanes_ne, + BuiltinProc_simd_lanes_lt, + BuiltinProc_simd_lanes_le, + BuiltinProc_simd_lanes_gt, + BuiltinProc_simd_lanes_ge, BuiltinProc_simd_extract, BuiltinProc_simd_replace, @@ -428,12 +428,12 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_max"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_clamp"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_eq"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_ne"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_lt"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_le"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_gt"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_ge"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_eq"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_ne"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_lt"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_le"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_gt"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_ge"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_extract"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_replace"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 99c023311..4c4000fec 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1151,22 +1151,22 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const res.value = LLVMBuildSelect(p->builder, cond, arg0.value, arg1.value, ""); } return res; - case BuiltinProc_simd_eq: - case BuiltinProc_simd_ne: - case BuiltinProc_simd_lt: - case BuiltinProc_simd_le: - case BuiltinProc_simd_gt: - case BuiltinProc_simd_ge: + case BuiltinProc_simd_lanes_eq: + case BuiltinProc_simd_lanes_ne: + case BuiltinProc_simd_lanes_lt: + case BuiltinProc_simd_lanes_le: + case BuiltinProc_simd_lanes_gt: + case BuiltinProc_simd_lanes_ge: arg1 = lb_build_expr(p, ce->args[1]); if (is_float) { LLVMRealPredicate pred = cast(LLVMRealPredicate)0; switch (builtin_id) { - case BuiltinProc_simd_eq: pred = LLVMRealOEQ; break; - case BuiltinProc_simd_ne: pred = LLVMRealONE; break; - case BuiltinProc_simd_lt: pred = LLVMRealOLT; break; - case BuiltinProc_simd_le: pred = LLVMRealOLE; break; - case BuiltinProc_simd_gt: pred = LLVMRealOGT; break; - case BuiltinProc_simd_ge: pred = LLVMRealOGE; break; + case BuiltinProc_simd_lanes_eq: pred = LLVMRealOEQ; break; + case BuiltinProc_simd_lanes_ne: pred = LLVMRealONE; break; + case BuiltinProc_simd_lanes_lt: pred = LLVMRealOLT; break; + case BuiltinProc_simd_lanes_le: pred = LLVMRealOLE; break; + case BuiltinProc_simd_lanes_gt: pred = LLVMRealOGT; break; + case BuiltinProc_simd_lanes_ge: pred = LLVMRealOGE; break; } if (pred) { res.value = LLVMBuildFCmp(p->builder, pred, arg0.value, arg1.value, ""); @@ -1176,12 +1176,12 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const } else { LLVMIntPredicate pred = cast(LLVMIntPredicate)0; switch (builtin_id) { - case BuiltinProc_simd_eq: pred = LLVMIntEQ; break; - case BuiltinProc_simd_ne: pred = LLVMIntNE; break; - case BuiltinProc_simd_lt: pred = is_signed ? LLVMIntSLT :LLVMIntULT; break; - case BuiltinProc_simd_le: pred = is_signed ? LLVMIntSLE :LLVMIntULE; break; - case BuiltinProc_simd_gt: pred = is_signed ? LLVMIntSGT :LLVMIntUGT; break; - case BuiltinProc_simd_ge: pred = is_signed ? LLVMIntSGE :LLVMIntUGE; break; + case BuiltinProc_simd_lanes_eq: pred = LLVMIntEQ; break; + case BuiltinProc_simd_lanes_ne: pred = LLVMIntNE; break; + case BuiltinProc_simd_lanes_lt: pred = is_signed ? LLVMIntSLT :LLVMIntULT; break; + case BuiltinProc_simd_lanes_le: pred = is_signed ? LLVMIntSLE :LLVMIntULE; break; + case BuiltinProc_simd_lanes_gt: pred = is_signed ? LLVMIntSGT :LLVMIntUGT; break; + case BuiltinProc_simd_lanes_ge: pred = is_signed ? LLVMIntSGE :LLVMIntUGE; break; } if (pred) { res.value = LLVMBuildICmp(p->builder, pred, arg0.value, arg1.value, ""); From 20e7b5c88acb2e0cee64ccdec7247227306a345f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 17:48:51 +0100 Subject: [PATCH 39/86] Support `count_ones` etc with #simd --- core/intrinsics/intrinsics.odin | 8 ++++---- core/simd/simd.odin | 15 ++++++++++----- src/check_builtin.cpp | 9 ++++++++- src/llvm_backend_utility.cpp | 6 ++++-- 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 7c211d8fc..bf8f56e63 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -22,10 +22,10 @@ alloca :: proc(size, align: int) -> [^]u8 --- cpu_relax :: proc() --- read_cycle_counter :: proc() -> i64 --- -count_ones :: proc(x: $T) -> T where type_is_integer(T) --- -count_zeros :: proc(x: $T) -> T where type_is_integer(T) --- -count_trailing_zeros :: proc(x: $T) -> T where type_is_integer(T) --- -count_leading_zeros :: proc(x: $T) -> T where type_is_integer(T) --- +count_ones :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- +count_zeros :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- +count_trailing_zeros :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- +count_leading_zeros :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- reverse_bits :: proc(x: $T) -> T where type_is_integer(T) --- byte_swap :: proc(x: $T) -> T where type_is_integer(T) || type_is_float(T) --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index c54d2a480..17d97f918 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -104,6 +104,11 @@ reverse :: intrinsics.simd_reverse rotate_left :: intrinsics.simd_rotate_left rotate_right :: intrinsics.simd_rotate_right +count_ones :: intrinsics.count_ones +count_zeros :: intrinsics.count_zeros +count_trailing_zeros :: intrinsics.count_trailing_zeros +count_leading_zeros :: intrinsics.count_leading_zeros + to_array_ptr :: #force_inline proc "contextless" (v: ^#simd[$LANES]$E) -> ^[LANES]E { return (^[LANES]E)(v) } @@ -129,16 +134,16 @@ bit_not :: #force_inline proc "contextless" (v: $T/#simd[$LANES]$E) -> T where i copysign :: #force_inline proc "contextless" (v, sign: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_float(E) { neg_zero := to_bits(T(-0.0)) - sign_bit := and(to_bits(sign), neg_zero) - magnitude := and(to_bits(v), bit_not(neg_zero)) - return transmute(T)or(sign_bit, magnitude) + sign_bit := to_bits(sign) & neg_zero + magnitude := to_bits(v) &~ neg_zero + return transmute(T)(sign_bit|magnitude) } signum :: #force_inline proc "contextless" (v: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_float(E) { - is_nan := ne(v, v) + is_nan := lanes_ne(v, v) return select(is_nan, v, copysign(T(1), v)) } recip :: #force_inline proc "contextless" (v: $T/#simd[$LANES]$E) -> T where intrinsics.type_is_float(E) { - return div(T(1), v) + return T(1) / v } diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index c63c67d90..ee805702d 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -3559,7 +3559,14 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 return false; } - if (!is_type_integer_like(x.type)) { + if (is_type_simd_vector(x.type) && id != BuiltinProc_reverse_bits) { + Type *elem = base_array_type(x.type); + if (!is_type_integer_like(elem)) { + gbString xts = type_to_string(x.type); + error(x.expr, "#simd values passed to '%.*s' must have an element of an integer-like type (integer, boolean, enum, bit_set), got %s", LIT(builtin_name), xts); + gb_string_free(xts); + } + } else if (!is_type_integer_like(x.type)) { gbString xts = type_to_string(x.type); error(x.expr, "Values passed to '%.*s' must be an integer-like type (integer, boolean, enum, bit_set), got %s", LIT(builtin_name), xts); gb_string_free(xts); diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index bfd21bedb..52d3a17cf 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -485,8 +485,10 @@ lbValue lb_emit_count_ones(lbProcedure *p, lbValue x, Type *type) { } lbValue lb_emit_count_zeros(lbProcedure *p, lbValue x, Type *type) { - i64 sz = 8*type_size_of(type); - lbValue size = lb_const_int(p->module, type, cast(u64)sz); + Type *elem = base_array_type(type); + i64 sz = 8*type_size_of(elem); + lbValue size = lb_const_int(p->module, elem, cast(u64)sz); + size = lb_emit_conv(p, size, type); lbValue count = lb_emit_count_ones(p, x, type); return lb_emit_arith(p, Token_Sub, size, count, type); } From 421d45a7a76e53946e7441212af9d08a0d93ff68 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 18:06:26 +0100 Subject: [PATCH 40/86] Add `intrinsics.fused_mul_add` --- core/intrinsics/intrinsics.odin | 2 ++ core/simd/simd.odin | 3 ++ src/check_builtin.cpp | 53 +++++++++++++++++++++++++++++++++ src/checker_builtin_procs.hpp | 2 ++ src/llvm_backend_proc.cpp | 25 ++++++++++++++++ 5 files changed, 85 insertions(+) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index bf8f56e63..c13e099c5 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -35,6 +35,8 @@ overflow_mul :: proc(lhs, rhs: $T) -> (T, bool) #optional_ok --- sqrt :: proc(x: $T) -> T where type_is_float(T) --- +fused_mul_add :: proc(a, b, c: $T) -> T where type_is_float(T) || (type_is_simd_vector(T) && type_is_float(type_elem_type(T))) --- + mem_copy :: proc(dst, src: rawptr, len: int) --- mem_copy_non_overlapping :: proc(dst, src: rawptr, len: int) --- mem_zero :: proc(ptr: rawptr, len: int) --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 17d97f918..ce278bce7 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -109,6 +109,9 @@ count_zeros :: intrinsics.count_zeros count_trailing_zeros :: intrinsics.count_trailing_zeros count_leading_zeros :: intrinsics.count_leading_zeros +fused_mul_add :: intrinsics.fused_mul_add +fma :: intrinsics.fused_mul_add + to_array_ptr :: #force_inline proc "contextless" (v: ^#simd[$LANES]$E) -> ^[LANES]E { return (^[LANES]E)(v) } diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index ee805702d..19b78b46e 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -3681,6 +3681,59 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 } break; + case BuiltinProc_fused_mul_add: + { + Operand x = {}; + Operand y = {}; + Operand z = {}; + check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; + check_expr(c, &y, ce->args[1]); if (y.mode == Addressing_Invalid) return false; + check_expr(c, &z, ce->args[2]); if (z.mode == Addressing_Invalid) return false; + + convert_to_typed(c, &y, x.type); if (y.mode == Addressing_Invalid) return false; + convert_to_typed(c, &x, y.type); if (x.mode == Addressing_Invalid) return false; + convert_to_typed(c, &z, x.type); if (z.mode == Addressing_Invalid) return false; + convert_to_typed(c, &x, z.type); if (x.mode == Addressing_Invalid) return false; + if (is_type_untyped(x.type)) { + gbString xts = type_to_string(x.type); + error(x.expr, "Expected a typed floating point value or #simd vector for '%.*s', got %s", LIT(builtin_name), xts); + gb_string_free(xts); + return false; + } + + Type *elem = core_array_type(x.type); + if (!is_type_float(x.type) && !(is_type_simd_vector(x.type) && is_type_float(elem))) { + gbString xts = type_to_string(x.type); + error(x.expr, "Expected a floating point or #simd vector value for '%.*s', got %s", LIT(builtin_name), xts); + gb_string_free(xts); + return false; + } + if (is_type_different_to_arch_endianness(elem)) { + GB_ASSERT(elem->kind == Type_Basic); + if (elem->Basic.flags & (BasicFlag_EndianLittle|BasicFlag_EndianBig)) { + gbString xts = type_to_string(x.type); + error(x.expr, "Expected a float which does not specify the explicit endianness for '%.*s', got %s", LIT(builtin_name), xts); + gb_string_free(xts); + return false; + } + } + + if (!are_types_identical(x.type, y.type) || !are_types_identical(y.type, z.type)) { + gbString xts = type_to_string(x.type); + gbString yts = type_to_string(y.type); + gbString zts = type_to_string(z.type); + error(x.expr, "Mismatched types for '%.*s', got %s vs %s vs %s", LIT(builtin_name), xts, yts, zts); + gb_string_free(zts); + gb_string_free(yts); + gb_string_free(xts); + return false; + } + + operand->mode = Addressing_Value; + operand->type = default_type(x.type); + } + break; + case BuiltinProc_mem_copy: case BuiltinProc_mem_copy_non_overlapping: { diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 1b2c105f1..5859ce3ab 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -65,6 +65,7 @@ enum BuiltinProcId { BuiltinProc_overflow_mul, BuiltinProc_sqrt, + BuiltinProc_fused_mul_add, BuiltinProc_mem_copy, BuiltinProc_mem_copy_non_overlapping, @@ -348,6 +349,7 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("overflow_mul"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("sqrt"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("fused_mul_add"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("mem_copy"), 3, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("mem_copy_non_overlapping"), 3, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 4c4000fec..a5dda7815 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -2005,6 +2005,31 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, return res; } + case BuiltinProc_fused_mul_add: + { + Type *type = tv.type; + lbValue x = lb_emit_conv(p, lb_build_expr(p, ce->args[0]), type); + lbValue y = lb_emit_conv(p, lb_build_expr(p, ce->args[1]), type); + lbValue z = lb_emit_conv(p, lb_build_expr(p, ce->args[2]), type); + + + char const *name = "llvm.fma"; + LLVMTypeRef types[1] = {lb_type(p->module, type)}; + unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); + GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); + LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + + LLVMValueRef args[3] = {}; + args[0] = x.value; + args[1] = y.value; + args[2] = z.value; + + lbValue res = {}; + res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); + res.type = type; + return res; + } + case BuiltinProc_mem_copy: { lbValue dst = lb_build_expr(p, ce->args[0]); From 1f438d4e6c0a979b249683cb2da048a0ff36dcce Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 18:09:59 +0100 Subject: [PATCH 41/86] Merge `intrinsics.simd_sqrt` with `intrinsics.sqrt` --- core/intrinsics/intrinsics.odin | 3 +-- core/simd/simd.odin | 2 +- src/check_builtin.cpp | 21 +++++++++++++++++---- src/checker_builtin_procs.hpp | 2 -- src/llvm_backend_proc.cpp | 2 -- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index c13e099c5..4f10c5a32 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -33,7 +33,7 @@ overflow_add :: proc(lhs, rhs: $T) -> (T, bool) #optional_ok --- overflow_sub :: proc(lhs, rhs: $T) -> (T, bool) #optional_ok --- overflow_mul :: proc(lhs, rhs: $T) -> (T, bool) #optional_ok --- -sqrt :: proc(x: $T) -> T where type_is_float(T) --- +sqrt :: proc(x: $T) -> T where type_is_float(T) || (type_is_simd_vector(T) && type_is_float(type_elem_type(T))) --- fused_mul_add :: proc(a, b, c: $T) -> T where type_is_float(T) || (type_is_simd_vector(T) && type_is_float(type_elem_type(T))) --- @@ -247,7 +247,6 @@ simd_shuffle :: proc(a, b: #simd[N]T, indices: ..int) -> #simd[len(indices)]T -- simd_select :: proc(cond: #simd[N]boolean_or_integer, true, false: #simd[N]T) -> #simd[N]T --- // Lane-wise operations -simd_sqrt :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- // IEEE sqrt simd_ceil :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- simd_floor :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- simd_trunc :: proc(a: #simd[N]any_float) -> #simd[N]any_float --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index ce278bce7..263402c43 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -91,7 +91,7 @@ shuffle :: intrinsics.simd_shuffle select :: intrinsics.simd_select -sqrt :: intrinsics.simd_sqrt +sqrt :: intrinsics.sqrt ceil :: intrinsics.simd_ceil floor :: intrinsics.simd_floor trunc :: intrinsics.simd_trunc diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 19b78b46e..87f7358f0 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -894,7 +894,6 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } - case BuiltinProc_simd_sqrt: case BuiltinProc_simd_ceil: case BuiltinProc_simd_floor: case BuiltinProc_simd_trunc: @@ -3661,14 +3660,28 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 if (x.mode == Addressing_Invalid) { return false; } + + Type *elem = core_array_type(x.type); + if (!is_type_float(x.type) && !(is_type_simd_vector(x.type) && is_type_float(elem))) { + gbString xts = type_to_string(x.type); + error(x.expr, "Expected a floating point or #simd vector value for '%.*s', got %s", LIT(builtin_name), xts); + gb_string_free(xts); + return false; + } else if (is_type_different_to_arch_endianness(elem)) { + GB_ASSERT(elem->kind == Type_Basic); + if (elem->Basic.flags & (BasicFlag_EndianLittle|BasicFlag_EndianBig)) { + gbString xts = type_to_string(x.type); + error(x.expr, "Expected a float which does not specify the explicit endianness for '%.*s', got %s", LIT(builtin_name), xts); + gb_string_free(xts); + return false; + } + } if (!is_type_float(x.type)) { gbString xts = type_to_string(x.type); error(x.expr, "Expected a floating point value for '%.*s', got %s", LIT(builtin_name), xts); gb_string_free(xts); return false; - } - - if (x.mode == Addressing_Constant) { + } else if (x.mode == Addressing_Constant) { f64 v = exact_value_to_f64(x.value); operand->mode = Addressing_Constant; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 5859ce3ab..28e62e5d6 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -166,7 +166,6 @@ BuiltinProc__simd_begin, BuiltinProc_simd_shuffle, BuiltinProc_simd_select, - BuiltinProc_simd_sqrt, BuiltinProc_simd_ceil, BuiltinProc_simd_floor, BuiltinProc_simd_trunc, @@ -451,7 +450,6 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_shuffle"), 2, true, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_select"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_sqrt") , 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_ceil") , 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_floor"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_trunc"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index a5dda7815..13643ccfe 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1315,7 +1315,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } - case BuiltinProc_simd_sqrt: case BuiltinProc_simd_ceil: case BuiltinProc_simd_floor: case BuiltinProc_simd_trunc: @@ -1323,7 +1322,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const { char const *name = nullptr; switch (builtin_id) { - case BuiltinProc_simd_sqrt: name = "llvm.sqrt"; break; case BuiltinProc_simd_ceil: name = "llvm.ceil"; break; case BuiltinProc_simd_floor: name = "llvm.floor"; break; case BuiltinProc_simd_trunc: name = "llvm.trunc"; break; From 70451f9335b819a056a06536016cdad7784cb8ea Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 20:40:48 +0100 Subject: [PATCH 42/86] Support reverse_bits for #simd --- core/intrinsics/intrinsics.odin | 2 +- core/simd/simd.odin | 3 ++- src/check_builtin.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 4f10c5a32..fa9b0ecec 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -26,7 +26,7 @@ count_ones :: proc(x: $T) -> T where type_is_integer(T) || type_is_sim count_zeros :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- count_trailing_zeros :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- count_leading_zeros :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- -reverse_bits :: proc(x: $T) -> T where type_is_integer(T) --- +reverse_bits :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- byte_swap :: proc(x: $T) -> T where type_is_integer(T) || type_is_float(T) --- overflow_add :: proc(lhs, rhs: $T) -> (T, bool) #optional_ok --- diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 263402c43..fc3c95e82 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -99,7 +99,7 @@ nearest :: intrinsics.simd_nearest to_bits :: intrinsics.simd_to_bits -reverse :: intrinsics.simd_reverse +lanes_reverse :: intrinsics.simd_reverse rotate_left :: intrinsics.simd_rotate_left rotate_right :: intrinsics.simd_rotate_right @@ -108,6 +108,7 @@ count_ones :: intrinsics.count_ones count_zeros :: intrinsics.count_zeros count_trailing_zeros :: intrinsics.count_trailing_zeros count_leading_zeros :: intrinsics.count_leading_zeros +reverse_bits :: intrinsics.reverse_bits fused_mul_add :: intrinsics.fused_mul_add fma :: intrinsics.fused_mul_add diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 87f7358f0..ecaba8b49 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -3558,7 +3558,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 return false; } - if (is_type_simd_vector(x.type) && id != BuiltinProc_reverse_bits) { + if (is_type_simd_vector(x.type)) { Type *elem = base_array_type(x.type); if (!is_type_integer_like(elem)) { gbString xts = type_to_string(x.type); From 1ff8b97dae79be3f5985de66dc4665640dd12c81 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Thu, 26 May 2022 20:44:37 +0100 Subject: [PATCH 43/86] Add prefix of `lanes_` --- core/simd/simd.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index fc3c95e82..43b3b3ee2 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -101,8 +101,8 @@ to_bits :: intrinsics.simd_to_bits lanes_reverse :: intrinsics.simd_reverse -rotate_left :: intrinsics.simd_rotate_left -rotate_right :: intrinsics.simd_rotate_right +lanes_rotate_left :: intrinsics.simd_rotate_left +lanes_rotate_right :: intrinsics.simd_rotate_right count_ones :: intrinsics.count_ones count_zeros :: intrinsics.count_zeros From 833f9dd037d2638a9d2a5bd2a1346057b097aed5 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 11:55:03 +0100 Subject: [PATCH 44/86] Minor change --- src/llvm_backend_proc.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 13643ccfe..93481352b 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1104,8 +1104,9 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_or: op_code = LLVMOr; break; case BuiltinProc_simd_xor: op_code = LLVMXor; break; case BuiltinProc_simd_and_not: - res.value = LLVMBuildAnd(p->builder, arg0.value, LLVMBuildNot(p->builder, arg1.value, ""), ""); - return res; + op_code = LLVMAnd; + arg1.value = LLVMBuildNot(p->builder, arg1.value, ""); + break; } if (op_code) { res.value = LLVMBuildBinOp(p->builder, op_code, arg0.value, arg1.value, ""); From c23274adb040ef1ffbdfd8dc1689bb320f274005 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 12:11:58 +0100 Subject: [PATCH 45/86] Remove useless check --- src/check_type.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/check_type.cpp b/src/check_type.cpp index f84c15a19..fc5b7aed7 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -2813,11 +2813,6 @@ bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, Type *named_t *type = alloc_type_simd_vector(count, elem, generic_type); - if (is_arch_wasm()) { - if (type_size_of(*type) != 16) { - error(at->count, "wasm based targets are limited to 128-bit types"); - } - } if (count > SIMD_ELEMENT_COUNT_MAX) { error(at->count, "#simd support a maximum element count of %d, got %lld", SIMD_ELEMENT_COUNT_MAX, cast(long long)count); } From 952f294bcef26e47d4a9e0f3ab2a1bc6f159c8b4 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 12:20:48 +0100 Subject: [PATCH 46/86] Add loads of aliases of vector types --- core/simd/simd.odin | 67 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index 43b3b3ee2..ed7e418f3 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -3,24 +3,59 @@ package simd import "core:builtin" import "core:intrinsics" -// boolx16 :: #simd[16]bool -// b8x16 :: #simd[16]b8 -// b16x8 :: #simd[8]b16 -// b32x4 :: #simd[4]b32 -// b64x2 :: #simd[2]b64 +// 128-bit vector aliases +u8x16 :: #simd[16]u8 +i8x16 :: #simd[16]i8 +u16x8 :: #simd[8]u16 +i16x8 :: #simd[8]i16 +u32x4 :: #simd[4]u32 +i32x4 :: #simd[4]i32 +u64x2 :: #simd[2]u64 +i64x2 :: #simd[2]i64 +f32x4 :: #simd[4]f32 +f64x2 :: #simd[2]f64 -// u8x16 :: #simd[16]u8 -// i8x16 :: #simd[16]i8 -// u16x8 :: #simd[8]u16 -// i16x8 :: #simd[8]i16 -// u32x4 :: #simd[4]u32 -// i32x4 :: #simd[4]i32 -// u64x2 :: #simd[2]u64 -// i64x2 :: #simd[2]i64 +boolx16 :: #simd[16]bool +b8x16 :: #simd[16]b8 +b16x8 :: #simd[8]b16 +b32x4 :: #simd[4]b32 +b64x2 :: #simd[2]b64 -// f16x8 :: #simd[8]f16 -// f32x4 :: #simd[4]f32 -// f64x2 :: #simd[2]f64 +// 256-bit vector aliases +u8x32 :: #simd[32]u8 +i8x32 :: #simd[32]i8 +u16x16 :: #simd[16]u16 +i16x16 :: #simd[16]i16 +u32x8 :: #simd[8]u32 +i32x8 :: #simd[8]i32 +u64x4 :: #simd[4]u64 +i64x4 :: #simd[4]i64 +f32x8 :: #simd[8]f32 +f64x4 :: #simd[4]f64 + +boolx32 :: #simd[32]bool +b8x32 :: #simd[32]b8 +b16x16 :: #simd[16]b16 +b32x8 :: #simd[8]b32 +b64x4 :: #simd[4]b64 + +// 512-bit vector aliases +u8x64 :: #simd[64]u8 +i8x64 :: #simd[64]i8 +u16x32 :: #simd[32]u16 +i16x32 :: #simd[32]i16 +u32x16 :: #simd[16]u32 +i32x16 :: #simd[16]i32 +u64x8 :: #simd[8]u64 +i64x8 :: #simd[8]i64 +f32x16 :: #simd[16]f32 +f64x8 :: #simd[8]f64 + +boolx64 :: #simd[64]bool +b8x64 :: #simd[64]b8 +b16x32 :: #simd[32]b16 +b32x16 :: #simd[16]b32 +b64x8 :: #simd[8]b64 add :: intrinsics.simd_add From 432b2b19e9b3a79e7ee5fecbf981d794e9d022cc Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 12:54:28 +0100 Subject: [PATCH 47/86] Add `intrinsics.simd_x86__MM_SHUFFLE` --- src/check_builtin.cpp | 28 ++++++++++++++++++++++++++++ src/checker_builtin_procs.hpp | 6 ++++++ 2 files changed, 34 insertions(+) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index ecaba8b49..e93e63d4d 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -1033,6 +1033,34 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } + case BuiltinProc_simd_x86__MM_SHUFFLE: + { + Operand x[4] = {}; + for (unsigned i = 0; i < 4; i++) { + check_expr(c, x+i, ce->args[i]); if (x[i].mode == Addressing_Invalid) return false; + } + + u32 offsets[4] = {6, 4, 2, 0}; + u32 result = 0; + for (unsigned i = 0; i < 4; i++) { + if (!is_type_integer(x[i].type) || x[i].mode != Addressing_Constant) { + gbString xs = type_to_string(x[i].type); + error(x[i].expr, "'%.*s' expected a constant integer", LIT(builtin_name), xs); + gb_string_free(xs); + return false; + } + i64 val = exact_value_to_i64(x[i].value); + if (val < 0 || val > 3) { + error(x[i].expr, "'%.*s' expected a constant integer in the range 0..<4, got %lld", LIT(builtin_name), cast(long long)val); + return false; + } + result |= cast(u32)(val) << offsets[i]; + } + + operand->type = t_untyped_integer; + operand->mode = Addressing_Constant; + operand->value = exact_value_i64(result); + } default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 28e62e5d6..3ef97b361 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -176,6 +176,10 @@ BuiltinProc__simd_begin, BuiltinProc_simd_reverse, BuiltinProc_simd_rotate_left, BuiltinProc_simd_rotate_right, + + + // Platform specific SIMD intrinsics + BuiltinProc_simd_x86__MM_SHUFFLE, BuiltinProc__simd_end, // Platform specific intrinsics @@ -460,6 +464,8 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_reverse"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_rotate_left"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_rotate_right"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + + {STR_LIT("simd_x86__MM_SHUFFLE"), 4, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, From 0b08080119f500e65a381d7e76a010789b4dcef0 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 14:23:31 +0100 Subject: [PATCH 48/86] Add `core:simd/x86` SSE Proof of Concept to show intrinsics specific to a certain target platform --- core/simd/x86/sse.odin | 500 +++++++++++++++++++++++++++++++++++++++ core/simd/x86/types.odin | 44 ++++ 2 files changed, 544 insertions(+) create mode 100644 core/simd/x86/sse.odin create mode 100644 core/simd/x86/types.odin diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin new file mode 100644 index 000000000..79b13f667 --- /dev/null +++ b/core/simd/x86/sse.odin @@ -0,0 +1,500 @@ +//+build amd64 +package simd_amd64 + +import "core:intrinsics" +import "core:simd" + +// _MM_SHUFFLE(z, y, x, w) -> (z<<6 | y<<4 | x<<2 | w) +_MM_SHUFFLE :: intrinsics.simd_x86__MM_SHUFFLE + +_MM_HINT_T0 :: 3 +_MM_HINT_T1 :: 2 +_MM_HINT_T2 :: 1 +_MM_HINT_NTA :: 0 +_MM_HINT_ET0 :: 7 +_MM_HINT_ET1 :: 6 + + +_MM_EXCEPT_INVALID :: 0x0001 +_MM_EXCEPT_DENORM :: 0x0002 +_MM_EXCEPT_DIV_ZERO :: 0x0004 +_MM_EXCEPT_OVERFLOW :: 0x0008 +_MM_EXCEPT_UNDERFLOW :: 0x0010 +_MM_EXCEPT_INEXACT :: 0x0020 +_MM_EXCEPT_MASK :: 0x003f + +_MM_MASK_INVALID :: 0x0080 +_MM_MASK_DENORM :: 0x0100 +_MM_MASK_DIV_ZERO :: 0x0200 +_MM_MASK_OVERFLOW :: 0x0400 +_MM_MASK_UNDERFLOW :: 0x0800 +_MM_MASK_INEXACT :: 0x1000 +_MM_MASK_MASK :: 0x1f80 + +_MM_ROUND_NEAREST :: 0x0000 +_MM_ROUND_DOWN :: 0x2000 +_MM_ROUND_UP :: 0x4000 +_MM_ROUND_TOWARD_ZERO :: 0x6000 + +_MM_ROUND_MASK :: 0x6000 + +_MM_FLUSH_ZERO_MASK :: 0x8000 +_MM_FLUSH_ZERO_ON :: 0x8000 +_MM_FLUSH_ZERO_OFF :: 0x0000 + + +_mm_add_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return addss(a, b) +} +_mm_add_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.add(a, b) +} + +_mm_sub_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return subss(a, b) +} +_mm_sub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.sub(a, b) +} + +_mm_mul_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return mulss(a, b) +} +_mm_mul_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.mul(a, b) +} + +_mm_div_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return divss(a, b) +} +_mm_div_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.div(a, b) +} + +_mm_sqrt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return sqrtss(a) +} +_mm_sqrt_ps :: #force_inline proc "c" (a: __m128) -> __m128 { + return sqrtps(a) +} + +_mm_rcp_ss :: #force_inline proc "c" (a: __m128) -> __m128 { + return rcpss(a) +} +_mm_rcp_ps :: #force_inline proc "c" (a: __m128) -> __m128 { + return rcpps(a) +} + +_mm_rsqrt_ss :: #force_inline proc "c" (a: __m128) -> __m128 { + return rsqrtss(a) +} +_mm_rsqrt_ps :: #force_inline proc "c" (a: __m128) -> __m128 { + return rsqrtps(a) +} + +_mm_min_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return minss(a, b) +} +_mm_min_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return minps(a, b) +} + +_mm_max_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return maxss(a, b) +} +_mm_max_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return maxps(a, b) +} + +_mm_and_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + x := transmute(__m128i)a + y := transmute(__m128i)b + return transmute(__m128)simd.and(x, y) +} +_mm_andnot_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + x := transmute(__m128i)a + y := transmute(__m128i)b + return transmute(__m128)simd.and_not(x, y) +} +_mm_or_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + x := transmute(__m128i)a + y := transmute(__m128i)b + return transmute(__m128)simd.or(x, y) +} +_mm_xor_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + x := transmute(__m128i)a + y := transmute(__m128i)b + return transmute(__m128)simd.xor(x, y) +} + + +_mm_cmpeq_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpss(a, b, 0) +} +_mm_cmplt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpss(a, b, 1) +} +_mm_cmple_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpss(a, b, 2) +} +_mm_cmpgt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, cmpss(b, a, 1), 4, 1, 2, 3) +} +_mm_cmpge_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, cmpss(b, a, 2), 4, 1, 2, 3) +} +_mm_cmpneq_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpss(a, b, 4) +} +_mm_cmpnlt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpss(a, b, 5) +} +_mm_cmpnle_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpss(a, b, 6) +} +_mm_cmpngt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, cmpss(b, a, 5), 4, 1, 2, 3) +} +_mm_cmpnge_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, cmpss(b, a, 6), 4, 1, 2, 3) +} +_mm_cmpord_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpss(a, b, 7) +} +_mm_cmpunord_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpss(a, b, 3) +} + + +_mm_cmpeq_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(a, b, 0) +} +_mm_cmplt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(a, b, 1) +} +_mm_cmple_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(a, b, 2) +} +_mm_cmpgt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(b, a, 1) +} +_mm_cmpge_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(b, a, 2) +} +_mm_cmpneq_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(a, b, 4) +} +_mm_cmpnlt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(a, b, 5) +} +_mm_cmpnle_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(a, b, 6) +} +_mm_cmpngt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(b, a, 5) +} +_mm_cmpnge_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(b, a, 6) +} +_mm_cmpord_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(b, a, 7) +} +_mm_cmpunord_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return cmpps(b, a, 3) +} + + +_mm_comieq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return comieq_ss(a, b) +} +_mm_comilt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return comilt_ss(a, b) +} +_mm_comile_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return comile_ss(a, b) +} +_mm_comigt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return comigt_ss(a, b) +} +_mm_comige_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return comige_ss(a, b) +} +_mm_comineq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return comineq_ss(a, b) +} + +_mm_ucomieq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return ucomieq_ss(a, b) +} +_mm_ucomilt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return ucomilt_ss(a, b) +} +_mm_ucomile_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return ucomile_ss(a, b) +} +_mm_ucomigt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return ucomigt_ss(a, b) +} +_mm_ucomige_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return ucomige_ss(a, b) +} +_mm_ucomineq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { + return ucomineq_ss(a, b) +} + +_mm_cvtss_si32 :: #force_inline proc "c" (a: __m128) -> i32 { + return cvtss2si(a) +} +_mm_cvt_ss2si :: _mm_cvtss_si32 +_mm_cvttss_si32 :: _mm_cvtss_si32 + +_mm_cvtss_f32 :: #force_inline proc "c" (a: __m128) -> f32 { + return simd.extract(a, 0) +} + +_mm_cvtsi32_ss :: #force_inline proc "c" (a: __m128, b: i32) -> __m128 { + return cvtsi2ss(a, b) +} +_mm_cvt_si2ss :: _mm_cvtsi32_ss + + +_mm_set_ss :: #force_inline proc "c" (a: f32) -> __m128 { + return __m128{a, 0, 0, 0} +} +_mm_set1_ps :: #force_inline proc "c" (a: f32) -> __m128 { + return __m128(a) +} +_mm_set_ps1 :: _mm_set1_ps + +_mm_set_ps :: #force_inline proc "c" (a, b, c, d: f32) -> __m128 { + return __m128{d, c, b, a} +} +_mm_setr_ps :: #force_inline proc "c" (a, b, c, d: f32) -> __m128 { + return __m128{a, b, c, d} +} + +_mm_setzero_ps :: #force_inline proc "c" () -> __m128 { + return __m128{0, 0, 0, 0} +} + +_mm_shuffle_ps :: #force_inline proc "c" (a, b: __m128, $MASK: u32) -> __m128 { + return simd.shuffle( + a, b, + u32(MASK) & 0b11, + (u32(MASK)>>2) & 0b11, + ((u32(MASK)>>4) & 0b11)+4, + ((u32(MASK)>>6) & 0b11)+4) +} + + +_mm_unpackhi_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, b, 2, 6, 3, 7) +} +_mm_unpacklo_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, b, 0, 4, 1, 5) +} + +_mm_movehl_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, b, 6, 7, 2, 3) +} +_mm_movelh_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, b, 0, 1, 4, 5) +} + +_mm_movemask_ps :: proc(a: __m128) -> u32 { + return movmskps(a) +} + +_mm_load_ss :: #force_inline proc "c" (p: ^f32) -> __m128 { + return __m128{p^, 0, 0, 0} +} +_mm_load1_ps :: #force_inline proc "c" (p: ^f32) -> __m128 { + a := p^ + return __m128(a) +} +_mm_load_ps1 :: _mm_load1_ps + +_mm_load_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { + return (^__m128)(p)^ +} + +_mm_loadu_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { + dst := _mm_undefined_ps() + intrinsics.mem_copy_non_overlapping(&dst, p, size_of(__m128)) + return dst +} + +_mm_loadr_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { + return simd.lanes_reverse(_mm_load_ps(p)) +} + +_mm_loadu_si64 :: #force_inline proc "c" (mem_addr: rawptr) -> __m128i { + a := intrinsics.unaligned_load((^i64)(mem_addr)) + return __m128i{a, 0} +} + +_mm_store_ss :: #force_inline proc "c" (p: ^f32, a: __m128) { + p^ = simd.extract(a, 0) +} + +_mm_store1_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { + b := simd.swizzle(a, 0, 0, 0, 0) + (^__m128)(p)^ = b +} +_mm_store_ps1 :: _mm_store1_ps + + +_mm_store_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { + (^__m128)(p)^ = a +} +_mm_storeu_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { + b := a + intrinsics.mem_copy_non_overlapping(p, &b, size_of(__m128)) +} +_mm_storer_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { + (^__m128)(p)^ = simd.lanes_reverse(a) +} + + +_mm_move_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return simd.shuffle(a, b, 4, 1, 2, 3) +} + +_mm_sfence :: #force_inline proc "c" () { + sfence() +} + +_mm_getcsr :: #force_inline proc "c" () -> (result: u32) { + stmxcsr(&result) + return result +} + +_mm_setcsr :: #force_inline proc "c" (val: u32) { + val := val + ldmxcsr(&val) +} + +_MM_GET_EXCEPTION_MASK :: #force_inline proc "c" () -> u32 { + return _mm_getcsr() & _MM_MASK_MASK +} +_MM_GET_EXCEPTION_STATE :: #force_inline proc "c" () -> u32 { + return _mm_getcsr() & _MM_EXCEPT_MASK +} +_MM_GET_FLUSH_ZERO_MODE :: #force_inline proc "c" () -> u32 { + return _mm_getcsr() & _MM_FLUSH_ZERO_MASK +} +_MM_GET_ROUNDING_MODE :: #force_inline proc "c" () -> u32 { + return _mm_getcsr() & _MM_ROUND_MASK +} + +_MM_SET_EXCEPTION_MASK :: #force_inline proc "c" (x: u32) { + _mm_setcsr((_mm_getcsr() &~ _MM_MASK_MASK) | x) +} +_MM_SET_EXCEPTION_STATE :: #force_inline proc "c" (x: u32) { + _mm_setcsr((_mm_getcsr() &~ _MM_EXCEPT_MASK) | x) +} +_MM_SET_FLUSH_ZERO_MODE :: #force_inline proc "c" (x: u32) { + _mm_setcsr((_mm_getcsr() &~ _MM_FLUSH_ZERO_MASK) | x) +} +_MM_SET_ROUNDING_MODE :: #force_inline proc "c" (x: u32) { + _mm_setcsr((_mm_getcsr() &~ _MM_ROUND_MASK) | x) +} + +_mm_prefetch :: #force_inline proc "c" (p: rawptr, $STRATEGY: u32) { + prefetch(p, (STRATEGY>>2)&1, STRATEGY&3, 1) +} + + +_mm_undefined_ps :: #force_inline proc "c" () -> __m128 { + return _mm_set1_ps(0) +} + +_MM_TRANSPOSE4_PS :: #force_inline proc "c" (row0, row1, row2, row3: ^__m128) { + tmp0 := _mm_unpacklo_ps(row0^, row1^) + tmp1 := _mm_unpacklo_ps(row2^, row3^) + tmp2 := _mm_unpackhi_ps(row0^, row1^) + tmp3 := _mm_unpackhi_ps(row2^, row3^) + + row0^ = _mm_movelh_ps(tmp0, tmp2) + row1^ = _mm_movelh_ps(tmp2, tmp0) + row2^ = _mm_movelh_ps(tmp1, tmp3) + row3^ = _mm_movelh_ps(tmp3, tmp1) +} + + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name="llvm.x86.sse.add.ss") + addss :: proc(a, b: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.sub.ss") + subss :: proc(a, b: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.mul.ss") + mulss :: proc(a, b: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.div.ss") + divss :: proc(a, b: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.sqrt.ss") + sqrtss :: proc(a: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.sqrt.ps") + sqrtps :: proc(a: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.rcp.ss") + rcpss :: proc(a: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.rcp.ps") + rcpps :: proc(a: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.rsqrt.ss") + rsqrtss :: proc(a: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.rsqrt.ps") + rsqrtps :: proc(a: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.min.ss") + minss :: proc(a, b: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.min.ps") + minps :: proc(a, b: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.max.ss") + maxss :: proc(a, b: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.max.ps") + maxps :: proc(a, b: __m128) -> __m128 --- + @(link_name="llvm.x86.sse.movmsk.ps") + movmskps :: proc(a: __m128) -> u32 --- + @(link_name="llvm.x86.sse.cmp.ps") + cmpps :: proc(a, b: __m128, #const imm8: u8) -> __m128 --- + @(link_name="llvm.x86.sse.comieq.ss") + comieq_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.comilt.ss") + comilt_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.comile.ss") + comile_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.comigt.ss") + comigt_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.comige.ss") + comige_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.comineq.ss") + comineq_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.ucomieq.ss") + ucomieq_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.ucomilt.ss") + ucomilt_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.ucomile.ss") + ucomile_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.ucomigt.ss") + ucomigt_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.ucomige.ss") + ucomige_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.ucomineq.ss") + ucomineq_ss :: proc(a, b: __m128) -> b32 --- + @(link_name="llvm.x86.sse.cvtss2si") + cvtss2si :: proc(a: __m128) -> i32 --- + @(link_name="llvm.x86.sse.cvttss2si") + cvttss2si :: proc(a: __m128) -> i32 --- + @(link_name="llvm.x86.sse.cvtsi2ss") + cvtsi2ss :: proc(a: __m128, b: i32) -> __m128 --- + @(link_name="llvm.x86.sse.sfence") + sfence :: proc() --- + @(link_name="llvm.x86.sse.stmxcsr") + stmxcsr :: proc(p: rawptr) --- + @(link_name="llvm.x86.sse.ldmxcsr") + ldmxcsr :: proc(p: rawptr) --- + @(link_name="llvm.prefetch") + prefetch :: proc(p: rawptr, #const rw, loc, ty: u32) --- + @(link_name="llvm.x86.sse.cmp.ss") + cmpss :: proc(a, b: __m128, #const imm8: u8) -> __m128 --- +} diff --git a/core/simd/x86/types.odin b/core/simd/x86/types.odin new file mode 100644 index 000000000..ef6342542 --- /dev/null +++ b/core/simd/x86/types.odin @@ -0,0 +1,44 @@ +//+build amd64 +package simd_amd64 + +bf16 :: u16 + +__m128i :: #simd[2]i64 +__m128 :: #simd[4]f32 +__m128d :: #simd[2]f64 + +__m256i :: #simd[4]i64 +__m256 :: #simd[8]f32 +__m256d :: #simd[4]f64 + +__m512i :: #simd[8]i64 +__m512 :: #simd[16]f32 +__m512d :: #simd[8]f64 + +__m128bh :: #simd[8]bf16 +__m256bh :: #simd[16]bf16 +__m512bh :: #simd[32]bf16 + + +/// The `__mmask64` type used in AVX-512 intrinsics, a 64-bit integer +__mmask64 :: u64 + +/// The `__mmask32` type used in AVX-512 intrinsics, a 32-bit integer +__mmask32 :: u32 + +/// The `__mmask16` type used in AVX-512 intrinsics, a 16-bit integer +__mmask16 :: u16 + +/// The `__mmask8` type used in AVX-512 intrinsics, a 8-bit integer +__mmask8 :: u8 + +/// The `_MM_CMPINT_ENUM` type used to specify comparison operations in AVX-512 intrinsics. +_MM_CMPINT_ENUM :: i32 + +/// The `MM_MANTISSA_NORM_ENUM` type used to specify mantissa normalized operations in AVX-512 intrinsics. +_MM_MANTISSA_NORM_ENUM :: i32 + +/// The `MM_MANTISSA_SIGN_ENUM` type used to specify mantissa signed operations in AVX-512 intrinsics. +_MM_MANTISSA_SIGN_ENUM :: i32 + +_MM_PERM_ENUM :: i32 \ No newline at end of file From 2185dada56a7c649d11c4e2edea63b0595f69061 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 14:26:02 +0100 Subject: [PATCH 49/86] Change package name --- core/simd/x86/sse.odin | 4 ++-- core/simd/x86/types.odin | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin index 79b13f667..1af5a65f1 100644 --- a/core/simd/x86/sse.odin +++ b/core/simd/x86/sse.odin @@ -1,5 +1,5 @@ -//+build amd64 -package simd_amd64 +//+build i386, amd64 +package simd_x86 import "core:intrinsics" import "core:simd" diff --git a/core/simd/x86/types.odin b/core/simd/x86/types.odin index ef6342542..3c94d74a0 100644 --- a/core/simd/x86/types.odin +++ b/core/simd/x86/types.odin @@ -1,5 +1,5 @@ -//+build amd64 -package simd_amd64 +//+build i386, amd64 +package simd_x86 bf16 :: u16 From 609ddf28b73817f4043aed27fb8056eb1eacffc0 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 14:56:36 +0100 Subject: [PATCH 50/86] Add intrinsics `nontemporal_store` and `nontemporal_load` --- core/intrinsics/intrinsics.odin | 3 +++ src/check_builtin.cpp | 6 ++---- src/checker_builtin_procs.hpp | 4 ++++ src/llvm_backend_proc.cpp | 8 ++++++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index fa9b0ecec..8becd998d 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -12,6 +12,9 @@ soa_struct :: proc($N: int, $T: typeid) -> type/#soa[N]T volatile_load :: proc(dst: ^$T) -> T --- volatile_store :: proc(dst: ^$T, val: T) -> T --- +nontemporal_load :: proc(dst: ^$T) -> T --- +nontemporal_store :: proc(dst: ^$T, val: T) -> T --- + // Trapping debug_trap :: proc() --- trap :: proc() -> ! --- diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index e93e63d4d..ba34a177b 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -4025,9 +4025,8 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 break; case BuiltinProc_volatile_store: - /*fallthrough*/ case BuiltinProc_unaligned_store: - /*fallthrough*/ + case BuiltinProc_nontemporal_store: case BuiltinProc_atomic_store: { Type *elem = nullptr; @@ -4074,9 +4073,8 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 case BuiltinProc_volatile_load: - /*fallthrough*/ case BuiltinProc_unaligned_load: - /*fallthrough*/ + case BuiltinProc_nontemporal_load: case BuiltinProc_atomic_load: { Type *elem = nullptr; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 3ef97b361..2dd775193 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -80,6 +80,8 @@ enum BuiltinProcId { BuiltinProc_unaligned_store, BuiltinProc_unaligned_load, + BuiltinProc_nontemporal_store, + BuiltinProc_nontemporal_load, BuiltinProc_prefetch_read_instruction, BuiltinProc_prefetch_read_data, @@ -367,6 +369,8 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("unaligned_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("unaligned_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("nontemporal_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + {STR_LIT("nontemporal_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("prefetch_read_instruction"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("prefetch_read_data"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 93481352b..2b7cad5cd 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -2111,6 +2111,7 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, return {}; case BuiltinProc_volatile_store: + case BuiltinProc_nontemporal_store: case BuiltinProc_atomic_store: case BuiltinProc_atomic_store_explicit: { lbValue dst = lb_build_expr(p, ce->args[0]); @@ -2120,6 +2121,9 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, LLVMValueRef instr = LLVMBuildStore(p->builder, val.value, dst.value); switch (id) { case BuiltinProc_volatile_store: LLVMSetVolatile(instr, true); break; + case BuiltinProc_nontemporal_store: + // TODO(bill): BuiltinProc_nontemporal_store + break; case BuiltinProc_atomic_store: LLVMSetOrdering(instr, LLVMAtomicOrderingSequentiallyConsistent); break; case BuiltinProc_atomic_store_explicit: LLVMSetOrdering(instr, llvm_atomic_ordering_from_odin(ce->args[2])); break; } @@ -2130,6 +2134,7 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, } case BuiltinProc_volatile_load: + case BuiltinProc_nontemporal_load: case BuiltinProc_atomic_load: case BuiltinProc_atomic_load_explicit: { lbValue dst = lb_build_expr(p, ce->args[0]); @@ -2137,6 +2142,9 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, LLVMValueRef instr = LLVMBuildLoad(p->builder, dst.value, ""); switch (id) { case BuiltinProc_volatile_load: LLVMSetVolatile(instr, true); break; + case BuiltinProc_nontemporal_load: + // TODO(bill): BuiltinProc_nontemporal_load + break; case BuiltinProc_atomic_load: LLVMSetOrdering(instr, LLVMAtomicOrderingSequentiallyConsistent); break; case BuiltinProc_atomic_load_explicit: LLVMSetOrdering(instr, llvm_atomic_ordering_from_odin(ce->args[1])); break; } From f383bf3136835d93af5e9110ab14c8529d9cfb82 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 14:59:09 +0100 Subject: [PATCH 51/86] Add `_mm_stream_ps` --- core/simd/x86/sse.odin | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin index 1af5a65f1..b43ccb953 100644 --- a/core/simd/x86/sse.odin +++ b/core/simd/x86/sse.odin @@ -421,6 +421,10 @@ _MM_TRANSPOSE4_PS :: #force_inline proc "c" (row0, row1, row2, row3: ^__m128) { row3^ = _mm_movelh_ps(tmp3, tmp1) } +_mm_stream_ps :: proc(addr: [^]f32, a: __m128) { + intrinsics.nontemporal_store((^__m128)(addr), a) +} + @(default_calling_convention="c") @(private) From e079a7009ddcf8c2d662ebf9a4a76f5365c30f4e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 16:09:31 +0100 Subject: [PATCH 52/86] Begin work on sse2.odin --- core/simd/x86/sse2.odin | 274 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 core/simd/x86/sse2.odin diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin new file mode 100644 index 000000000..b50e33d28 --- /dev/null +++ b/core/simd/x86/sse2.odin @@ -0,0 +1,274 @@ +//+build i386, amd64 +package simd_x86 + +import "core:simd" + +_mm_pause :: #force_inline proc "c" () { + pause() +} +_mm_clflush :: #force_inline proc "c" (p: rawptr) { + clflush(p) +} +_mm_lfence :: #force_inline proc "c" () { + lfence() +} +_mm_mfence :: #force_inline proc "c" () { + mfence() +} + +_mm_add_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i8x16)a + y := transmute(simd.i8x16)b + return transmute(__m128i)simd.add(x, y) +} +_mm_add_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i16x8)a + y := transmute(simd.i16x8)b + return transmute(__m128i)simd.add(x, y) +} +_mm_add_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i32x4)a + y := transmute(simd.i32x4)b + return transmute(__m128i)simd.add(x, y) +} +_mm_add_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i64x2)a + y := transmute(simd.i64x2)b + return transmute(__m128i)simd.add(x, y) +} +_mm_adds_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i8x16)a + y := transmute(simd.i8x16)b + return transmute(__m128i)simd.add_sat(x, y) +} +_mm_adds_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i16x8)a + y := transmute(simd.i16x8)b + return transmute(__m128i)simd.add_sat(x, y) +} +_mm_adds_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.u8x16)a + y := transmute(simd.u8x16)b + return transmute(__m128i)simd.add_sat(x, y) +} +_mm_adds_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.u16x8)a + y := transmute(simd.u16x8)b + return transmute(__m128i)simd.add_sat(x, y) +} +_mm_avg_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.u8x16)a + y := transmute(simd.u8x16)b + return transmute(__m128i)pavgb(x, y) +} +_mm_avg_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.u16x8)a + y := transmute(simd.u16x8)b + return transmute(__m128i)pavgw(x, y) +} + +_mm_madd_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i16x8)a + y := transmute(simd.i16x8)b + return transmute(__m128i)pmaddwd(x, y) +} +_mm_max_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i16x8)a + y := transmute(simd.i16x8)b + return transmute(__m128i)pmaxsw(x, y) +} +_mm_max_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.u8x16)a + y := transmute(simd.u8x16)b + return transmute(__m128i)pmaxub(x, y) +} +_mm_min_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.i16x8)a + y := transmute(simd.i16x8)b + return transmute(__m128i)pminsw(x, y) +} +_mm_min_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + x := transmute(simd.u8x16)a + y := transmute(simd.u8x16)b + return transmute(__m128i)pminub(x, y) +} + + +_mm_castpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { + return transmute(__m128)a +} +_mm_castpd_si128 :: #force_inline proc "c" (a: __m128d) -> __m128i { + return transmute(__m128i)a +} +_mm_castps_pd :: #force_inline proc "c" (a: __m128) -> __m128d { + return transmute(__m128d)a +} +_mm_castps_si128 :: #force_inline proc "c" (a: __m128) -> __m128i { + return transmute(__m128i)a +} +_mm_castsi128_pd :: #force_inline proc "c" (a: __m128i) -> __m128d { + return transmute(__m128d)a +} +_mm_castsi128_ps :: #force_inline proc "c" (a: __m128i) -> __m128 { + return transmute(__m128)a +} + + +_mm_undefined_pd :: #force_inline proc "c" () -> __m128d { + return __m128d{0, 0} +} +_mm_undefined_si128 :: #force_inline proc "c" () -> __m128i { + return __m128i{0, 0} +} +_mm_unpackhi_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.shuffle(a, b, 1, 3) +} +_mm_unpacklo_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.shuffle(a, b, 0, 2) +} + + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name="llvm.x86.sse2.pause") + pause :: proc() --- + @(link_name="llvm.x86.sse2.clflush") + clflush :: proc(p: rawptr) --- + @(link_name="llvm.x86.sse2.lfence") + lfence :: proc() --- + @(link_name="llvm.x86.sse2.mfence") + mfence :: proc() --- + @(link_name="llvm.x86.sse2.pavg.b") + pavgb :: proc(a, b: simd.u8x16) -> simd.u8x16 --- + @(link_name="llvm.x86.sse2.pavg.w") + pavgw :: proc(a, b: simd.u16x8) -> simd.u16x8 --- + @(link_name="llvm.x86.sse2.pmadd.wd") + pmaddwd :: proc(a, b: simd.i16x8) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.pmaxs.w") + pmaxsw :: proc(a, b: simd.i16x8) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.pmaxu.b") + pmaxub :: proc(a, b: simd.u8x16) -> simd.u8x16 --- + @(link_name="llvm.x86.sse2.pmins.w") + pminsw :: proc(a, b: simd.i16x8) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.pminu.b") + pminub :: proc(a, b: simd.u8x16) -> simd.u8x16 --- + @(link_name="llvm.x86.sse2.pmulh.w") + pmulhw :: proc(a, b: simd.i16x8) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.pmulhu.w") + pmulhuw :: proc(a, b: simd.u16x8) -> simd.u16x8 --- + @(link_name="llvm.x86.sse2.pmulu.dq") + pmuludq :: proc(a, b: simd.u32x4) -> simd.u64x2 --- + @(link_name="llvm.x86.sse2.psad.bw") + psadbw :: proc(a, b: simd.u8x16) -> simd.u64x2 --- + @(link_name="llvm.x86.sse2.pslli.w") + pslliw :: proc(a: simd.i16x8, #const imm8: u32) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.psll.w") + psllw :: proc(a: simd.i16x8, count: simd.i16x8) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.pslli.d") + psllid :: proc(a: simd.i32x4, #const imm8: u32) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.psll.d") + pslld :: proc(a: simd.i32x4, count: simd.i32x4) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.pslli.q") + pslliq :: proc(a: simd.i64x2, #const imm8: u32) -> simd.i64x2 --- + @(link_name="llvm.x86.sse2.psll.q") + psllq :: proc(a: simd.i64x2, count: simd.i64x2) -> simd.i64x2 --- + @(link_name="llvm.x86.sse2.psrai.w") + psraiw :: proc(a: simd.i16x8, #const imm8: u32) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.psra.w") + psraw :: proc(a: simd.i16x8, count: simd.i16x8) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.psrai.d") + psraid :: proc(a: simd.i32x4, #const imm8: u32) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.psra.d") + psrad :: proc(a: simd.i32x4, count: simd.i32x4) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.psrli.w") + psrliw :: proc(a: simd.i16x8, #const imm8: u32) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.psrl.w") + psrlw :: proc(a: simd.i16x8, count: simd.i16x8) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.psrli.d") + psrlid :: proc(a: simd.i32x4, #const imm8: u32) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.psrl.d") + psrld :: proc(a: simd.i32x4, count: simd.i32x4) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.psrli.q") + psrliq :: proc(a: simd.i64x2, #const imm8: u32) -> simd.i64x2 --- + @(link_name="llvm.x86.sse2.psrl.q") + psrlq :: proc(a: simd.i64x2, count: simd.i64x2) -> simd.i64x2 --- + @(link_name="llvm.x86.sse2.cvtdq2ps") + cvtdq2ps :: proc(a: simd.i32x4) -> __m128 --- + @(link_name="llvm.x86.sse2.cvtps2dq") + cvtps2dq :: proc(a: __m128) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.maskmov.dqu") + maskmovdqu :: proc(a: simd.i8x16, mask: simd.i8x16, mem_addr: rawptr) --- + @(link_name="llvm.x86.sse2.packsswb.128") + packsswb :: proc(a: simd.i16x8, b: simd.i16x8) -> simd.i8x16 --- + @(link_name="llvm.x86.sse2.packssdw.128") + packssdw :: proc(a: simd.i32x4, b: simd.i32x4) -> simd.i16x8 --- + @(link_name="llvm.x86.sse2.packuswb.128") + packuswb :: proc(a: simd.i16x8, b: simd.i16x8) -> simd.u8x16 --- + @(link_name="llvm.x86.sse2.pmovmskb.128") + pmovmskb :: proc(a: simd.i8x16) -> i32 --- + @(link_name="llvm.x86.sse2.max.sd") + maxsd :: proc(a: __m128d, b: __m128d) -> __m128d --- + @(link_name="llvm.x86.sse2.max.pd") + maxpd :: proc(a: __m128d, b: __m128d) -> __m128d --- + @(link_name="llvm.x86.sse2.min.sd") + minsd :: proc(a: __m128d, b: __m128d) -> __m128d --- + @(link_name="llvm.x86.sse2.min.pd") + minpd :: proc(a: __m128d, b: __m128d) -> __m128d --- + @(link_name="llvm.x86.sse2.sqrt.sd") + sqrtsd :: proc(a: __m128d) -> __m128d --- + @(link_name="llvm.x86.sse2.sqrt.pd") + sqrtpd :: proc(a: __m128d) -> __m128d --- + @(link_name="llvm.x86.sse2.cmp.sd") + cmpsd :: proc(a: __m128d, b: __m128d, imm8: i8) -> __m128d --- + @(link_name="llvm.x86.sse2.cmp.pd") + cmppd :: proc(a: __m128d, b: __m128d, imm8: i8) -> __m128d --- + @(link_name="llvm.x86.sse2.comieq.sd") + comieqsd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.comilt.sd") + comiltsd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.comile.sd") + comilesd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.comigt.sd") + comigtsd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.comige.sd") + comigesd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.comineq.sd") + comineqsd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.ucomieq.sd") + ucomieqsd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.ucomilt.sd") + ucomiltsd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.ucomile.sd") + ucomilesd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.ucomigt.sd") + ucomigtsd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.ucomige.sd") + ucomigesd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.ucomineq.sd") + ucomineqsd :: proc(a: __m128d, b: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.movmsk.pd") + movmskpd :: proc(a: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.cvtpd2ps") + cvtpd2ps :: proc(a: __m128d) -> __m128 --- + @(link_name="llvm.x86.sse2.cvtps2pd") + cvtps2pd :: proc(a: __m128) -> __m128d --- + @(link_name="llvm.x86.sse2.cvtpd2dq") + cvtpd2dq :: proc(a: __m128d) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.cvtsd2si") + cvtsd2si :: proc(a: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.cvtsd2ss") + cvtsd2ss :: proc(a: __m128, b: __m128d) -> __m128 --- + @(link_name="llvm.x86.sse2.cvtss2sd") + cvtss2sd :: proc(a: __m128d, b: __m128) -> __m128d --- + @(link_name="llvm.x86.sse2.cvttpd2dq") + cvttpd2dq :: proc(a: __m128d) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.cvttsd2si") + cvttsd2si :: proc(a: __m128d) -> i32 --- + @(link_name="llvm.x86.sse2.cvttps2dq") + cvttps2dq :: proc(a: __m128) -> simd.i32x4 --- + @(link_name="llvm.x86.sse2.storeu.dq") + storeudq :: proc(mem_addr: rawptr, a: __m128i) --- + @(link_name="llvm.x86.sse2.storeu.pd") + storeupd :: proc(mem_addr: rawptr, a: __m128d) --- +} From c48ef7d70bed18edc8feea63d6deceb10133aa12 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 16:39:54 +0100 Subject: [PATCH 53/86] Add shifts --- core/simd/x86/sse.odin | 20 +-- core/simd/x86/sse2.odin | 295 ++++++++++++++++++++++++++++----------- core/simd/x86/types.odin | 15 +- 3 files changed, 233 insertions(+), 97 deletions(-) diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin index b43ccb953..50211872e 100644 --- a/core/simd/x86/sse.odin +++ b/core/simd/x86/sse.odin @@ -107,24 +107,16 @@ _mm_max_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { } _mm_and_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { - x := transmute(__m128i)a - y := transmute(__m128i)b - return transmute(__m128)simd.and(x, y) + return transmute(__m128)simd.and(transmute(__m128i)a, transmute(__m128i)b) } _mm_andnot_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { - x := transmute(__m128i)a - y := transmute(__m128i)b - return transmute(__m128)simd.and_not(x, y) + return transmute(__m128)simd.and_not(transmute(__m128i)a, transmute(__m128i)b) } _mm_or_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { - x := transmute(__m128i)a - y := transmute(__m128i)b - return transmute(__m128)simd.or(x, y) + return transmute(__m128)simd.or(transmute(__m128i)a, transmute(__m128i)b) } _mm_xor_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { - x := transmute(__m128i)a - y := transmute(__m128i)b - return transmute(__m128)simd.xor(x, y) + return transmute(__m128)simd.xor(transmute(__m128i)a, transmute(__m128i)b) } @@ -301,7 +293,7 @@ _mm_movelh_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 0, 1, 4, 5) } -_mm_movemask_ps :: proc(a: __m128) -> u32 { +_mm_movemask_ps :: #force_inline proc "c" (a: __m128) -> u32 { return movmskps(a) } @@ -421,7 +413,7 @@ _MM_TRANSPOSE4_PS :: #force_inline proc "c" (row0, row1, row2, row3: ^__m128) { row3^ = _mm_movelh_ps(tmp3, tmp1) } -_mm_stream_ps :: proc(addr: [^]f32, a: __m128) { +_mm_stream_ps :: #force_inline proc "c" (addr: [^]f32, a: __m128) { intrinsics.nontemporal_store((^__m128)(addr), a) } diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index b50e33d28..dd616c219 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -17,83 +17,214 @@ _mm_mfence :: #force_inline proc "c" () { } _mm_add_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i8x16)a - y := transmute(simd.i8x16)b - return transmute(__m128i)simd.add(x, y) + return transmute(__m128i)simd.add(transmute(i8x16)a, transmute(i8x16)b) } _mm_add_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i16x8)a - y := transmute(simd.i16x8)b - return transmute(__m128i)simd.add(x, y) + return transmute(__m128i)simd.add(transmute(i16x8)a, transmute(i16x8)b) } _mm_add_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i32x4)a - y := transmute(simd.i32x4)b - return transmute(__m128i)simd.add(x, y) + return transmute(__m128i)simd.add(transmute(i32x4)a, transmute(i32x4)b) } _mm_add_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i64x2)a - y := transmute(simd.i64x2)b - return transmute(__m128i)simd.add(x, y) + return transmute(__m128i)simd.add(transmute(i64x2)a, transmute(i64x2)b) } _mm_adds_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i8x16)a - y := transmute(simd.i8x16)b - return transmute(__m128i)simd.add_sat(x, y) + return transmute(__m128i)simd.add_sat(transmute(i8x16)a, transmute(i8x16)b) } _mm_adds_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i16x8)a - y := transmute(simd.i16x8)b - return transmute(__m128i)simd.add_sat(x, y) + return transmute(__m128i)simd.add_sat(transmute(i16x8)a, transmute(i16x8)b) } _mm_adds_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.u8x16)a - y := transmute(simd.u8x16)b - return transmute(__m128i)simd.add_sat(x, y) + return transmute(__m128i)simd.add_sat(transmute(u8x16)a, transmute(u8x16)b) } _mm_adds_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.u16x8)a - y := transmute(simd.u16x8)b - return transmute(__m128i)simd.add_sat(x, y) + return transmute(__m128i)simd.add_sat(transmute(u16x8)a, transmute(u16x8)b) } _mm_avg_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.u8x16)a - y := transmute(simd.u8x16)b - return transmute(__m128i)pavgb(x, y) + return transmute(__m128i)pavgb(transmute(u8x16)a, transmute(u8x16)b) } _mm_avg_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.u16x8)a - y := transmute(simd.u16x8)b - return transmute(__m128i)pavgw(x, y) + return transmute(__m128i)pavgw(transmute(u16x8)a, transmute(u16x8)b) } _mm_madd_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i16x8)a - y := transmute(simd.i16x8)b - return transmute(__m128i)pmaddwd(x, y) + return transmute(__m128i)pmaddwd(transmute(i16x8)a, transmute(i16x8)b) } _mm_max_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i16x8)a - y := transmute(simd.i16x8)b - return transmute(__m128i)pmaxsw(x, y) + return transmute(__m128i)pmaxsw(transmute(i16x8)a, transmute(i16x8)b) } _mm_max_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.u8x16)a - y := transmute(simd.u8x16)b - return transmute(__m128i)pmaxub(x, y) + return transmute(__m128i)pmaxub(transmute(u8x16)a, transmute(u8x16)b) } _mm_min_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.i16x8)a - y := transmute(simd.i16x8)b - return transmute(__m128i)pminsw(x, y) + return transmute(__m128i)pminsw(transmute(i16x8)a, transmute(i16x8)b) } _mm_min_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { - x := transmute(simd.u8x16)a - y := transmute(simd.u8x16)b - return transmute(__m128i)pminub(x, y) + return transmute(__m128i)pminub(transmute(u8x16)a, transmute(u8x16)b) } +_mm_mulhi_epi16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)pmulhw(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_mulhi_epu16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)pmulhuw(transmute(u16x8)a, transmute(u16x8)b) +} +_mm_mullo_epi16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.mul(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_mul_epu32 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)pmuludq(transmute(u32x4)a, transmute(u32x4)b) +} +_mm_sad_epu8 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)psadbw(transmute(u8x16)a, transmute(u8x16)b) +} +_mm_sub_epi8 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.sub(transmute(i8x16)a, transmute(i8x16)b) +} +_mm_sub_epi16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.sub(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_sub_epi32 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.sub(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_sub_epi64 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.sub(transmute(i64x2)a, transmute(i64x2)b) +} +_mm_subs_epi8 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.sub_sat(transmute(i8x16)a, transmute(i8x16)b) +} +_mm_subs_epi16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.sub_sat(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_subs_epu8 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.sub_sat(transmute(u8x16)a, transmute(u8x16)b) +} +_mm_subs_epu16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.sub_sat(transmute(u16x8)a, transmute(u16x8)b) +} + + + +@(private) +_mm_slli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + shift :: IMM8 & 0xff + + return transmute(__m128i)simd.shuffle( + transmute(i8x16)a, + i8x16(0), + 0 when shift > 15 else (16 - shift + 0), + 1 when shift > 15 else (16 - shift + 1), + 2 when shift > 15 else (16 - shift + 2), + 3 when shift > 15 else (16 - shift + 3), + 4 when shift > 15 else (16 - shift + 4), + 5 when shift > 15 else (16 - shift + 5), + 6 when shift > 15 else (16 - shift + 6), + 7 when shift > 15 else (16 - shift + 7), + 8 when shift > 15 else (16 - shift + 8), + 9 when shift > 15 else (16 - shift + 9), + 10 when shift > 15 else (16 - shift + 10), + 11 when shift > 15 else (16 - shift + 11), + 12 when shift > 15 else (16 - shift + 12), + 13 when shift > 15 else (16 - shift + 13), + 14 when shift > 15 else (16 - shift + 14), + 15 when shift > 15 else (16 - shift + 15), + ) +} + +@(private) +_mm_srli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + shift :: IMM8 + return transmute(__m128i)simd.shuffle( + transmute(i8x16)a, + i8x16(0), + 0 + 16 when shift > 15 else (shift + 0), + 1 + 16 when shift > 15 else (shift + 1), + 2 + 16 when shift > 15 else (shift + 2), + 3 + 16 when shift > 15 else (shift + 3), + 4 + 16 when shift > 15 else (shift + 4), + 5 + 16 when shift > 15 else (shift + 5), + 6 + 16 when shift > 15 else (shift + 6), + 7 + 16 when shift > 15 else (shift + 7), + 8 + 16 when shift > 15 else (shift + 8), + 9 + 16 when shift > 15 else (shift + 9), + 10 + 16 when shift > 15 else (shift + 10), + 11 + 16 when shift > 15 else (shift + 11), + 12 + 16 when shift > 15 else (shift + 12), + 13 + 16 when shift > 15 else (shift + 13), + 14 + 16 when shift > 15 else (shift + 14), + 15 + 16 when shift > 15 else (shift + 15), + ) +} + + +_mm_slli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return _mm_slli_si128_impl(a, IMM8) +} +_mm_bslli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return _mm_slli_si128_impl(a, IMM8) +} + + + +_mm_bsrli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return _mm_srli_si128_impl(a, IMM8) +} +_mm_slli_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return transmute(__m128i)pslliw(transmute(i16x8)a, IMM8) +} +_mm_sll_epi16 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { + return transmute(__m128i)psllw(transmute(i16x8)a, transmute(i16x8)count) +} +_mm_slli_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return transmute(__m128i)psllid(transmute(i32x4)a, IMM8) +} +_mm_sll_epi32 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { + return transmute(__m128i)pslld(transmute(i32x4)a, transmute(i32x4)count) +} +_mm_slli_epi64 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return transmute(__m128i)pslliq(transmute(i64x2)a, IMM8) +} +_mm_sll_epi64 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { + return transmute(__m128i)psllq(transmute(i64x2)a, transmute(i64x2)count) +} +_mm_srai_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return transmute(__m128i)psraiw(transmute(i16x8)a. IMM8) +} +_mm_sra_epi16 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { + return transmute(__m128i)psraw(transmute(i16x8)a, transmute(i16x8)count) +} +_mm_srai_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return transmute(__m128i)psraid(transmute(i32x4)a, IMM8) +} +_mm_sra_epi32 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { + return transmute(__m128i)psrad(transmute(i32x4)a, transmute(i32x4)count) +} + +_mm_srli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return _mm_srli_si128_impl(a, IMM8) +} +_mm_srli_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return transmute(__m128i)psrliw(transmute(i16x8)a. IMM8) +} +_mm_srl_epi16 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { + return transmute(__m128i)psrlw(transmute(i16x8)a, transmute(i16x8)count) +} +_mm_srli_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return transmute(__m128i)psrlid(transmute(i32x4)a, IMM8) +} +_mm_srl_epi32 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { + return transmute(__m128i)psrld(transmute(i32x4)a, transmute(i32x4)count) +} +_mm_srli_epi64 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + return transmute(__m128i)psrliq(transmute(i64x2)a, IMM8) +} +_mm_srl_epi64 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { + return transmute(__m128i)psrlq(transmute(i64x2)a, transmute(i64x2)count) +} + + + _mm_castpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { return transmute(__m128)a } @@ -140,73 +271,73 @@ foreign _ { @(link_name="llvm.x86.sse2.mfence") mfence :: proc() --- @(link_name="llvm.x86.sse2.pavg.b") - pavgb :: proc(a, b: simd.u8x16) -> simd.u8x16 --- + pavgb :: proc(a, b: u8x16) -> u8x16 --- @(link_name="llvm.x86.sse2.pavg.w") - pavgw :: proc(a, b: simd.u16x8) -> simd.u16x8 --- + pavgw :: proc(a, b: u16x8) -> u16x8 --- @(link_name="llvm.x86.sse2.pmadd.wd") - pmaddwd :: proc(a, b: simd.i16x8) -> simd.i32x4 --- + pmaddwd :: proc(a, b: i16x8) -> i32x4 --- @(link_name="llvm.x86.sse2.pmaxs.w") - pmaxsw :: proc(a, b: simd.i16x8) -> simd.i16x8 --- + pmaxsw :: proc(a, b: i16x8) -> i16x8 --- @(link_name="llvm.x86.sse2.pmaxu.b") - pmaxub :: proc(a, b: simd.u8x16) -> simd.u8x16 --- + pmaxub :: proc(a, b: u8x16) -> u8x16 --- @(link_name="llvm.x86.sse2.pmins.w") - pminsw :: proc(a, b: simd.i16x8) -> simd.i16x8 --- + pminsw :: proc(a, b: i16x8) -> i16x8 --- @(link_name="llvm.x86.sse2.pminu.b") - pminub :: proc(a, b: simd.u8x16) -> simd.u8x16 --- + pminub :: proc(a, b: u8x16) -> u8x16 --- @(link_name="llvm.x86.sse2.pmulh.w") - pmulhw :: proc(a, b: simd.i16x8) -> simd.i16x8 --- + pmulhw :: proc(a, b: i16x8) -> i16x8 --- @(link_name="llvm.x86.sse2.pmulhu.w") - pmulhuw :: proc(a, b: simd.u16x8) -> simd.u16x8 --- + pmulhuw :: proc(a, b: u16x8) -> u16x8 --- @(link_name="llvm.x86.sse2.pmulu.dq") - pmuludq :: proc(a, b: simd.u32x4) -> simd.u64x2 --- + pmuludq :: proc(a, b: u32x4) -> u64x2 --- @(link_name="llvm.x86.sse2.psad.bw") - psadbw :: proc(a, b: simd.u8x16) -> simd.u64x2 --- + psadbw :: proc(a, b: u8x16) -> u64x2 --- @(link_name="llvm.x86.sse2.pslli.w") - pslliw :: proc(a: simd.i16x8, #const imm8: u32) -> simd.i16x8 --- + pslliw :: proc(a: i16x8, #const imm8: u32) -> i16x8 --- @(link_name="llvm.x86.sse2.psll.w") - psllw :: proc(a: simd.i16x8, count: simd.i16x8) -> simd.i16x8 --- + psllw :: proc(a: i16x8, count: i16x8) -> i16x8 --- @(link_name="llvm.x86.sse2.pslli.d") - psllid :: proc(a: simd.i32x4, #const imm8: u32) -> simd.i32x4 --- + psllid :: proc(a: i32x4, #const imm8: u32) -> i32x4 --- @(link_name="llvm.x86.sse2.psll.d") - pslld :: proc(a: simd.i32x4, count: simd.i32x4) -> simd.i32x4 --- + pslld :: proc(a: i32x4, count: i32x4) -> i32x4 --- @(link_name="llvm.x86.sse2.pslli.q") - pslliq :: proc(a: simd.i64x2, #const imm8: u32) -> simd.i64x2 --- + pslliq :: proc(a: i64x2, #const imm8: u32) -> i64x2 --- @(link_name="llvm.x86.sse2.psll.q") - psllq :: proc(a: simd.i64x2, count: simd.i64x2) -> simd.i64x2 --- + psllq :: proc(a: i64x2, count: i64x2) -> i64x2 --- @(link_name="llvm.x86.sse2.psrai.w") - psraiw :: proc(a: simd.i16x8, #const imm8: u32) -> simd.i16x8 --- + psraiw :: proc(a: i16x8, #const imm8: u32) -> i16x8 --- @(link_name="llvm.x86.sse2.psra.w") - psraw :: proc(a: simd.i16x8, count: simd.i16x8) -> simd.i16x8 --- + psraw :: proc(a: i16x8, count: i16x8) -> i16x8 --- @(link_name="llvm.x86.sse2.psrai.d") - psraid :: proc(a: simd.i32x4, #const imm8: u32) -> simd.i32x4 --- + psraid :: proc(a: i32x4, #const imm8: u32) -> i32x4 --- @(link_name="llvm.x86.sse2.psra.d") - psrad :: proc(a: simd.i32x4, count: simd.i32x4) -> simd.i32x4 --- + psrad :: proc(a: i32x4, count: i32x4) -> i32x4 --- @(link_name="llvm.x86.sse2.psrli.w") - psrliw :: proc(a: simd.i16x8, #const imm8: u32) -> simd.i16x8 --- + psrliw :: proc(a: i16x8, #const imm8: u32) -> i16x8 --- @(link_name="llvm.x86.sse2.psrl.w") - psrlw :: proc(a: simd.i16x8, count: simd.i16x8) -> simd.i16x8 --- + psrlw :: proc(a: i16x8, count: i16x8) -> i16x8 --- @(link_name="llvm.x86.sse2.psrli.d") - psrlid :: proc(a: simd.i32x4, #const imm8: u32) -> simd.i32x4 --- + psrlid :: proc(a: i32x4, #const imm8: u32) -> i32x4 --- @(link_name="llvm.x86.sse2.psrl.d") - psrld :: proc(a: simd.i32x4, count: simd.i32x4) -> simd.i32x4 --- + psrld :: proc(a: i32x4, count: i32x4) -> i32x4 --- @(link_name="llvm.x86.sse2.psrli.q") - psrliq :: proc(a: simd.i64x2, #const imm8: u32) -> simd.i64x2 --- + psrliq :: proc(a: i64x2, #const imm8: u32) -> i64x2 --- @(link_name="llvm.x86.sse2.psrl.q") - psrlq :: proc(a: simd.i64x2, count: simd.i64x2) -> simd.i64x2 --- + psrlq :: proc(a: i64x2, count: i64x2) -> i64x2 --- @(link_name="llvm.x86.sse2.cvtdq2ps") - cvtdq2ps :: proc(a: simd.i32x4) -> __m128 --- + cvtdq2ps :: proc(a: i32x4) -> __m128 --- @(link_name="llvm.x86.sse2.cvtps2dq") - cvtps2dq :: proc(a: __m128) -> simd.i32x4 --- + cvtps2dq :: proc(a: __m128) -> i32x4 --- @(link_name="llvm.x86.sse2.maskmov.dqu") - maskmovdqu :: proc(a: simd.i8x16, mask: simd.i8x16, mem_addr: rawptr) --- + maskmovdqu :: proc(a: i8x16, mask: i8x16, mem_addr: rawptr) --- @(link_name="llvm.x86.sse2.packsswb.128") - packsswb :: proc(a: simd.i16x8, b: simd.i16x8) -> simd.i8x16 --- + packsswb :: proc(a: i16x8, b: i16x8) -> i8x16 --- @(link_name="llvm.x86.sse2.packssdw.128") - packssdw :: proc(a: simd.i32x4, b: simd.i32x4) -> simd.i16x8 --- + packssdw :: proc(a: i32x4, b: i32x4) -> i16x8 --- @(link_name="llvm.x86.sse2.packuswb.128") - packuswb :: proc(a: simd.i16x8, b: simd.i16x8) -> simd.u8x16 --- + packuswb :: proc(a: i16x8, b: i16x8) -> u8x16 --- @(link_name="llvm.x86.sse2.pmovmskb.128") - pmovmskb :: proc(a: simd.i8x16) -> i32 --- + pmovmskb :: proc(a: i8x16) -> i32 --- @(link_name="llvm.x86.sse2.max.sd") maxsd :: proc(a: __m128d, b: __m128d) -> __m128d --- @(link_name="llvm.x86.sse2.max.pd") @@ -254,7 +385,7 @@ foreign _ { @(link_name="llvm.x86.sse2.cvtps2pd") cvtps2pd :: proc(a: __m128) -> __m128d --- @(link_name="llvm.x86.sse2.cvtpd2dq") - cvtpd2dq :: proc(a: __m128d) -> simd.i32x4 --- + cvtpd2dq :: proc(a: __m128d) -> i32x4 --- @(link_name="llvm.x86.sse2.cvtsd2si") cvtsd2si :: proc(a: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.cvtsd2ss") @@ -262,11 +393,11 @@ foreign _ { @(link_name="llvm.x86.sse2.cvtss2sd") cvtss2sd :: proc(a: __m128d, b: __m128) -> __m128d --- @(link_name="llvm.x86.sse2.cvttpd2dq") - cvttpd2dq :: proc(a: __m128d) -> simd.i32x4 --- + cvttpd2dq :: proc(a: __m128d) -> i32x4 --- @(link_name="llvm.x86.sse2.cvttsd2si") cvttsd2si :: proc(a: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.cvttps2dq") - cvttps2dq :: proc(a: __m128) -> simd.i32x4 --- + cvttps2dq :: proc(a: __m128) -> i32x4 --- @(link_name="llvm.x86.sse2.storeu.dq") storeudq :: proc(mem_addr: rawptr, a: __m128i) --- @(link_name="llvm.x86.sse2.storeu.pd") diff --git a/core/simd/x86/types.odin b/core/simd/x86/types.odin index 3c94d74a0..06a2cd41e 100644 --- a/core/simd/x86/types.odin +++ b/core/simd/x86/types.odin @@ -1,6 +1,8 @@ //+build i386, amd64 package simd_x86 +import "core:simd" + bf16 :: u16 __m128i :: #simd[2]i64 @@ -41,4 +43,15 @@ _MM_MANTISSA_NORM_ENUM :: i32 /// The `MM_MANTISSA_SIGN_ENUM` type used to specify mantissa signed operations in AVX-512 intrinsics. _MM_MANTISSA_SIGN_ENUM :: i32 -_MM_PERM_ENUM :: i32 \ No newline at end of file +_MM_PERM_ENUM :: i32 + +@(private) u8x16 :: simd.u8x16 +@(private) i8x16 :: simd.i8x16 +@(private) u16x8 :: simd.u16x8 +@(private) i16x8 :: simd.i16x8 +@(private) u32x4 :: simd.u32x4 +@(private) i32x4 :: simd.i32x4 +@(private) u64x2 :: simd.u64x2 +@(private) i64x2 :: simd.i64x2 +@(private) f32x4 :: simd.f32x4 +@(private) f64x2 :: simd.f64x2 From 4e30a64d9fbc8b8eede7a0d857bfa26df5857fe2 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 16:49:52 +0100 Subject: [PATCH 54/86] Add more sse2 intrinsics --- core/simd/x86/sse2.odin | 109 ++++++++++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 22 deletions(-) diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index dd616c219..970704bb7 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -64,43 +64,43 @@ _mm_min_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { } -_mm_mulhi_epi16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_mulhi_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmulhw(transmute(i16x8)a, transmute(i16x8)b) } -_mm_mulhi_epu16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_mulhi_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmulhuw(transmute(u16x8)a, transmute(u16x8)b) } -_mm_mullo_epi16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_mullo_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.mul(transmute(i16x8)a, transmute(i16x8)b) } -_mm_mul_epu32 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_mul_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmuludq(transmute(u32x4)a, transmute(u32x4)b) } -_mm_sad_epu8 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_sad_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psadbw(transmute(u8x16)a, transmute(u8x16)b) } -_mm_sub_epi8 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_sub_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i8x16)a, transmute(i8x16)b) } -_mm_sub_epi16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_sub_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i16x8)a, transmute(i16x8)b) } -_mm_sub_epi32 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_sub_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i32x4)a, transmute(i32x4)b) } -_mm_sub_epi64 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_sub_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i64x2)a, transmute(i64x2)b) } -_mm_subs_epi8 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_subs_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(i8x16)a, transmute(i8x16)b) } -_mm_subs_epi16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_subs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(i16x8)a, transmute(i16x8)b) } -_mm_subs_epu8 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_subs_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(u8x16)a, transmute(u8x16)b) } -_mm_subs_epu16 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { +_mm_subs_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(u16x8)a, transmute(u16x8)b) } @@ -166,64 +166,129 @@ _mm_bslli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { } - _mm_bsrli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_srli_si128_impl(a, IMM8) } _mm_slli_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)pslliw(transmute(i16x8)a, IMM8) } -_mm_sll_epi16 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { +_mm_sll_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psllw(transmute(i16x8)a, transmute(i16x8)count) } _mm_slli_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psllid(transmute(i32x4)a, IMM8) } -_mm_sll_epi32 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { +_mm_sll_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)pslld(transmute(i32x4)a, transmute(i32x4)count) } _mm_slli_epi64 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)pslliq(transmute(i64x2)a, IMM8) } -_mm_sll_epi64 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { +_mm_sll_epi64 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psllq(transmute(i64x2)a, transmute(i64x2)count) } _mm_srai_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psraiw(transmute(i16x8)a. IMM8) } -_mm_sra_epi16 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { +_mm_sra_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psraw(transmute(i16x8)a, transmute(i16x8)count) } _mm_srai_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psraid(transmute(i32x4)a, IMM8) } -_mm_sra_epi32 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { +_mm_sra_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrad(transmute(i32x4)a, transmute(i32x4)count) } + _mm_srli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_srli_si128_impl(a, IMM8) } _mm_srli_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrliw(transmute(i16x8)a. IMM8) } -_mm_srl_epi16 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { +_mm_srl_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrlw(transmute(i16x8)a, transmute(i16x8)count) } _mm_srli_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrlid(transmute(i32x4)a, IMM8) } -_mm_srl_epi32 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { +_mm_srl_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrld(transmute(i32x4)a, transmute(i32x4)count) } _mm_srli_epi64 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrliq(transmute(i64x2)a, IMM8) } -_mm_srl_epi64 :: #force_inline proc "c" (a: __m128i, count: __m128i) -> __m128i { +_mm_srl_epi64 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrlq(transmute(i64x2)a, transmute(i64x2)count) } +_mm_and_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return simd.and(a, b) +} +_mm_andnot_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return simd.and_not(b, a) +} +_mm_or_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return simd.or(a, b) +} +_mm_xor_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return simd.xor(a, b) +} +_mm_cmpeq_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_eq(transmute(i8x16)a, transmute(i8x16)b) +} +_mm_cmpeq_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_eq(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_cmpeq_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_eq(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_cmpgt_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_gt(transmute(i8x16)a, transmute(i8x16)b) +} +_mm_cmpgt_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_gt(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_cmpgt_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_gt(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_cmplt_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_lt(transmute(i8x16)a, transmute(i8x16)b) +} +_mm_cmplt_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_lt(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_cmplt_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_lt(transmute(i32x4)a, transmute(i32x4)b) +} + + +_mm_cvtepi32_pd :: #force_inline proc "c" (a: __m128i) -> __m128d { + v := transmute(i32x4)a + return cast(__m128d)simd.shuffle(v, v, 0, 1) +} +_mm_cvtsi32_sd :: #force_inline proc "c" (a: __m128d, b: i32) -> __m128d { + return simd.replace(a, 0, f64(b)) +} +_mm_cvtepi32_ps :: #force_inline proc "c" (a: __m128i) -> __m128 { + return cvtdq2ps(transmute(i32x4)a) +} +_mm_cvtps_epi32 :: #force_inline proc "c" (a: __m128) -> __m128i { + return transmute(__m128i)cvtps2dq(a) +} +_mm_cvtsi32_si128 :: #force_inline proc "c" (a: i32) -> __m128i { + return transmute(__m128i)i32x4{a, 0, 0, 0} +} +_mm_cvtsi128_si32 :: #force_inline proc "c" (a: __m128i) -> i32 { + return simd.extract(transmute(i32x4)a, 0) +} + + + + + _mm_castpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { return transmute(__m128)a From 20fe6d102a8469eac180bc0051c5359acad07ac1 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 16:58:35 +0100 Subject: [PATCH 55/86] Add load and stores and sets --- core/simd/x86/sse2.odin | 75 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index 970704bb7..bba842820 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -1,6 +1,7 @@ //+build i386, amd64 package simd_x86 +import "core:intrinsics" import "core:simd" _mm_pause :: #force_inline proc "c" () { @@ -287,6 +288,80 @@ _mm_cvtsi128_si32 :: #force_inline proc "c" (a: __m128i) -> i32 { +_mm_set_epi64x :: #force_inline proc "c" (e1, e0: i64) -> __m128i { + return transmute(__m128i)i64x2{e0, e1} +} +_mm_set_epi32 :: #force_inline proc "c" (e3, e2, e1, e0: i32) -> __m128i { + return transmute(__m128i)i32x4{e0, e1, e2, e3} +} +_mm_set_epi16 :: #force_inline proc "c" (e7, e6, e5, e4, e3, e2, e1, e0: i16) -> __m128i { + return transmute(__m128i)i16x8{e0, e1, e2, e3, e4, e5, e6, e7} +} +_mm_set_epi8 :: #force_inline proc "c" (e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0: i8) -> __m128i { + return transmute(__m128i)i8x16{e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15} +} +_mm_set1_epi64x :: #force_inline proc "c" (a: i64) -> __m128i { + return _mm_set_epi64x(a, a) +} +_mm_set1_epi32 :: #force_inline proc "c" (a: i32) -> __m128i { + return _mm_set_epi32(a, a, a, a) +} +_mm_set1_epi16 :: #force_inline proc "c" (a: i16) -> __m128i { + return _mm_set_epi16(a, a, a, a, a, a, a, a) +} +_mm_set1_epi8 :: #force_inline proc "c" (a: i8) -> __m128i { + return _mm_set_epi8(a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a) +} +_mm_setr_epi32 :: #force_inline proc "c" (e3, e2, e1, e0: i32) -> __m128i { + return _mm_set_epi32(e0, e1, e2, e3) +} +_mm_setr_epi16 :: #force_inline proc "c" (e7, e6, e5, e4, e3, e2, e1, e0: i16) -> __m128i { + return _mm_set_epi16(e0, e1, e2, e3, e4, e5, e6, e7) +} +_mm_setr_epi8 :: #force_inline proc "c" (e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0: i8) -> __m128i { + return _mm_set_epi8(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15) +} +_mm_setzero_si128 :: #force_inline proc "c" () -> __m128i { + return _mm_set1_epi64x(0) +} + + +_mm_loadl_epi64 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { + return _mm_set_epi64x(0, intrinsics.unaligned_load((^i64)(mem_addr))) +} +_mm_load_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { + return mem_addr^ +} +_mm_loadu_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { + dst := _mm_undefined_si128() + intrinsics.mem_copy_non_overlapping(&dst, mem_addr, size_of(__m128i)) + return dst +} +_mm_maskmoveu_si128 :: #force_inline proc "c" (a, mask: __m128i, mem_addr: rawptr) { + maskmovdqu(transmute(i8x16)a, transmute(i8x16)mask, mem_addr) +} +_mm_store_si128 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { + mem_addr^ = a +} +_mm_storeu_si128 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { + storeudq(mem_addr, a) +} +_mm_storel_epi64 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { + a := a + intrinsics.mem_copy_non_overlapping(mem_addr, &a, 8) +} +_mm_stream_si128 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { + intrinsics.nontemporal_store(mem_addr, a) +} +_mm_stream_si32 :: #force_inline proc "c" (mem_addr: ^i32, a: i32) { + intrinsics.nontemporal_store(mem_addr, a) +} +_mm_move_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { + zero := _mm_setzero_si128() + return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)zero, 0, 2) +} + + From 20c5033b38e108b1e127cc6b8be61cce043cba9e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 17:07:48 +0100 Subject: [PATCH 56/86] Add pack and unpack --- core/simd/x86/sse2.odin | 95 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index bba842820..f52981639 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -364,6 +364,101 @@ _mm_move_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { +_mm_packs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)packsswb(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_packs_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)packssdw(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_packus_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)packuswb(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_extract_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> i32 { + return i32(simd.extract(transmute(u16x8)a, IMM8)) +} +_mm_insert_epi16 :: #force_inline proc "c" (a: __m128i, i: i32, $IMM8: u32) -> __m128i { + return i32(simd.replace(transmute(u16x8)a, IMM8, i16(i))) +} +_mm_movemask_epi8 :: #force_inline proc "c" (a: __m128i) -> i32 { + return pmovmskb(transmute(i8x16)a) +} +_mm_shuffle_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + v := transmute(i32x4)a + return transmute(__m128i)simd.shuffle( + v, + v, + IMM8 & 0b11, + (IMM8 >> 2) & 0b11, + (IMM8 >> 4) & 0b11, + (IMM8 >> 6) & 0b11, + ) +} +_mm_shufflehi_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + v := transmute(i16x8)a + return transmute(__m128i)simd.shuffle( + v, + v, + 0, + 1, + 2, + 3, + (IMM8 & 0b11) + 4, + ((IMM8 >> 2) & 0b11) + 4, + ((IMM8 >> 4) & 0b11) + 4, + ((IMM8 >> 6) & 0b11) + 4, + ) +} +_mm_shufflelo_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { + v := transmute(i16x8)a + return transmute(__m128i)simd.shuffle( + v, + v, + IMM8 & 0b11, + (IMM8 >> 2) & 0b11, + (IMM8 >> 4) & 0b11, + (IMM8 >> 6) & 0b11, + 4, + 5, + 6, + 7, + ) +} +_mm_unpackhi_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.shuffle( + transmute(i8x16)a, + transmute(i8x16)b, + 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, + ) +} +_mm_unpackhi_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.shuffle(transmute(i16x8)a, transmute(i16x8)b, 4, 12, 5, 13, 6, 14, 7, 15) +} +_mm_unpackhi_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.shuffle(transmute(i32x4)a, transmute(i32x4)b, 2, 6, 3, 7) +} +_mm_unpackhi_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)b, 1, 3) +} +_mm_unpacklo_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.shuffle( + transmute(i8x16)a, + transmute(i8x16)b, + 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23, + ) +} +_mm_unpacklo_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.shuffle(transmute(i16x8)a, transmute(i16x8)b, 0, 8, 1, 9, 2, 10, 3, 11) +} +_mm_unpacklo_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.shuffle(transmute(i32x4)a, transmute(i32x4)b, 0, 4, 1, 5) +} +_mm_unpacklo_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)b, 0, 2) +} + + + + _mm_castpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { return transmute(__m128)a From 5c10b35df7b174e883a5523b82f4124a1951a27d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 22:26:04 +0100 Subject: [PATCH 57/86] Fix sqrt for simd --- src/check_builtin.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index ba34a177b..ad227489b 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -3704,12 +3704,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 return false; } } - if (!is_type_float(x.type)) { - gbString xts = type_to_string(x.type); - error(x.expr, "Expected a floating point value for '%.*s', got %s", LIT(builtin_name), xts); - gb_string_free(xts); - return false; - } else if (x.mode == Addressing_Constant) { + if (is_type_float(x.type) && x.mode == Addressing_Constant) { f64 v = exact_value_to_f64(x.value); operand->mode = Addressing_Constant; From 1c1f5e2231dc8e1ec992ec7399407f78c98dc283 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 22:56:11 +0100 Subject: [PATCH 58/86] Complete SSE2 --- core/simd/x86/sse2.odin | 332 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 309 insertions(+), 23 deletions(-) diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index f52981639..359f19062 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -459,6 +459,292 @@ _mm_unpacklo_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { +_mm_add_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(a, 0, _mm_cvtsd_f64(a) + _mm_cvtsd_f64(b)) +} +_mm_add_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.add(a, b) +} +_mm_div_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(a, 0, _mm_cvtsd_f64(a) / _mm_cvtsd_f64(b)) +} +_mm_div_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.div(a, b) +} +_mm_max_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return maxsd(a, b) +} +_mm_max_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return maxpd(a, b) +} +_mm_min_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return minsd(a, b) +} +_mm_min_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return minpd(a, b) +} +_mm_mul_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(a, 0, _mm_cvtsd_f64(a) * _mm_cvtsd_f64(b)) +} +_mm_mul_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.mul(a, b) +} +_mm_sqrt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(a, 0, _mm_cvtsd_f64(sqrtsd(b))) +} +_mm_sqrt_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { + return simd.sqrt(a) +} +_mm_sub_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(a, 0, _mm_cvtsd_f64(a) - _mm_cvtsd_f64(b)) +} +_mm_sub_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.sub(a, b) +} +_mm_and_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return transmute(__m128d)_mm_and_si128(transmute(__m128i)a, transmute(__m128i)b) +} +_mm_andnot_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return transmute(__m128d)_mm_andnot_si128(transmute(__m128i)a, transmute(__m128i)b) +} +_mm_or_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return transmute(__m128d)_mm_or_si128(transmute(__m128i)a, transmute(__m128i)b) +} +_mm_xor_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return transmute(__m128d)_mm_xor_si128(transmute(__m128i)a, transmute(__m128i)b) +} + + + + +_mm_cmpeq_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmpsd(a, b, 0) +} +_mm_cmplt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmpsd(a, b, 1) +} +_mm_cmple_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmpsd(a, b, 2) +} +_mm_cmpgt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(_mm_cmplt_sd(b, a), 1, simd.extract(a, 1)) +} +_mm_cmpge_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(_mm_cmple_sd(b, a), 1, simd.extract(a, 1)) +} +_mm_cmpord_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmpsd(a, b, 7) +} +_mm_cmpunord_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmpsd(a, b, 3) +} +_mm_cmpneq_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmpsd(a, b, 4) +} +_mm_cmpnlt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmpsd(a, b, 5) +} +_mm_cmpnle_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmpsd(a, b, 6) +} +_mm_cmpngt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(_mm_cmpnlt_sd(b, a), 1, simd.extract(a, 1)) +} +_mm_cmpnge_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return simd.replace(_mm_cmpnle_sd(b, a), 1, simd.extract(a, 1)) +} +_mm_cmpeq_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmppd(a, b, 0) +} +_mm_cmplt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmppd(a, b, 1) +} +_mm_cmple_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmppd(a, b, 2) +} +_mm_cmpgt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return _mm_cmplt_pd(b, a) +} +_mm_cmpge_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return _mm_cmple_pd(b, a) +} +_mm_cmpord_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmppd(a, b, 7) +} +_mm_cmpunord_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmppd(a, b, 3) +} +_mm_cmpneq_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmppd(a, b, 4) +} +_mm_cmpnlt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmppd(a, b, 5) +} +_mm_cmpnle_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return cmppd(a, b, 6) +} +_mm_cmpngt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return _mm_cmpnlt_pd(b, a) +} +_mm_cmpnge_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return _mm_cmpnle_pd(b, a) +} +_mm_comieq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return comieqsd(a, b) +} +_mm_comilt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return comiltsd(a, b) +} +_mm_comile_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return comilesd(a, b) +} +_mm_comigt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return comigtsd(a, b) +} +_mm_comige_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return comigesd(a, b) +} +_mm_comineq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return comineqsd(a, b) +} +_mm_ucomieq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return ucomieqsd(a, b) +} +_mm_ucomilt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return ucomiltsd(a, b) +} +_mm_ucomile_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return ucomilesd(a, b) +} +_mm_ucomigt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return ucomigtsd(a, b) +} +_mm_ucomige_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return ucomigesd(a, b) +} +_mm_ucomineq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { + return ucomineqsd(a, b) +} + + + + + +_mm_cvtpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { + return cvtpd2ps(a) +} +_mm_cvtps_pd :: #force_inline proc "c" (a: __m128) -> __m128d { + return cvtps2pd(a) +} +_mm_cvtpd_epi32 :: #force_inline proc "c" (a: __m128d) -> __m128i { + return transmute(__m128i)cvtpd2dq(a) +} +_mm_cvtsd_si32 :: #force_inline proc "c" (a: __m128d) -> i32 { + return cvtsd2si(a) +} +_mm_cvtsd_ss :: #force_inline proc "c" (a, b: __m128d) -> __m128 { + return cvtsd2ss(a, b) +} +_mm_cvtsd_f64 :: #force_inline proc "c" (a: __m128d) -> f64 { + return simd.extract(a, 0) +} +_mm_cvtss_sd :: #force_inline proc "c" (a, b: __m128) -> __m128d { + return cvtss2sd(a, b) +} +_mm_cvttpd_epi32 :: #force_inline proc "c" (a: __m128d) -> __m128i { + return transmute(__m128i)cvttpd2dq(a) +} +_mm_cvttsd_si32 :: #force_inline proc "c" (a: __m128d) -> i32 { + return cvttsd2si(a) +} +_mm_cvttps_epi32 :: #force_inline proc "c" (a: __m128) -> __m128i { + return transmute(__m128i)cvttps2dq(a) +} +_mm_set_sd :: #force_inline proc "c" (a: f64) -> __m128d { + return _mm_set_pd(0.0, a) +} +_mm_set1_pd :: #force_inline proc "c" (a: f64) -> __m128d { + return _mm_set_pd(a, a) +} +_mm_set_pd1 :: #force_inline proc "c" (a: f64) -> __m128d { + return _mm_set_pd(a, a) +} +_mm_set_pd :: #force_inline proc "c" (a: f64, b: f64) -> __m128d { + return __m128d{b, a} +} +_mm_setr_pd :: #force_inline proc "c" (a: f64, b: f64) -> __m128d { + return _mm_set_pd(b, a) +} +_mm_setzero_pd :: #force_inline proc "c" () -> __m128d { + return _mm_set_pd(0.0, 0.0) +} +_mm_movemask_pd :: #force_inline proc "c" (a: __m128d) -> i32 { + return movmskpd(a) +} +_mm_load_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { + return (^__m128d)(mem_addr)^ +} +_mm_load_sd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { + return _mm_setr_pd(mem_addr^, 0.) +} +_mm_loadh_pd :: #force_inline proc "c" (a: __m128d, mem_addr: ^f64) -> __m128d { + return _mm_setr_pd(simd.extract(a, 0), mem_addr^) +} +_mm_loadl_pd :: #force_inline proc "c" (a: __m128d, mem_addr: ^f64) -> __m128d { + return _mm_setr_pd(mem_addr^, simd.extract(a, 1)) +} +_mm_stream_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + intrinsics.nontemporal_store((^__m128d)(mem_addr), a) +} +_mm_store_sd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + mem_addr^ = simd.extract(a, 0) +} +_mm_store_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + (^__m128d)(mem_addr)^ = a +} +_mm_storeu_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + storeupd(mem_addr, a) +} +_mm_store1_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + (^__m128d)(mem_addr)^ = simd.shuffle(a, a, 0, 0) +} +_mm_store_pd1 :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + (^__m128d)(mem_addr)^ = simd.shuffle(a, a, 0, 0) +} +_mm_storer_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + (^__m128d)(mem_addr)^ = simd.shuffle(a, a, 1, 0) +} +_mm_storeh_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + mem_addr^ = simd.extract(a, 1) +} +_mm_storel_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { + mem_addr^ = simd.extract(a, 0) +} +_mm_load1_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { + d := mem_addr^ + return _mm_setr_pd(d, d) +} +_mm_load_pd1 :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { + return _mm_load1_pd(mem_addr) +} +_mm_loadr_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { + a := _mm_load_pd(mem_addr) + return simd.shuffle(a, a, 1, 0) +} +_mm_loadu_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { + dst := _mm_undefined_pd() + intrinsics.mem_copy_non_overlapping(&dst, mem_addr, size_of(__m128d)) + return dst +} +_mm_shuffle_pd :: #force_inline proc "c" (a, b: __m128d, $MASK: u32) -> __m128d { + return simd.shuffle(a, b, MASK&0b1, ((MASK>>1)&0b1) + 2) +} +_mm_move_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return _mm_setr_pd(simd.extract(b, 0), simd.extract(a, 1)) +} + + + _mm_castpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { return transmute(__m128)a @@ -566,53 +852,53 @@ foreign _ { @(link_name="llvm.x86.sse2.maskmov.dqu") maskmovdqu :: proc(a: i8x16, mask: i8x16, mem_addr: rawptr) --- @(link_name="llvm.x86.sse2.packsswb.128") - packsswb :: proc(a: i16x8, b: i16x8) -> i8x16 --- + packsswb :: proc(a, b: i16x8) -> i8x16 --- @(link_name="llvm.x86.sse2.packssdw.128") - packssdw :: proc(a: i32x4, b: i32x4) -> i16x8 --- + packssdw :: proc(a, b: i32x4) -> i16x8 --- @(link_name="llvm.x86.sse2.packuswb.128") - packuswb :: proc(a: i16x8, b: i16x8) -> u8x16 --- + packuswb :: proc(a, b: i16x8) -> u8x16 --- @(link_name="llvm.x86.sse2.pmovmskb.128") pmovmskb :: proc(a: i8x16) -> i32 --- @(link_name="llvm.x86.sse2.max.sd") - maxsd :: proc(a: __m128d, b: __m128d) -> __m128d --- + maxsd :: proc(a, b: __m128d) -> __m128d --- @(link_name="llvm.x86.sse2.max.pd") - maxpd :: proc(a: __m128d, b: __m128d) -> __m128d --- + maxpd :: proc(a, b: __m128d) -> __m128d --- @(link_name="llvm.x86.sse2.min.sd") - minsd :: proc(a: __m128d, b: __m128d) -> __m128d --- + minsd :: proc(a, b: __m128d) -> __m128d --- @(link_name="llvm.x86.sse2.min.pd") - minpd :: proc(a: __m128d, b: __m128d) -> __m128d --- + minpd :: proc(a, b: __m128d) -> __m128d --- @(link_name="llvm.x86.sse2.sqrt.sd") sqrtsd :: proc(a: __m128d) -> __m128d --- @(link_name="llvm.x86.sse2.sqrt.pd") sqrtpd :: proc(a: __m128d) -> __m128d --- @(link_name="llvm.x86.sse2.cmp.sd") - cmpsd :: proc(a: __m128d, b: __m128d, imm8: i8) -> __m128d --- + cmpsd :: proc(a, b: __m128d, imm8: i8) -> __m128d --- @(link_name="llvm.x86.sse2.cmp.pd") - cmppd :: proc(a: __m128d, b: __m128d, imm8: i8) -> __m128d --- + cmppd :: proc(a, b: __m128d, imm8: i8) -> __m128d --- @(link_name="llvm.x86.sse2.comieq.sd") - comieqsd :: proc(a: __m128d, b: __m128d) -> i32 --- + comieqsd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.comilt.sd") - comiltsd :: proc(a: __m128d, b: __m128d) -> i32 --- + comiltsd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.comile.sd") - comilesd :: proc(a: __m128d, b: __m128d) -> i32 --- + comilesd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.comigt.sd") - comigtsd :: proc(a: __m128d, b: __m128d) -> i32 --- + comigtsd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.comige.sd") - comigesd :: proc(a: __m128d, b: __m128d) -> i32 --- + comigesd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.comineq.sd") - comineqsd :: proc(a: __m128d, b: __m128d) -> i32 --- + comineqsd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.ucomieq.sd") - ucomieqsd :: proc(a: __m128d, b: __m128d) -> i32 --- + ucomieqsd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.ucomilt.sd") - ucomiltsd :: proc(a: __m128d, b: __m128d) -> i32 --- + ucomiltsd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.ucomile.sd") - ucomilesd :: proc(a: __m128d, b: __m128d) -> i32 --- + ucomilesd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.ucomigt.sd") - ucomigtsd :: proc(a: __m128d, b: __m128d) -> i32 --- + ucomigtsd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.ucomige.sd") - ucomigesd :: proc(a: __m128d, b: __m128d) -> i32 --- + ucomigesd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.ucomineq.sd") - ucomineqsd :: proc(a: __m128d, b: __m128d) -> i32 --- + ucomineqsd :: proc(a, b: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.movmsk.pd") movmskpd :: proc(a: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.cvtpd2ps") @@ -624,9 +910,9 @@ foreign _ { @(link_name="llvm.x86.sse2.cvtsd2si") cvtsd2si :: proc(a: __m128d) -> i32 --- @(link_name="llvm.x86.sse2.cvtsd2ss") - cvtsd2ss :: proc(a: __m128, b: __m128d) -> __m128 --- + cvtsd2ss :: proc(a, b: __m128d) -> __m128 --- @(link_name="llvm.x86.sse2.cvtss2sd") - cvtss2sd :: proc(a: __m128d, b: __m128) -> __m128d --- + cvtss2sd :: proc(a, b: __m128) -> __m128d --- @(link_name="llvm.x86.sse2.cvttpd2dq") cvttpd2dq :: proc(a: __m128d) -> i32x4 --- @(link_name="llvm.x86.sse2.cvttsd2si") From 8518d3b2327885538993afa2655b83abb71b82e3 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 22:57:16 +0100 Subject: [PATCH 59/86] Rename to `non_temporaral_*` --- core/simd/x86/sse.odin | 2 +- core/simd/x86/sse2.odin | 6 +++--- src/check_builtin.cpp | 4 ++-- src/checker_builtin_procs.hpp | 8 ++++---- src/llvm_backend_proc.cpp | 12 ++++++------ 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin index 50211872e..e697d5f5e 100644 --- a/core/simd/x86/sse.odin +++ b/core/simd/x86/sse.odin @@ -414,7 +414,7 @@ _MM_TRANSPOSE4_PS :: #force_inline proc "c" (row0, row1, row2, row3: ^__m128) { } _mm_stream_ps :: #force_inline proc "c" (addr: [^]f32, a: __m128) { - intrinsics.nontemporal_store((^__m128)(addr), a) + intrinsics.non_temporal_store((^__m128)(addr), a) } diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index 359f19062..a47d2a09d 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -351,10 +351,10 @@ _mm_storel_epi64 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { intrinsics.mem_copy_non_overlapping(mem_addr, &a, 8) } _mm_stream_si128 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { - intrinsics.nontemporal_store(mem_addr, a) + intrinsics.non_temporal_store(mem_addr, a) } _mm_stream_si32 :: #force_inline proc "c" (mem_addr: ^i32, a: i32) { - intrinsics.nontemporal_store(mem_addr, a) + intrinsics.non_temporal_store(mem_addr, a) } _mm_move_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { zero := _mm_setzero_si128() @@ -694,7 +694,7 @@ _mm_loadl_pd :: #force_inline proc "c" (a: __m128d, mem_addr: ^f64) -> __m128d { return _mm_setr_pd(mem_addr^, simd.extract(a, 1)) } _mm_stream_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { - intrinsics.nontemporal_store((^__m128d)(mem_addr), a) + intrinsics.non_temporal_store((^__m128d)(mem_addr), a) } _mm_store_sd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { mem_addr^ = simd.extract(a, 0) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index ad227489b..9fa9cc590 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -4021,7 +4021,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 case BuiltinProc_volatile_store: case BuiltinProc_unaligned_store: - case BuiltinProc_nontemporal_store: + case BuiltinProc_non_temporal_store: case BuiltinProc_atomic_store: { Type *elem = nullptr; @@ -4069,7 +4069,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 case BuiltinProc_volatile_load: case BuiltinProc_unaligned_load: - case BuiltinProc_nontemporal_load: + case BuiltinProc_non_temporal_load: case BuiltinProc_atomic_load: { Type *elem = nullptr; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 2dd775193..1d7ee0a34 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -80,8 +80,8 @@ enum BuiltinProcId { BuiltinProc_unaligned_store, BuiltinProc_unaligned_load, - BuiltinProc_nontemporal_store, - BuiltinProc_nontemporal_load, + BuiltinProc_non_temporal_store, + BuiltinProc_non_temporal_load, BuiltinProc_prefetch_read_instruction, BuiltinProc_prefetch_read_data, @@ -369,8 +369,8 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("unaligned_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("unaligned_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("nontemporal_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, - {STR_LIT("nontemporal_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("non_temporal_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, + {STR_LIT("non_temporal_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("prefetch_read_instruction"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("prefetch_read_data"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 2b7cad5cd..4d0df2861 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -2111,7 +2111,7 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, return {}; case BuiltinProc_volatile_store: - case BuiltinProc_nontemporal_store: + case BuiltinProc_non_temporal_store: case BuiltinProc_atomic_store: case BuiltinProc_atomic_store_explicit: { lbValue dst = lb_build_expr(p, ce->args[0]); @@ -2121,8 +2121,8 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, LLVMValueRef instr = LLVMBuildStore(p->builder, val.value, dst.value); switch (id) { case BuiltinProc_volatile_store: LLVMSetVolatile(instr, true); break; - case BuiltinProc_nontemporal_store: - // TODO(bill): BuiltinProc_nontemporal_store + case BuiltinProc_non_temporal_store: + // TODO(bill): BuiltinProc_non_temporal_store break; case BuiltinProc_atomic_store: LLVMSetOrdering(instr, LLVMAtomicOrderingSequentiallyConsistent); break; case BuiltinProc_atomic_store_explicit: LLVMSetOrdering(instr, llvm_atomic_ordering_from_odin(ce->args[2])); break; @@ -2134,7 +2134,7 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, } case BuiltinProc_volatile_load: - case BuiltinProc_nontemporal_load: + case BuiltinProc_non_temporal_load: case BuiltinProc_atomic_load: case BuiltinProc_atomic_load_explicit: { lbValue dst = lb_build_expr(p, ce->args[0]); @@ -2142,8 +2142,8 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, LLVMValueRef instr = LLVMBuildLoad(p->builder, dst.value, ""); switch (id) { case BuiltinProc_volatile_load: LLVMSetVolatile(instr, true); break; - case BuiltinProc_nontemporal_load: - // TODO(bill): BuiltinProc_nontemporal_load + case BuiltinProc_non_temporal_load: + // TODO(bill): BuiltinProc_non_temporal_load break; case BuiltinProc_atomic_load: LLVMSetOrdering(instr, LLVMAtomicOrderingSequentiallyConsistent); break; case BuiltinProc_atomic_load_explicit: LLVMSetOrdering(instr, llvm_atomic_ordering_from_odin(ce->args[1])); break; From 026540040df94309280d6991eba6b2dcc303bd76 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 23:00:52 +0100 Subject: [PATCH 60/86] Add SSE3 support --- core/simd/x86/sse3.odin | 58 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 core/simd/x86/sse3.odin diff --git a/core/simd/x86/sse3.odin b/core/simd/x86/sse3.odin new file mode 100644 index 000000000..6468ea268 --- /dev/null +++ b/core/simd/x86/sse3.odin @@ -0,0 +1,58 @@ +//+build i386, amd64 +package simd_x86 + +import "core:intrinsics" +import "core:simd" + +_mm_addsub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return addsubps(a, b) +} +_mm_addsub_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { + return addsubpd(a, b) +} +_mm_hadd_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { + return haddpd(a, b) +} +_mm_hadd_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return haddps(a, b) +} +_mm_hsub_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { + return hsubpd(a, b) +} +_mm_hsub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return hsubps(a, b) +} +_mm_lddqu_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { + return transmute(__m128i)lddqu(mem_addr) +} +_mm_movedup_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { + return simd.shuffle(a, a, 0, 0) +} +_mm_loaddup_pd :: #force_inline proc "c" (mem_addr: [^]f64) -> __m128d { + return _mm_load1_pd(mem_addr) +} +_mm_movehdup_ps :: #force_inline proc "c" (a: __m128) -> __m128 { + return simd.shuffle(a, a, 1, 1, 3, 3) +} +_mm_moveldup_ps :: #force_inline proc "c" (a: __m128) -> __m128 { + return simd.shuffle(a, a, 0, 0, 2, 2) +} + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name = "llvm.x86.sse3.addsub.ps") + addsubps :: proc(a, b: __m128) -> __m128 --- + @(link_name = "llvm.x86.sse3.addsub.pd") + addsubpd :: proc(a: __m128d, b: __m128d) -> __m128d --- + @(link_name = "llvm.x86.sse3.hadd.pd") + haddpd :: proc(a: __m128d, b: __m128d) -> __m128d --- + @(link_name = "llvm.x86.sse3.hadd.ps") + haddps :: proc(a, b: __m128) -> __m128 --- + @(link_name = "llvm.x86.sse3.hsub.pd") + hsubpd :: proc(a: __m128d, b: __m128d) -> __m128d --- + @(link_name = "llvm.x86.sse3.hsub.ps") + hsubps :: proc(a, b: __m128) -> __m128 --- + @(link_name = "llvm.x86.sse3.ldu.dq") + lddqu :: proc(mem_addr: rawptr) -> i8x16 --- +} \ No newline at end of file From 4db533ff71e9f11ff21e845509790a355020a944 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Fri, 27 May 2022 23:07:33 +0100 Subject: [PATCH 61/86] Add ssse3 support --- core/simd/x86/ssse3.odin | 124 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 core/simd/x86/ssse3.odin diff --git a/core/simd/x86/ssse3.odin b/core/simd/x86/ssse3.odin new file mode 100644 index 000000000..920dddd85 --- /dev/null +++ b/core/simd/x86/ssse3.odin @@ -0,0 +1,124 @@ +//+build i386, amd64 +package simd_x86 + +import "core:intrinsics" +import "core:simd" +_ :: simd + +_mm_abs_epi8 :: #force_inline proc "c" (a: __m128i) -> __m128i { + return transmute(__m128i)pabsb128(transmute(i8x16)a) +} +_mm_abs_epi16 :: #force_inline proc "c" (a: __m128i) -> __m128i { + return transmute(__m128i)pabsw128(transmute(i16x8)a) +} +_mm_abs_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { + return transmute(__m128i)pabsd128(transmute(i32x4)a) +} +_mm_shuffle_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pshufb128(transmute(u8x16)a, transmute(u8x16)b) +} +_mm_alignr_epi8 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u32) -> __m128i { + shift :: IMM8 + + // If palignr is shifting the pair of vectors more than the size of two + // lanes, emit zero. + if shift > 32 { + return _mm_set1_epi8(0) + } + a, b := a, b + if shift > 16 { + a, b = _mm_set1_epi8(0), a + } + + return transmute(__m128i)simd.shuffle( + transmute(i8x16)b, + transmute(i8x16)a, + 0 when shift > 32 else shift - 16 + 0 when shift > 16 else shift + 0, + 1 when shift > 32 else shift - 16 + 1 when shift > 16 else shift + 1, + 2 when shift > 32 else shift - 16 + 2 when shift > 16 else shift + 2, + 3 when shift > 32 else shift - 16 + 3 when shift > 16 else shift + 3, + 4 when shift > 32 else shift - 16 + 4 when shift > 16 else shift + 4, + 5 when shift > 32 else shift - 16 + 5 when shift > 16 else shift + 5, + 6 when shift > 32 else shift - 16 + 6 when shift > 16 else shift + 6, + 7 when shift > 32 else shift - 16 + 7 when shift > 16 else shift + 7, + 8 when shift > 32 else shift - 16 + 8 when shift > 16 else shift + 8, + 9 when shift > 32 else shift - 16 + 9 when shift > 16 else shift + 9, + 10 when shift > 32 else shift - 16 + 10 when shift > 16 else shift + 10, + 11 when shift > 32 else shift - 16 + 11 when shift > 16 else shift + 11, + 12 when shift > 32 else shift - 16 + 12 when shift > 16 else shift + 12, + 13 when shift > 32 else shift - 16 + 13 when shift > 16 else shift + 13, + 14 when shift > 32 else shift - 16 + 14 when shift > 16 else shift + 14, + 15 when shift > 32 else shift - 16 + 15 when shift > 16 else shift + 15, + ) +} + + +_mm_hadd_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)phaddw128(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_hadds_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)phaddsw128(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_hadd_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)phaddd128(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_hsub_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)phsubw128(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_hsubs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)phsubsw128(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_hsub_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)phsubd128(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_maddubs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pmaddubsw128(transmute(u8x16)a, transmute(i8x16)b) +} +_mm_mulhrs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pmulhrsw128(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_sign_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)psignb128(transmute(i8x16)a, transmute(i8x16)b) +} +_mm_sign_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)psignw128(transmute(i16x8)a, transmute(i16x8)b) +} +_mm_sign_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)psignd128(transmute(i32x4)a, transmute(i32x4)b) +} + + + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name = "llvm.x86.ssse3.pabs.b.128") + pabsb128 :: proc(a: i8x16) -> u8x16 --- + @(link_name = "llvm.x86.ssse3.pabs.w.128") + pabsw128 :: proc(a: i16x8) -> u16x8 --- + @(link_name = "llvm.x86.ssse3.pabs.d.128") + pabsd128 :: proc(a: i32x4) -> u32x4 --- + @(link_name = "llvm.x86.ssse3.pshuf.b.128") + pshufb128 :: proc(a, b: u8x16) -> u8x16 --- + @(link_name = "llvm.x86.ssse3.phadd.w.128") + phaddw128 :: proc(a, b: i16x8) -> i16x8 --- + @(link_name = "llvm.x86.ssse3.phadd.sw.128") + phaddsw128 :: proc(a, b: i16x8) -> i16x8 --- + @(link_name = "llvm.x86.ssse3.phadd.d.128") + phaddd128 :: proc(a, b: i32x4) -> i32x4 --- + @(link_name = "llvm.x86.ssse3.phsub.w.128") + phsubw128 :: proc(a, b: i16x8) -> i16x8 --- + @(link_name = "llvm.x86.ssse3.phsub.sw.128") + phsubsw128 :: proc(a, b: i16x8) -> i16x8 --- + @(link_name = "llvm.x86.ssse3.phsub.d.128") + phsubd128 :: proc(a, b: i32x4) -> i32x4 --- + @(link_name = "llvm.x86.ssse3.pmadd.ub.sw.128") + pmaddubsw128 :: proc(a: u8x16, b: i8x16) -> i16x8 --- + @(link_name = "llvm.x86.ssse3.pmul.hr.sw.128") + pmulhrsw128 :: proc(a, b: i16x8) -> i16x8 --- + @(link_name = "llvm.x86.ssse3.psign.b.128") + psignb128 :: proc(a, b: i8x16) -> i8x16 --- + @(link_name = "llvm.x86.ssse3.psign.w.128") + psignw128 :: proc(a, b: i16x8) -> i16x8 --- + @(link_name = "llvm.x86.ssse3.psign.d.128") + psignd128 :: proc(a, b: i32x4) -> i32x4 ---} \ No newline at end of file From 618d3bf62fbcfa6ca7f827ad4090143b8535b4a2 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 28 May 2022 13:42:58 +0100 Subject: [PATCH 62/86] Improve vector comparison `==` `!=` for horizontal reduction --- src/llvm_backend_expr.cpp | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index 55b76b93a..1894e85f6 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -2585,16 +2585,35 @@ lbValue lb_emit_comp(lbProcedure *p, TokenKind op_kind, lbValue left, lbValue ri } GB_ASSERT_MSG(mask != nullptr, "Unhandled comparison kind %s (%s) %.*s %s (%s)", type_to_string(left.type), type_to_string(base_type(left.type)), LIT(token_strings[op_kind]), type_to_string(right.type), type_to_string(base_type(right.type))); - // TODO(bill): is this a good approach to dealing with comparisons of vectors? - char const *name = "llvm.vector.reduce.umax"; - LLVMTypeRef types[1] = {LLVMTypeOf(mask)}; - unsigned id = LLVMLookupIntrinsicID(name, gb_strlen(name)); - GB_ASSERT_MSG(id != 0, "Unable to find %s.%s", name, LLVMPrintTypeToString(types[0])); - LLVMValueRef ip = LLVMGetIntrinsicDeclaration(p->module->mod, id, types, gb_count_of(types)); + /* NOTE(bill, 2022-05-28): + Thanks to Per Vognsen, sign extending to + a vector of the same width as the input vector, bit casting to an integer, + and then comparing against zero is the better option + See: https://lists.llvm.org/pipermail/llvm-dev/2012-September/053046.html - LLVMValueRef args[1] = {}; - args[0] = mask; - res.value = LLVMBuildCall(p->builder, ip, args, gb_count_of(args), ""); + // Example assuming 128-bit vector + + %1 = <4 x float> ... + %2 = <4 x float> ... + %3 = fcmp oeq <4 x float> %1, %2 + %4 = sext <4 x i1> %3 to <4 x i32> + %5 = bitcast <4 x i32> %4 to i128 + %6 = icmp ne i128 %5, 0 + br i1 %6, label %true1, label %false2 + + This will result in 1 cmpps + 1 ptest + 1 br + (even without SSE4.1, contrary to what the mail list states, because of pmovmskb) + + */ + + unsigned count = cast(unsigned)get_array_type_count(a); + unsigned elem_sz = cast(unsigned)(type_size_of(elem)*8); + LLVMTypeRef mask_type = LLVMVectorType(LLVMIntTypeInContext(p->module->ctx, elem_sz), count); + mask = LLVMBuildSExtOrBitCast(p->builder, mask, mask_type, ""); + + LLVMTypeRef mask_int_type = LLVMIntTypeInContext(p->module->ctx, cast(unsigned)(8*type_size_of(a))); + LLVMValueRef mask_int = LLVMBuildBitCast(p->builder, mask, mask_int_type, ""); + res.value = LLVMBuildICmp(p->builder, LLVMIntNE, mask_int, LLVMConstNull(LLVMTypeOf(mask_int)), ""); return res; } else { From d7eaf0f87b7677e84bf1f65c34305801748c39ee Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 28 May 2022 15:41:11 +0100 Subject: [PATCH 63/86] Add `intrinsics.x86_cpuid` and `intrinsics.x86_xgetbv` --- core/intrinsics/intrinsics.odin | 4 +++ core/simd/x86/ssse3.odin | 3 +- src/build_settings.cpp | 9 +++++ src/check_builtin.cpp | 60 ++++++++++++++++++++++++++++++- src/checker_builtin_procs.hpp | 6 ++++ src/llvm_backend_proc.cpp | 63 +++++++++++++++++++++++---------- 6 files changed, 124 insertions(+), 21 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 8becd998d..89f9a5f20 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -278,6 +278,10 @@ wasm_memory_size :: proc(index: uintptr) -> int --- wasm_memory_atomic_wait32 :: proc(ptr: ^u32, expected: u32, timeout_ns: i64) -> u32 --- wasm_memory_atomic_notify32 :: proc(ptr: ^u32, waiters: u32) -> (waiters_woken_up: u32) --- +// x86 Targets (i386, amd64) +cpuid :: proc(ax, cx: u32) -> (eax, ebc, ecx, edx: u32) --- +xgetbv :: proc(cx: u32) -> (eax, edx: u32) --- + // Darwin targets only objc_object :: struct{} diff --git a/core/simd/x86/ssse3.odin b/core/simd/x86/ssse3.odin index 920dddd85..4abd4c84c 100644 --- a/core/simd/x86/ssse3.odin +++ b/core/simd/x86/ssse3.odin @@ -121,4 +121,5 @@ foreign _ { @(link_name = "llvm.x86.ssse3.psign.w.128") psignw128 :: proc(a, b: i16x8) -> i16x8 --- @(link_name = "llvm.x86.ssse3.psign.d.128") - psignd128 :: proc(a, b: i32x4) -> i32x4 ---} \ No newline at end of file + psignd128 :: proc(a, b: i32x4) -> i32x4 --- +} \ No newline at end of file diff --git a/src/build_settings.cpp b/src/build_settings.cpp index b458d8308..27e09a0db 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -629,6 +629,15 @@ bool is_arch_wasm(void) { return false; } +bool is_arch_x86(void) { + switch (build_context.metrics.arch) { + case TargetArch_i386: + case TargetArch_amd64: + return true; + } + return false; +} + bool allow_check_foreign_filepath(void) { switch (build_context.metrics.arch) { case TargetArch_wasm32: diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 9fa9cc590..f8ac545be 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -1060,8 +1060,8 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call operand->type = t_untyped_integer; operand->mode = Addressing_Constant; operand->value = exact_value_i64(result); + return true; } - default: GB_PANIC("Unhandled simd intrinsic: %.*s", LIT(builtin_name)); } @@ -5275,6 +5275,64 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 } break; + case BuiltinProc_x86_cpuid: + { + if (!is_arch_x86()) { + error(call, "'%.*s' is only allowed on x86 targets (i386, amd64)", LIT(builtin_name)); + return false; + } + + Operand ax = {}; + Operand cx = {}; + + check_expr_with_type_hint(c, &ax, ce->args[0], t_u32); if (ax.mode == Addressing_Invalid) return false; + check_expr_with_type_hint(c, &cx, ce->args[1], t_u32); if (cx.mode == Addressing_Invalid) return false; + convert_to_typed(c, &ax, t_u32); if (ax.mode == Addressing_Invalid) return false; + convert_to_typed(c, &cx, t_u32); if (cx.mode == Addressing_Invalid) return false; + if (!are_types_identical(ax.type, t_u32)) { + gbString str = type_to_string(ax.type); + error(ax.expr, "'%.*s' expected a u32, got %s", LIT(builtin_name), str); + gb_string_free(str); + return false; + } + if (!are_types_identical(cx.type, t_u32)) { + gbString str = type_to_string(cx.type); + error(cx.expr, "'%.*s' expected a u32, got %s", LIT(builtin_name), str); + gb_string_free(str); + return false; + } + Type *types[4] = {t_u32, t_u32, t_u32, t_u32}; // eax ebc ecx edx + operand->type = alloc_type_tuple_from_field_types(types, gb_count_of(types), false, false); + operand->mode = Addressing_Value; + operand->value = {}; + return true; + } + break; + case BuiltinProc_x86_xgetbv: + { + if (!is_arch_x86()) { + error(call, "'%.*s' is only allowed on x86 targets (i386, amd64)", LIT(builtin_name)); + return false; + } + + Operand cx = {}; + check_expr_with_type_hint(c, &cx, ce->args[0], t_u32); if (cx.mode == Addressing_Invalid) return false; + convert_to_typed(c, &cx, t_u32); if (cx.mode == Addressing_Invalid) return false; + if (!are_types_identical(cx.type, t_u32)) { + gbString str = type_to_string(cx.type); + error(cx.expr, "'%.*s' expected a u32, got %s", LIT(builtin_name), str); + gb_string_free(str); + return false; + } + + Type *types[2] = {t_u32, t_u32}; + operand->type = alloc_type_tuple_from_field_types(types, gb_count_of(types), false, false); + operand->mode = Addressing_Value; + operand->value = {}; + return true; + } + break; + } return true; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 1d7ee0a34..35f14c6a8 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -187,6 +187,9 @@ BuiltinProc__simd_end, // Platform specific intrinsics BuiltinProc_syscall, + BuiltinProc_x86_cpuid, + BuiltinProc_x86_xgetbv, + // Constant type tests BuiltinProc__type_begin, @@ -470,10 +473,13 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_rotate_right"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_x86__MM_SHUFFLE"), 4, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("syscall"), 1, true, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("x86_cpuid"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("x86_xgetbv"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 4d0df2861..8cbb533bc 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1007,9 +1007,9 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const lbValue res = {}; res.type = tv.type; - lbValue arg0 = lb_build_expr(p, ce->args[0]); - lbValue arg1 = {}; - lbValue arg2 = {}; + lbValue arg0 = {}; if (ce->args.count > 0) arg0 = lb_build_expr(p, ce->args[0]); + lbValue arg1 = {}; if (ce->args.count > 1) arg0 = lb_build_expr(p, ce->args[1]); + lbValue arg2 = {}; if (ce->args.count > 2) arg0 = lb_build_expr(p, ce->args[2]); Type *elem = base_array_type(arg0.type); @@ -1024,7 +1024,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_mul: case BuiltinProc_simd_div: case BuiltinProc_simd_rem: - arg1 = lb_build_expr(p, ce->args[1]); if (is_float) { switch (builtin_id) { case BuiltinProc_simd_add: op_code = LLVMFAdd; break; @@ -1062,7 +1061,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_shr: // Odin logic case BuiltinProc_simd_shl_masked: // C logic case BuiltinProc_simd_shr_masked: // C logic - arg1 = lb_build_expr(p, ce->args[1]); { i64 sz = type_size_of(elem); GB_ASSERT(arg0.type->kind == Type_SimdVector); @@ -1098,7 +1096,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_or: case BuiltinProc_simd_xor: case BuiltinProc_simd_and_not: - arg1 = lb_build_expr(p, ce->args[1]); switch (builtin_id) { case BuiltinProc_simd_and: op_code = LLVMAnd; break; case BuiltinProc_simd_or: op_code = LLVMOr; break; @@ -1143,7 +1140,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const } return res; case BuiltinProc_simd_max: - arg1 = lb_build_expr(p, ce->args[1]); if (is_float) { LLVMValueRef cond = LLVMBuildFCmp(p->builder, LLVMRealOGT, arg0.value, arg1.value, ""); res.value = LLVMBuildSelect(p->builder, cond, arg0.value, arg1.value, ""); @@ -1158,7 +1154,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_lanes_le: case BuiltinProc_simd_lanes_gt: case BuiltinProc_simd_lanes_ge: - arg1 = lb_build_expr(p, ce->args[1]); if (is_float) { LLVMRealPredicate pred = cast(LLVMRealPredicate)0; switch (builtin_id) { @@ -1193,12 +1188,9 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const break; case BuiltinProc_simd_extract: - arg1 = lb_build_expr(p, ce->args[1]); res.value = LLVMBuildExtractElement(p->builder, arg0.value, arg1.value, ""); return res; case BuiltinProc_simd_replace: - arg1 = lb_build_expr(p, ce->args[1]); - arg2 = lb_build_expr(p, ce->args[2]); res.value = LLVMBuildInsertElement(p->builder, arg0.value, arg2.value, arg1.value, ""); return res; @@ -1283,12 +1275,9 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_shuffle: { - arg1 = lb_build_expr(p, ce->args[1]); - Type *vt = arg0.type; GB_ASSERT(vt->kind == Type_SimdVector); - i64 indices_count = ce->args.count-2; i64 max_count = vt->SimdVector.count*2; GB_ASSERT(indices_count <= max_count); @@ -1394,8 +1383,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_add_sat: case BuiltinProc_simd_sub_sat: { - arg1 = lb_build_expr(p, ce->args[1]); - char const *name = nullptr; switch (builtin_id) { case BuiltinProc_simd_add_sat: name = is_signed ? "llvm.sadd.sat" : "llvm.uadd.sat"; break; @@ -1417,9 +1404,6 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const case BuiltinProc_simd_clamp: { - arg1 = lb_build_expr(p, ce->args[1]); - arg2 = lb_build_expr(p, ce->args[2]); - LLVMValueRef v = arg0.value; LLVMValueRef min = arg1.value; LLVMValueRef max = arg2.value; @@ -2737,6 +2721,47 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, return res; } + + case BuiltinProc_x86_cpuid: + { + Type *param_types[2] = {t_u32, t_u32}; + Type *type = alloc_type_proc_from_types(param_types, gb_count_of(param_types), tv.type, false, ProcCC_None); + LLVMTypeRef func_type = LLVMGetElementType(lb_type(p->module, type)); + LLVMValueRef the_asm = llvm_get_inline_asm( + func_type, + str_lit("cpuid"), + str_lit("={ax},={bx},={cx},={dx},{ax},{cx}"), + true + ); + GB_ASSERT(the_asm != nullptr); + + LLVMValueRef args[2] = {}; + args[0] = lb_emit_conv(p, lb_build_expr(p, ce->args[0]), t_u32).value; + args[1] = lb_emit_conv(p, lb_build_expr(p, ce->args[1]), t_u32).value; + lbValue res = {}; + res.type = tv.type; + res.value = LLVMBuildCall2(p->builder, func_type, the_asm, args, gb_count_of(args), ""); + return res; + } + case BuiltinProc_x86_xgetbv: + { + Type *type = alloc_type_proc_from_types(&t_u32, 1, tv.type, false, ProcCC_None); + LLVMTypeRef func_type = LLVMGetElementType(lb_type(p->module, type)); + LLVMValueRef the_asm = llvm_get_inline_asm( + func_type, + str_lit("xgetbv"), + str_lit("={ax},={dx},{cx}"), + true + ); + GB_ASSERT(the_asm != nullptr); + + LLVMValueRef args[1] = {}; + args[0] = lb_emit_conv(p, lb_build_expr(p, ce->args[0]), t_u32).value; + lbValue res = {}; + res.type = tv.type; + res.value = LLVMBuildCall2(p->builder, func_type, the_asm, args, gb_count_of(args), ""); + return res; + } } GB_PANIC("Unhandled built-in procedure %.*s", LIT(builtin_procs[id].name)); From c60d7842cd829371129f52064dd8722516e5d197 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 28 May 2022 15:41:27 +0100 Subject: [PATCH 64/86] Remove old code --- core/sys/cpu/cpu.odin | 33 ------------------- core/sys/cpu/cpu_x86.odin | 67 --------------------------------------- 2 files changed, 100 deletions(-) delete mode 100644 core/sys/cpu/cpu.odin delete mode 100644 core/sys/cpu/cpu_x86.odin diff --git a/core/sys/cpu/cpu.odin b/core/sys/cpu/cpu.odin deleted file mode 100644 index b99fe01d8..000000000 --- a/core/sys/cpu/cpu.odin +++ /dev/null @@ -1,33 +0,0 @@ -package sys_cpu - -Cache_Line_Pad :: struct {_: [_cache_line_size]byte}; - -initialized: bool; - -x86: struct { - _: Cache_Line_Pad, - has_aes: bool, // AES hardware implementation (AES NI) - has_adx: bool, // Multi-precision add-carry instruction extensions - has_avx: bool, // Advanced vector extension - has_avx2: bool, // Advanced vector extension 2 - has_bmi1: bool, // Bit manipulation instruction set 1 - has_bmi2: bool, // Bit manipulation instruction set 2 - has_erms: bool, // Enhanced REP for MOVSB and STOSB - has_fma: bool, // Fused-multiply-add instructions - has_os_xsave: bool, // OS supports XSAVE/XRESTOR for saving/restoring XMM registers. - has_pclmulqdq: bool, // PCLMULQDQ instruction - most often used for AES-GCM - has_popcnt: bool, // Hamming weight instruction POPCNT. - has_rdrand: bool, // RDRAND instruction (on-chip random number generator) - has_rdseed: bool, // RDSEED instruction (on-chip random number generator) - has_sse2: bool, // Streaming SIMD extension 2 (always available on amd64) - has_sse3: bool, // Streaming SIMD extension 3 - has_ssse3: bool, // Supplemental streaming SIMD extension 3 - has_sse41: bool, // Streaming SIMD extension 4 and 4.1 - has_sse42: bool, // Streaming SIMD extension 4 and 4.2 - _: Cache_Line_Pad, -}; - - -init :: proc() { - _init(); -} diff --git a/core/sys/cpu/cpu_x86.odin b/core/sys/cpu/cpu_x86.odin deleted file mode 100644 index 146822e61..000000000 --- a/core/sys/cpu/cpu_x86.odin +++ /dev/null @@ -1,67 +0,0 @@ -//+build i386, amd64 -package sys_cpu - -_cache_line_size :: 64; - -cpuid :: proc(ax, cx: u32) -> (eax, ebc, ecx, edx: u32) { - return expand_to_tuple(asm(u32, u32) -> struct{eax, ebc, ecx, edx: u32} { - "cpuid", - "={ax},={bx},={cx},={dx},{ax},{cx}", - }(ax, cx)); -} - -xgetbv :: proc() -> (eax, edx: u32) { - return expand_to_tuple(asm(u32) -> struct{eax, edx: u32} { - "xgetbv", - "={ax},={dx},{cx}", - }(0)); -} - -_init :: proc() { - is_set :: proc(hwc: u32, value: u32) -> bool { - return hwc&value != 0; - } - - initialized = true; - - max_id, _, _, _ := cpuid(0, 0); - - if max_id < 1 { - return; - } - - _, _, ecx1, edx1 := cpuid(1, 0); - - x86.has_sse2 = is_set(26, edx1); - - x86.has_sse3 = is_set(0, ecx1); - x86.has_pclmulqdq = is_set(1, ecx1); - x86.has_ssse3 = is_set(9, ecx1); - x86.has_fma = is_set(12, ecx1); - x86.has_sse41 = is_set(19, ecx1); - x86.has_sse42 = is_set(20, ecx1); - x86.has_popcnt = is_set(23, ecx1); - x86.has_aes = is_set(25, ecx1); - x86.has_os_xsave = is_set(27, ecx1); - x86.has_rdrand = is_set(30, ecx1); - - os_supports_avx := false; - if x86.has_os_xsave { - eax, _ := xgetbv(); - os_supports_avx = is_set(1, eax) && is_set(2, eax); - } - - x86.has_avx = is_set(28, ecx1) && os_supports_avx; - - if max_id < 7 { - return; - } - - _, ebx7, _, _ := cpuid(7, 0); - x86.has_bmi1 = is_set(3, ebx7); - x86.has_avx2 = is_set(5, ebx7) && os_supports_avx; - x86.has_bmi2 = is_set(8, ebx7); - x86.has_erms = is_set(9, ebx7); - x86.has_rdseed = is_set(18, ebx7); - x86.has_adx = is_set(19, ebx7); -} From 910799cc5ff8032c1ff12709fced65d98b37902d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sat, 28 May 2022 15:54:41 +0100 Subject: [PATCH 65/86] Add `cpu_features` for `core:simd/x86` --- core/simd/x86/cpu.odin | 94 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 core/simd/x86/cpu.odin diff --git a/core/simd/x86/cpu.odin b/core/simd/x86/cpu.odin new file mode 100644 index 000000000..14e90c0f0 --- /dev/null +++ b/core/simd/x86/cpu.odin @@ -0,0 +1,94 @@ +//+build i386, amd64 +package simd_x86 + +import "core:intrinsics" + +// cpuid :: proc(ax, cx: u32) -> (eax, ebc, ecx, edx: u32) --- +cpuid :: intrinsics.x86_cpuid + +// xgetbv :: proc(cx: u32) -> (eax, edx: u32) --- +xgetbv :: intrinsics.x86_xgetbv + + +CPU_Feature :: enum u64 { + aes, // AES hardware implementation (AES NI) + adx, // Multi-precision add-carry instruction extensions + avx, // Advanced vector extension + avx2, // Advanced vector extension 2 + bmi1, // Bit manipulation instruction set 1 + bmi2, // Bit manipulation instruction set 2 + erms, // Enhanced REP for MOVSB and STOSB + fma, // Fused-multiply-add instructions + os_xsave, // OS supports XSAVE/XRESTOR for saving/restoring XMM registers. + pclmulqdq, // PCLMULQDQ instruction - most often used for AES-GCM + popcnt, // Hamming weight instruction POPCNT. + rdrand, // RDRAND instruction (on-chip random number generator) + rdseed, // RDSEED instruction (on-chip random number generator) + sse2, // Streaming SIMD extension 2 (always available on amd64) + sse3, // Streaming SIMD extension 3 + ssse3, // Supplemental streaming SIMD extension 3 + sse41, // Streaming SIMD extension 4 and 4.1 + sse42, // Streaming SIMD extension 4 and 4.2 +} + +CPU_Features :: distinct bit_set[CPU_Feature; u64] + +cpu_features: Maybe(CPU_Features) + +@(init, private) +init_cpu_features :: proc "c" () { + is_set :: #force_inline proc "c" (hwc: u32, value: u32) -> bool { + return hwc&value != 0 + } + try_set :: #force_inline proc "c" (set: ^CPU_Features, feature: CPU_Feature, hwc: u32, value: u32) { + if is_set(hwc, value) { + set^ += {feature} + } + } + + max_id, _, _, _ := cpuid(0, 0) + if max_id < 1 { + return + } + + set: CPU_Features + + _, _, ecx1, edx1 := cpuid(1, 0) + + try_set(&set, .sse2, 26, edx1) + try_set(&set, .sse3, 0, ecx1) + try_set(&set, .pclmulqdq, 1, ecx1) + try_set(&set, .ssse3, 9, ecx1) + try_set(&set, .fma, 12, ecx1) + try_set(&set, .sse41, 19, ecx1) + try_set(&set, .sse42, 20, ecx1) + try_set(&set, .popcnt, 23, ecx1) + try_set(&set, .aes, 25, ecx1) + try_set(&set, .os_xsave, 27, ecx1) + try_set(&set, .rdrand, 30, ecx1) + + os_supports_avx := false + if .os_xsave in set { + eax, _ := xgetbv(0) + os_supports_avx = is_set(1, eax) && is_set(2, eax) + } + if os_supports_avx { + try_set(&set, .avx, 28, ecx1) + } + + if max_id < 7 { + return + } + + _, ebx7, _, _ := cpuid(7, 0) + try_set(&set, .bmi1, 3, ebx7) + if os_supports_avx { + try_set(&set, .avx2, 5, ebx7) + } + try_set(&set, .bmi2, 8, ebx7) + try_set(&set, .erms, 9, ebx7) + try_set(&set, .rdseed, 18, ebx7) + try_set(&set, .adx, 19, ebx7) + + cpu_features = set +} From 3ad2cde833dc7b8ad83185f251f1ee544b4e3ef8 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 13:34:59 +0100 Subject: [PATCH 66/86] Add amd64 specific instructions --- core/simd/x86/sse.odin | 21 ++++++++++++++++++++ core/simd/x86/sse2.odin | 43 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin index e697d5f5e..eb1950ea3 100644 --- a/core/simd/x86/sse.odin +++ b/core/simd/x86/sse.odin @@ -417,6 +417,18 @@ _mm_stream_ps :: #force_inline proc "c" (addr: [^]f32, a: __m128) { intrinsics.non_temporal_store((^__m128)(addr), a) } +when ODIN_ARCH == .amd64 { + _mm_cvtss_si64 :: #force_inline proc "c"(a: __m128) -> i64 { + return cvtss2si64(a) + } + _mm_cvttss_si64 :: #force_inline proc "c"(a: __m128) -> i64 { + return cvttss2si64(a) + } + _mm_cvtsi64_ss :: #force_inline proc "c"(a: __m128, b: i64) -> __m128 { + return cvtsi642ss(a, b) + } +} + @(default_calling_convention="c") @(private) @@ -493,4 +505,13 @@ foreign _ { prefetch :: proc(p: rawptr, #const rw, loc, ty: u32) --- @(link_name="llvm.x86.sse.cmp.ss") cmpss :: proc(a, b: __m128, #const imm8: u8) -> __m128 --- + + + // amd64 only + @(link_name="llvm.x86.sse.cvtss2si64") + cvtss2si64 :: proc(a: __m128) -> i64 --- + @(link_name="llvm.x86.sse.cvttss2si64") + cvttss2si64 :: proc(a: __m128) -> i64 --- + @(link_name="llvm.x86.sse.cvtsi642ss") + cvtsi642ss :: proc(a: __m128, b: i64) -> __m128 --- } diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index a47d2a09d..8be1815fa 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -780,6 +780,43 @@ _mm_unpacklo_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { } +when ODIN_ARCH == .amd64 { + _mm_cvtsd_si64 :: #force_inline proc "c" (a: __m128d) -> i64 { + return cvtsd2si64(a) + } + _mm_cvtsd_si64x :: #force_inline proc "c" (a: __m128d) -> i64 { + return _mm_cvtsd_si64(a) + } + _mm_cvttsd_si64 :: #force_inline proc "c" (a: __m128d) -> i64 { + return cvttsd2si64(a) + } + _mm_cvttsd_si64x :: #force_inline proc "c" (a: __m128d) -> i64 { + return _mm_cvttsd_si64(a) + } + _mm_stream_si64 :: #force_inline proc "c" (mem_addr: ^i64, a: i64) { + intrinsics.non_temporal_store(mem_addr, a) + } + _mm_cvtsi64_si128 :: #force_inline proc "c" (a: i64) -> __m128i { + return _mm_set_epi64x(0, a) + } + _mm_cvtsi64x_si128 :: #force_inline proc "c" (a: i64) -> __m128i { + return _mm_cvtsi64_si128(a) + } + _mm_cvtsi128_si64 :: #force_inline proc "c" (a: __m128i) -> i64 { + return simd.extract(transmute(i64x2)a, 0) + } + _mm_cvtsi128_si64x :: #force_inline proc "c" (a: __m128i) -> i64 { + return _mm_cvtsi128_si64(a) + } + _mm_cvtsi64_sd :: #force_inline proc "c" (a: __m128d, b: i64) -> __m128d { + return simd.replace(a, 0, f64(b)) + } + _mm_cvtsi64x_sd :: #force_inline proc "c" (a: __m128d, b: i64) -> __m128d { + return _mm_cvtsi64_sd(a, b) + } +} + + @(default_calling_convention="c") @(private) foreign _ { @@ -923,4 +960,10 @@ foreign _ { storeudq :: proc(mem_addr: rawptr, a: __m128i) --- @(link_name="llvm.x86.sse2.storeu.pd") storeupd :: proc(mem_addr: rawptr, a: __m128d) --- + + // amd64 only + @(link_name="llvm.x86.sse2.cvtsd2si64") + cvtsd2si64 :: proc(a: __m128d) -> i64 --- + @(link_name="llvm.x86.sse2.cvttsd2si64") + cvttsd2si64 :: proc(a: __m128d) -> i64 --- } From 7f3540b7f541b75b1c4b4db558636d8295297c3f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 13:36:55 +0100 Subject: [PATCH 67/86] Add abm.odin --- core/simd/x86/abm.odin | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 core/simd/x86/abm.odin diff --git a/core/simd/x86/abm.odin b/core/simd/x86/abm.odin new file mode 100644 index 000000000..f1898811f --- /dev/null +++ b/core/simd/x86/abm.odin @@ -0,0 +1,20 @@ +//+build i386, amd64 +package simd_x86 + +import "core:intrinsics" + +_lzcnt_u32 :: #force_inline proc "c" (x: u32) -> u32 { + return intrinsics.count_leading_zeros(x) +} +_popcnt32 :: #force_inline proc "c" (x: u32) -> i32 { + return i32(intrinsics.count_ones(x)) +} + +when ODIN_ARCH == .amd64 { + _lzcnt_u64 :: #force_inline proc "c" (x: u64) -> u64 { + return intrinsics.count_leading_zeros(x) + } + _popcnt64 :: #force_inline proc "c" (x: u64) -> i32 { + return i32(intrinsics.count_ones(x)) + } +} \ No newline at end of file From 77d4409549d27b904676af5f33d8d034c6cae53e Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 13:40:16 +0100 Subject: [PATCH 68/86] Add adx.odin --- core/simd/x86/adx.odin | 53 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 core/simd/x86/adx.odin diff --git a/core/simd/x86/adx.odin b/core/simd/x86/adx.odin new file mode 100644 index 000000000..4de4324a2 --- /dev/null +++ b/core/simd/x86/adx.odin @@ -0,0 +1,53 @@ +//+build i386, amd64 +package simd_x86 + +import "core:intrinsics" + +_addcarry_u32 :: #force_inline proc "c" (c_in: u8, a: u32, b: u32, out: ^u32) -> u8 { + x, y := llvm_addcarry_u32(c_in, a, b) + out^ = y + return x +} +_addcarryx_u32 :: #force_inline proc "c" (c_in: u8, a: u32, b: u32, out: ^u32) -> u8 { + return llvm_addcarryx_u32(c_in, a, b, out) +} +_subborrow_u32 :: #force_inline proc "c" (c_in: u8, a: u32, b: u32, out: ^u32) -> u8 { + x, y := llvm_subborrow_u32(c_in, a, b) + out^ = y + return x +} + +when ODIN_ARCH == .amd64 { + _addcarry_u64 :: #force_inline proc "c" (c_in: u8, a: u64, b: u64, out: ^u64) -> u8 { + x, y := llvm_addcarry_u64(c_in, a, b) + out^ = y + return x + } + _addcarryx_u64 :: #force_inline proc "c" (c_in: u8, a: u64, b: u64, out: ^u64) -> u8 { + return llvm_addcarryx_u64(c_in, a, b, out) + } + _subborrow_u64 :: #force_inline proc "c" (c_in: u8, a: u64, b: u64, out: ^u64) -> u8 { + x, y := llvm_subborrow_u64(c_in, a, b) + out^ = y + return x + } +} + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name="llvm.x86.addcarry.32") + llvm_addcarry_u32 :: proc(a: u8, b: u32, c: u32) -> (u8, u32) --- + @(link_name="llvm.x86.addcarryx.u32") + llvm_addcarryx_u32 :: proc(a: u8, b: u32, c: u32, d: rawptr) -> u8 --- + @(link_name="llvm.x86.subborrow.32") + llvm_subborrow_u32 :: proc(a: u8, b: u32, c: u32) -> (u8, u32) --- + + // amd64 only + @(link_name="llvm.x86.addcarry.64") + llvm_addcarry_u64 :: proc(a: u8, b: u64, c: u64) -> (u8, u64) --- + @(link_name="llvm.x86.addcarryx.u64") + llvm_addcarryx_u64 :: proc(a: u8, b: u64, c: u64, d: rawptr) -> u8 --- + @(link_name="llvm.x86.subborrow.64") + llvm_subborrow_u64 :: proc(a: u8, b: u64, c: u64) -> (u8, u64) --- +} From 846f8377b2416a9f77618de00e50c7314a3b7322 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 13:44:00 +0100 Subject: [PATCH 69/86] Add fxsr.odin --- core/simd/x86/adx.odin | 2 -- core/simd/x86/fxsr.odin | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 core/simd/x86/fxsr.odin diff --git a/core/simd/x86/adx.odin b/core/simd/x86/adx.odin index 4de4324a2..13acf923d 100644 --- a/core/simd/x86/adx.odin +++ b/core/simd/x86/adx.odin @@ -1,8 +1,6 @@ //+build i386, amd64 package simd_x86 -import "core:intrinsics" - _addcarry_u32 :: #force_inline proc "c" (c_in: u8, a: u32, b: u32, out: ^u32) -> u8 { x, y := llvm_addcarry_u32(c_in, a, b) out^ = y diff --git a/core/simd/x86/fxsr.odin b/core/simd/x86/fxsr.odin new file mode 100644 index 000000000..24269391e --- /dev/null +++ b/core/simd/x86/fxsr.odin @@ -0,0 +1,33 @@ +//+build i386, amd64 +package simd_x86 + +_fxsave :: #force_inline proc "c" (mem_addr: rawptr) { + fxsave(mem_addr) +} +_fxrstor :: #force_inline proc "c" (mem_addr: rawptr) { + fxrstor(mem_addr) +} + +when ODIN_ARCH == .amd64 { + _fxsave64 :: #force_inline proc "c" (mem_addr: rawptr) { + fxsave64(mem_addr) + } + _fxrstor64 :: #force_inline proc "c" (mem_addr: rawptr) { + fxrstor64(mem_addr) + } +} + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name="llvm.x86.fxsave") + fxsave :: proc(p: rawptr) --- + @(link_name="llvm.x86.fxrstor") + fxrstor :: proc(p: rawptr) --- + + // amd64 only + @(link_name="llvm.x86.fxsave64") + fxsave64 :: proc(p: rawptr) --- + @(link_name="llvm.x86.fxrstor64") + fxrstor64 :: proc(p: rawptr) --- +} \ No newline at end of file From babfba5e8fd7be50e394d426be2db07c477e538d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 13:48:20 +0100 Subject: [PATCH 70/86] Add rdtsc.odin --- core/simd/x86/rdtsc.odin | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 core/simd/x86/rdtsc.odin diff --git a/core/simd/x86/rdtsc.odin b/core/simd/x86/rdtsc.odin new file mode 100644 index 000000000..0527fc084 --- /dev/null +++ b/core/simd/x86/rdtsc.odin @@ -0,0 +1,19 @@ +//+build i386, amd64 +package simd_x86 + +_rdtsc :: #force_inline proc "c" () -> u64 { + return rdtsc() +} + +__rdtscp :: #force_inline proc "c" (aux: ^u32) -> u64 { + return rdtscp(aux) +} + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name="llvm.x86.rdtsc") + rdtsc :: proc() -> u64 --- + @(link_name="llvm.x86.rdtscp") + rdtscp :: proc(aux: rawptr) -> u64 --- +} \ No newline at end of file From 0ccbea17aa874b50dd943d83ff39c1e1270f4ea2 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 13:50:54 +0100 Subject: [PATCH 71/86] Add pclmulqdq.odin --- core/simd/x86/pclmulqdq.odin | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 core/simd/x86/pclmulqdq.odin diff --git a/core/simd/x86/pclmulqdq.odin b/core/simd/x86/pclmulqdq.odin new file mode 100644 index 000000000..94496fb04 --- /dev/null +++ b/core/simd/x86/pclmulqdq.odin @@ -0,0 +1,13 @@ +//+build i386, amd64 +package simd_x86 + +_mm_clmulepi64_si128 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u8) -> __m128i { + return pclmulqdq(a, b, u8(IMM8)) +} + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name="llvm.x86.pclmulqdq") + pclmulqdq :: proc(a, round_key: __m128i, #const imm8: u8) -> __m128i --- +} \ No newline at end of file From f5e5eac3b9c8458eef17ffe062507fd0b531f0a0 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 14:46:05 +0100 Subject: [PATCH 72/86] Add cmpxchg16b --- core/simd/x86/cmpxchg16b.odin | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 core/simd/x86/cmpxchg16b.odin diff --git a/core/simd/x86/cmpxchg16b.odin b/core/simd/x86/cmpxchg16b.odin new file mode 100644 index 000000000..d575dd9df --- /dev/null +++ b/core/simd/x86/cmpxchg16b.odin @@ -0,0 +1,8 @@ +//+build amd64 +package simd_x86 + +import "core:intrinsics" + +cmpxchg16b :: #force_inline proc "c" (dst: ^u128, old, new: u128, $success, $failure: intrinsics.Atomic_Memory_Order) -> (val: u128) { + return intrinsics.atomic_compare_exchange_strong_explicit(dst, old, new, success, failure) +} \ No newline at end of file From bc3bf939e0760daeba75e29a82d2c1e8811651bb Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 14:56:25 +0100 Subject: [PATCH 73/86] Add sha.odin --- core/simd/x86/sha.odin | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 core/simd/x86/sha.odin diff --git a/core/simd/x86/sha.odin b/core/simd/x86/sha.odin new file mode 100644 index 000000000..c60293a8d --- /dev/null +++ b/core/simd/x86/sha.odin @@ -0,0 +1,43 @@ +//+build i386, amd64 +package simd_x86 + +_mm_sha1msg1_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)sha1msg1(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_sha1msg2_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)sha1msg2(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_sha1nexte_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)sha1nexte(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_sha1rnds4_epu32 :: #force_inline proc "c" (a, b: __m128i, $FUNC: u32) -> __m128i where 0 <= FUNC, FUNC <= 3 { + return transmute(__m128i)sha1rnds4(transmute(i32x4)a, transmute(i32x4)b, u8(FUNC & 0xff)) +} +_mm_sha256msg1_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)sha256msg1(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_sha256msg2_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)sha256msg2(transmute(i32x4)a, transmute(i32x4)b) +} +_mm_sha256rnds2_epu32 :: #force_inline proc "c" (a, b, k: __m128i) -> __m128i { + return transmute(__m128i)sha256rnds2(transmute(i32x4)a, transmute(i32x4)b, transmute(i32x4)k) +} + +@(default_calling_convention="c") +@(private) +foreign _ { + @(link_name="llvm.x86.sha1msg1") + sha1msg1 :: proc(a, b: i32x4) -> i32x4 --- + @(link_name="llvm.x86.sha1msg2") + sha1msg2 :: proc(a, b: i32x4) -> i32x4 --- + @(link_name="llvm.x86.sha1nexte") + sha1nexte :: proc(a, b: i32x4) -> i32x4 --- + @(link_name="llvm.x86.sha1rnds4") + sha1rnds4 :: proc(a, b: i32x4, #const c: u8) -> i32x4 --- + @(link_name="llvm.x86.sha256msg1") + sha256msg1 :: proc(a, b: i32x4) -> i32x4 --- + @(link_name="llvm.x86.sha256msg2") + sha256msg2 :: proc(a, b: i32x4) -> i32x4 --- + @(link_name="llvm.x86.sha256rnds2") + sha256rnds2 :: proc(a, b, k: i32x4) -> i32x4 --- +} \ No newline at end of file From f6dfa33697b0b5e3cb9b6b49214d8e02f26cb723 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 15:11:15 +0100 Subject: [PATCH 74/86] Use single line attributes --- core/simd/x86/adx.odin | 3 +-- core/simd/x86/fxsr.odin | 3 +-- core/simd/x86/pclmulqdq.odin | 3 +-- core/simd/x86/rdtsc.odin | 3 +-- core/simd/x86/sha.odin | 3 +-- core/simd/x86/sse.odin | 3 +-- core/simd/x86/sse2.odin | 3 +-- core/simd/x86/sse3.odin | 3 +-- core/simd/x86/ssse3.odin | 3 +-- 9 files changed, 9 insertions(+), 18 deletions(-) diff --git a/core/simd/x86/adx.odin b/core/simd/x86/adx.odin index 13acf923d..e73aa03a6 100644 --- a/core/simd/x86/adx.odin +++ b/core/simd/x86/adx.odin @@ -31,8 +31,7 @@ when ODIN_ARCH == .amd64 { } } -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name="llvm.x86.addcarry.32") llvm_addcarry_u32 :: proc(a: u8, b: u32, c: u32) -> (u8, u32) --- diff --git a/core/simd/x86/fxsr.odin b/core/simd/x86/fxsr.odin index 24269391e..847678d29 100644 --- a/core/simd/x86/fxsr.odin +++ b/core/simd/x86/fxsr.odin @@ -17,8 +17,7 @@ when ODIN_ARCH == .amd64 { } } -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name="llvm.x86.fxsave") fxsave :: proc(p: rawptr) --- diff --git a/core/simd/x86/pclmulqdq.odin b/core/simd/x86/pclmulqdq.odin index 94496fb04..ba4ecf35f 100644 --- a/core/simd/x86/pclmulqdq.odin +++ b/core/simd/x86/pclmulqdq.odin @@ -5,8 +5,7 @@ _mm_clmulepi64_si128 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u8) -> __m return pclmulqdq(a, b, u8(IMM8)) } -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name="llvm.x86.pclmulqdq") pclmulqdq :: proc(a, round_key: __m128i, #const imm8: u8) -> __m128i --- diff --git a/core/simd/x86/rdtsc.odin b/core/simd/x86/rdtsc.odin index 0527fc084..91dcc4ec9 100644 --- a/core/simd/x86/rdtsc.odin +++ b/core/simd/x86/rdtsc.odin @@ -9,8 +9,7 @@ __rdtscp :: #force_inline proc "c" (aux: ^u32) -> u64 { return rdtscp(aux) } -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name="llvm.x86.rdtsc") rdtsc :: proc() -> u64 --- diff --git a/core/simd/x86/sha.odin b/core/simd/x86/sha.odin index c60293a8d..d907ce6a6 100644 --- a/core/simd/x86/sha.odin +++ b/core/simd/x86/sha.odin @@ -23,8 +23,7 @@ _mm_sha256rnds2_epu32 :: #force_inline proc "c" (a, b, k: __m128i) -> __m128i { return transmute(__m128i)sha256rnds2(transmute(i32x4)a, transmute(i32x4)b, transmute(i32x4)k) } -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name="llvm.x86.sha1msg1") sha1msg1 :: proc(a, b: i32x4) -> i32x4 --- diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin index eb1950ea3..a564f243a 100644 --- a/core/simd/x86/sse.odin +++ b/core/simd/x86/sse.odin @@ -430,8 +430,7 @@ when ODIN_ARCH == .amd64 { } -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name="llvm.x86.sse.add.ss") addss :: proc(a, b: __m128) -> __m128 --- diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index 8be1815fa..cb2e61f46 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -817,8 +817,7 @@ when ODIN_ARCH == .amd64 { } -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name="llvm.x86.sse2.pause") pause :: proc() --- diff --git a/core/simd/x86/sse3.odin b/core/simd/x86/sse3.odin index 6468ea268..9766a43e6 100644 --- a/core/simd/x86/sse3.odin +++ b/core/simd/x86/sse3.odin @@ -38,8 +38,7 @@ _mm_moveldup_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return simd.shuffle(a, a, 0, 0, 2, 2) } -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name = "llvm.x86.sse3.addsub.ps") addsubps :: proc(a, b: __m128) -> __m128 --- diff --git a/core/simd/x86/ssse3.odin b/core/simd/x86/ssse3.odin index 4abd4c84c..6c6f28008 100644 --- a/core/simd/x86/ssse3.odin +++ b/core/simd/x86/ssse3.odin @@ -89,8 +89,7 @@ _mm_sign_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { -@(default_calling_convention="c") -@(private) +@(private, default_calling_convention="c") foreign _ { @(link_name = "llvm.x86.ssse3.pabs.b.128") pabsb128 :: proc(a: i8x16) -> u8x16 --- From cef022539ebd41a4a80707f1a702e09e6748ade0 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Sun, 29 May 2022 15:13:14 +0100 Subject: [PATCH 75/86] Rename to `lanes_rotate_left`, `lanes_rotate_right`, `lanes_reverse` --- core/simd/simd.odin | 6 +++--- src/check_builtin.cpp | 6 +++--- src/checker_builtin_procs.hpp | 12 ++++++------ src/llvm_backend_proc.cpp | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index ed7e418f3..390ff377a 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -134,10 +134,10 @@ nearest :: intrinsics.simd_nearest to_bits :: intrinsics.simd_to_bits -lanes_reverse :: intrinsics.simd_reverse +lanes_reverse :: intrinsics.simd_lanes_reverse -lanes_rotate_left :: intrinsics.simd_rotate_left -lanes_rotate_right :: intrinsics.simd_rotate_right +lanes_rotate_left :: intrinsics.simd_lanes_rotate_left +lanes_rotate_right :: intrinsics.simd_lanes_rotate_right count_ones :: intrinsics.count_ones count_zeros :: intrinsics.count_zeros diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index f8ac545be..92e3987a0 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -919,7 +919,7 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } - case BuiltinProc_simd_reverse: + case BuiltinProc_simd_lanes_reverse: { Operand x = {}; check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; @@ -933,8 +933,8 @@ bool check_builtin_simd_operation(CheckerContext *c, Operand *operand, Ast *call return true; } - case BuiltinProc_simd_rotate_left: - case BuiltinProc_simd_rotate_right: + case BuiltinProc_simd_lanes_rotate_left: + case BuiltinProc_simd_lanes_rotate_right: { Operand x = {}; check_expr(c, &x, ce->args[0]); if (x.mode == Addressing_Invalid) return false; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 35f14c6a8..2e27cc026 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -175,9 +175,9 @@ BuiltinProc__simd_begin, BuiltinProc_simd_to_bits, - BuiltinProc_simd_reverse, - BuiltinProc_simd_rotate_left, - BuiltinProc_simd_rotate_right, + BuiltinProc_simd_lanes_reverse, + BuiltinProc_simd_lanes_rotate_left, + BuiltinProc_simd_lanes_rotate_right, // Platform specific SIMD intrinsics @@ -468,9 +468,9 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("simd_to_bits"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_reverse"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_rotate_left"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("simd_rotate_right"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_reverse"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_rotate_left"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("simd_lanes_rotate_right"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("simd_x86__MM_SHUFFLE"), 4, false, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 8cbb533bc..1e3591bf1 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1330,7 +1330,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } - case BuiltinProc_simd_reverse: + case BuiltinProc_simd_lanes_reverse: { i64 count = get_array_type_count(arg0.type); LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, count); @@ -1345,8 +1345,8 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const return res; } - case BuiltinProc_simd_rotate_left: - case BuiltinProc_simd_rotate_right: + case BuiltinProc_simd_lanes_rotate_left: + case BuiltinProc_simd_lanes_rotate_right: { i64 count = get_array_type_count(arg0.type); @@ -1358,7 +1358,7 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const ExactValue val = exact_value_to_integer(tv.value); GB_ASSERT(val.kind == ExactValue_Integer); BigInt *bi = &val.value_integer; - if (builtin_id == BuiltinProc_simd_rotate_right) { + if (builtin_id == BuiltinProc_simd_lanes_rotate_right) { big_int_neg(bi, bi); } big_int_rem(bi, bi, &bi_count); From f3aefbc4434b92fc3fda74c942c953b08dd18a62 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 14:53:12 +0100 Subject: [PATCH 76/86] `@(require_target_feature=)` `@(enable_target_feature=)` require_target_feature - required by the target micro-architecture enable_target_feature - will be enabled for the specified procedure only --- core/simd/x86/fxsr.odin | 4 + core/simd/x86/pclmulqdq.odin | 1 + core/simd/x86/sha.odin | 7 ++ core/simd/x86/sse.odin | 102 ++++++++++++++++ core/simd/x86/sse2.odin | 223 +++++++++++++++++++++++++++++++++++ core/simd/x86/sse3.odin | 11 ++ core/simd/x86/ssse3.odin | 16 +++ src/build_settings.cpp | 109 ++++++++++++++++- src/check_decl.cpp | 12 ++ src/checker.cpp | 16 +++ src/checker.hpp | 3 + src/entity.cpp | 8 +- src/llvm_backend.cpp | 4 +- src/llvm_backend_proc.cpp | 13 ++ src/main.cpp | 4 +- src/string.cpp | 9 ++ 16 files changed, 533 insertions(+), 9 deletions(-) diff --git a/core/simd/x86/fxsr.odin b/core/simd/x86/fxsr.odin index 847678d29..cd78de7d4 100644 --- a/core/simd/x86/fxsr.odin +++ b/core/simd/x86/fxsr.odin @@ -1,17 +1,21 @@ //+build i386, amd64 package simd_x86 +@(enable_target_feature="fxsr") _fxsave :: #force_inline proc "c" (mem_addr: rawptr) { fxsave(mem_addr) } +@(enable_target_feature="fxsr") _fxrstor :: #force_inline proc "c" (mem_addr: rawptr) { fxrstor(mem_addr) } when ODIN_ARCH == .amd64 { + @(enable_target_feature="fxsr") _fxsave64 :: #force_inline proc "c" (mem_addr: rawptr) { fxsave64(mem_addr) } + @(enable_target_feature="fxsr") _fxrstor64 :: #force_inline proc "c" (mem_addr: rawptr) { fxrstor64(mem_addr) } diff --git a/core/simd/x86/pclmulqdq.odin b/core/simd/x86/pclmulqdq.odin index ba4ecf35f..8a665db03 100644 --- a/core/simd/x86/pclmulqdq.odin +++ b/core/simd/x86/pclmulqdq.odin @@ -1,6 +1,7 @@ //+build i386, amd64 package simd_x86 +@(enable_target_feature="pclmulqdq") _mm_clmulepi64_si128 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u8) -> __m128i { return pclmulqdq(a, b, u8(IMM8)) } diff --git a/core/simd/x86/sha.odin b/core/simd/x86/sha.odin index d907ce6a6..90f1d72ce 100644 --- a/core/simd/x86/sha.odin +++ b/core/simd/x86/sha.odin @@ -1,24 +1,31 @@ //+build i386, amd64 package simd_x86 +@(enable_target_feature="sha") _mm_sha1msg1_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha1msg1(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sha") _mm_sha1msg2_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha1msg2(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sha") _mm_sha1nexte_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha1nexte(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sha") _mm_sha1rnds4_epu32 :: #force_inline proc "c" (a, b: __m128i, $FUNC: u32) -> __m128i where 0 <= FUNC, FUNC <= 3 { return transmute(__m128i)sha1rnds4(transmute(i32x4)a, transmute(i32x4)b, u8(FUNC & 0xff)) } +@(enable_target_feature="sha") _mm_sha256msg1_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha256msg1(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sha") _mm_sha256msg2_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha256msg2(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sha") _mm_sha256rnds2_epu32 :: #force_inline proc "c" (a, b, k: __m128i) -> __m128i { return transmute(__m128i)sha256rnds2(transmute(i32x4)a, transmute(i32x4)b, transmute(i32x4)k) } diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin index a564f243a..6d8939b1b 100644 --- a/core/simd/x86/sse.odin +++ b/core/simd/x86/sse.odin @@ -43,232 +43,299 @@ _MM_FLUSH_ZERO_ON :: 0x8000 _MM_FLUSH_ZERO_OFF :: 0x0000 +@(enable_target_feature="sse") _mm_add_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return addss(a, b) } +@(enable_target_feature="sse") _mm_add_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.add(a, b) } +@(enable_target_feature="sse") _mm_sub_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return subss(a, b) } +@(enable_target_feature="sse") _mm_sub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.sub(a, b) } +@(enable_target_feature="sse") _mm_mul_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return mulss(a, b) } +@(enable_target_feature="sse") _mm_mul_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.mul(a, b) } +@(enable_target_feature="sse") _mm_div_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return divss(a, b) } +@(enable_target_feature="sse") _mm_div_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.div(a, b) } +@(enable_target_feature="sse") _mm_sqrt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return sqrtss(a) } +@(enable_target_feature="sse") _mm_sqrt_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return sqrtps(a) } +@(enable_target_feature="sse") _mm_rcp_ss :: #force_inline proc "c" (a: __m128) -> __m128 { return rcpss(a) } +@(enable_target_feature="sse") _mm_rcp_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return rcpps(a) } +@(enable_target_feature="sse") _mm_rsqrt_ss :: #force_inline proc "c" (a: __m128) -> __m128 { return rsqrtss(a) } +@(enable_target_feature="sse") _mm_rsqrt_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return rsqrtps(a) } +@(enable_target_feature="sse") _mm_min_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return minss(a, b) } +@(enable_target_feature="sse") _mm_min_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return minps(a, b) } +@(enable_target_feature="sse") _mm_max_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return maxss(a, b) } +@(enable_target_feature="sse") _mm_max_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return maxps(a, b) } +@(enable_target_feature="sse") _mm_and_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return transmute(__m128)simd.and(transmute(__m128i)a, transmute(__m128i)b) } +@(enable_target_feature="sse") _mm_andnot_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return transmute(__m128)simd.and_not(transmute(__m128i)a, transmute(__m128i)b) } +@(enable_target_feature="sse") _mm_or_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return transmute(__m128)simd.or(transmute(__m128i)a, transmute(__m128i)b) } +@(enable_target_feature="sse") _mm_xor_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return transmute(__m128)simd.xor(transmute(__m128i)a, transmute(__m128i)b) } +@(enable_target_feature="sse") _mm_cmpeq_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 0) } +@(enable_target_feature="sse") _mm_cmplt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 1) } +@(enable_target_feature="sse") _mm_cmple_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 2) } +@(enable_target_feature="sse") _mm_cmpgt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, cmpss(b, a, 1), 4, 1, 2, 3) } +@(enable_target_feature="sse") _mm_cmpge_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, cmpss(b, a, 2), 4, 1, 2, 3) } +@(enable_target_feature="sse") _mm_cmpneq_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 4) } +@(enable_target_feature="sse") _mm_cmpnlt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 5) } +@(enable_target_feature="sse") _mm_cmpnle_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 6) } +@(enable_target_feature="sse") _mm_cmpngt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, cmpss(b, a, 5), 4, 1, 2, 3) } +@(enable_target_feature="sse") _mm_cmpnge_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, cmpss(b, a, 6), 4, 1, 2, 3) } +@(enable_target_feature="sse") _mm_cmpord_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 7) } +@(enable_target_feature="sse") _mm_cmpunord_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 3) } +@(enable_target_feature="sse") _mm_cmpeq_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 0) } +@(enable_target_feature="sse") _mm_cmplt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 1) } +@(enable_target_feature="sse") _mm_cmple_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 2) } +@(enable_target_feature="sse") _mm_cmpgt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 1) } +@(enable_target_feature="sse") _mm_cmpge_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 2) } +@(enable_target_feature="sse") _mm_cmpneq_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 4) } +@(enable_target_feature="sse") _mm_cmpnlt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 5) } +@(enable_target_feature="sse") _mm_cmpnle_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 6) } +@(enable_target_feature="sse") _mm_cmpngt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 5) } +@(enable_target_feature="sse") _mm_cmpnge_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 6) } +@(enable_target_feature="sse") _mm_cmpord_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 7) } +@(enable_target_feature="sse") _mm_cmpunord_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 3) } +@(enable_target_feature="sse") _mm_comieq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comieq_ss(a, b) } +@(enable_target_feature="sse") _mm_comilt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comilt_ss(a, b) } +@(enable_target_feature="sse") _mm_comile_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comile_ss(a, b) } +@(enable_target_feature="sse") _mm_comigt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comigt_ss(a, b) } +@(enable_target_feature="sse") _mm_comige_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comige_ss(a, b) } +@(enable_target_feature="sse") _mm_comineq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comineq_ss(a, b) } +@(enable_target_feature="sse") _mm_ucomieq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomieq_ss(a, b) } +@(enable_target_feature="sse") _mm_ucomilt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomilt_ss(a, b) } +@(enable_target_feature="sse") _mm_ucomile_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomile_ss(a, b) } +@(enable_target_feature="sse") _mm_ucomigt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomigt_ss(a, b) } +@(enable_target_feature="sse") _mm_ucomige_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomige_ss(a, b) } +@(enable_target_feature="sse") _mm_ucomineq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomineq_ss(a, b) } +@(enable_target_feature="sse") _mm_cvtss_si32 :: #force_inline proc "c" (a: __m128) -> i32 { return cvtss2si(a) } _mm_cvt_ss2si :: _mm_cvtss_si32 _mm_cvttss_si32 :: _mm_cvtss_si32 +@(enable_target_feature="sse") _mm_cvtss_f32 :: #force_inline proc "c" (a: __m128) -> f32 { return simd.extract(a, 0) } +@(enable_target_feature="sse") _mm_cvtsi32_ss :: #force_inline proc "c" (a: __m128, b: i32) -> __m128 { return cvtsi2ss(a, b) } _mm_cvt_si2ss :: _mm_cvtsi32_ss +@(enable_target_feature="sse") _mm_set_ss :: #force_inline proc "c" (a: f32) -> __m128 { return __m128{a, 0, 0, 0} } +@(enable_target_feature="sse") _mm_set1_ps :: #force_inline proc "c" (a: f32) -> __m128 { return __m128(a) } _mm_set_ps1 :: _mm_set1_ps +@(enable_target_feature="sse") _mm_set_ps :: #force_inline proc "c" (a, b, c, d: f32) -> __m128 { return __m128{d, c, b, a} } +@(enable_target_feature="sse") _mm_setr_ps :: #force_inline proc "c" (a, b, c, d: f32) -> __m128 { return __m128{a, b, c, d} } +@(enable_target_feature="sse") _mm_setzero_ps :: #force_inline proc "c" () -> __m128 { return __m128{0, 0, 0, 0} } +@(enable_target_feature="sse") _mm_shuffle_ps :: #force_inline proc "c" (a, b: __m128, $MASK: u32) -> __m128 { return simd.shuffle( a, b, @@ -279,56 +346,69 @@ _mm_shuffle_ps :: #force_inline proc "c" (a, b: __m128, $MASK: u32) -> __m128 { } +@(enable_target_feature="sse") _mm_unpackhi_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 2, 6, 3, 7) } +@(enable_target_feature="sse") _mm_unpacklo_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 0, 4, 1, 5) } +@(enable_target_feature="sse") _mm_movehl_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 6, 7, 2, 3) } +@(enable_target_feature="sse") _mm_movelh_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 0, 1, 4, 5) } +@(enable_target_feature="sse") _mm_movemask_ps :: #force_inline proc "c" (a: __m128) -> u32 { return movmskps(a) } +@(enable_target_feature="sse") _mm_load_ss :: #force_inline proc "c" (p: ^f32) -> __m128 { return __m128{p^, 0, 0, 0} } +@(enable_target_feature="sse") _mm_load1_ps :: #force_inline proc "c" (p: ^f32) -> __m128 { a := p^ return __m128(a) } _mm_load_ps1 :: _mm_load1_ps +@(enable_target_feature="sse") _mm_load_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { return (^__m128)(p)^ } +@(enable_target_feature="sse") _mm_loadu_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { dst := _mm_undefined_ps() intrinsics.mem_copy_non_overlapping(&dst, p, size_of(__m128)) return dst } +@(enable_target_feature="sse") _mm_loadr_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { return simd.lanes_reverse(_mm_load_ps(p)) } +@(enable_target_feature="sse") _mm_loadu_si64 :: #force_inline proc "c" (mem_addr: rawptr) -> __m128i { a := intrinsics.unaligned_load((^i64)(mem_addr)) return __m128i{a, 0} } +@(enable_target_feature="sse") _mm_store_ss :: #force_inline proc "c" (p: ^f32, a: __m128) { p^ = simd.extract(a, 0) } +@(enable_target_feature="sse") _mm_store1_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { b := simd.swizzle(a, 0, 0, 0, 0) (^__m128)(p)^ = b @@ -336,71 +416,89 @@ _mm_store1_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { _mm_store_ps1 :: _mm_store1_ps +@(enable_target_feature="sse") _mm_store_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { (^__m128)(p)^ = a } +@(enable_target_feature="sse") _mm_storeu_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { b := a intrinsics.mem_copy_non_overlapping(p, &b, size_of(__m128)) } +@(enable_target_feature="sse") _mm_storer_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { (^__m128)(p)^ = simd.lanes_reverse(a) } +@(enable_target_feature="sse") _mm_move_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 4, 1, 2, 3) } +@(enable_target_feature="sse") _mm_sfence :: #force_inline proc "c" () { sfence() } +@(enable_target_feature="sse") _mm_getcsr :: #force_inline proc "c" () -> (result: u32) { stmxcsr(&result) return result } +@(enable_target_feature="sse") _mm_setcsr :: #force_inline proc "c" (val: u32) { val := val ldmxcsr(&val) } +@(enable_target_feature="sse") _MM_GET_EXCEPTION_MASK :: #force_inline proc "c" () -> u32 { return _mm_getcsr() & _MM_MASK_MASK } +@(enable_target_feature="sse") _MM_GET_EXCEPTION_STATE :: #force_inline proc "c" () -> u32 { return _mm_getcsr() & _MM_EXCEPT_MASK } +@(enable_target_feature="sse") _MM_GET_FLUSH_ZERO_MODE :: #force_inline proc "c" () -> u32 { return _mm_getcsr() & _MM_FLUSH_ZERO_MASK } +@(enable_target_feature="sse") _MM_GET_ROUNDING_MODE :: #force_inline proc "c" () -> u32 { return _mm_getcsr() & _MM_ROUND_MASK } +@(enable_target_feature="sse") _MM_SET_EXCEPTION_MASK :: #force_inline proc "c" (x: u32) { _mm_setcsr((_mm_getcsr() &~ _MM_MASK_MASK) | x) } +@(enable_target_feature="sse") _MM_SET_EXCEPTION_STATE :: #force_inline proc "c" (x: u32) { _mm_setcsr((_mm_getcsr() &~ _MM_EXCEPT_MASK) | x) } +@(enable_target_feature="sse") _MM_SET_FLUSH_ZERO_MODE :: #force_inline proc "c" (x: u32) { _mm_setcsr((_mm_getcsr() &~ _MM_FLUSH_ZERO_MASK) | x) } +@(enable_target_feature="sse") _MM_SET_ROUNDING_MODE :: #force_inline proc "c" (x: u32) { _mm_setcsr((_mm_getcsr() &~ _MM_ROUND_MASK) | x) } +@(enable_target_feature="sse") _mm_prefetch :: #force_inline proc "c" (p: rawptr, $STRATEGY: u32) { prefetch(p, (STRATEGY>>2)&1, STRATEGY&3, 1) } +@(enable_target_feature="sse") _mm_undefined_ps :: #force_inline proc "c" () -> __m128 { return _mm_set1_ps(0) } +@(enable_target_feature="sse") _MM_TRANSPOSE4_PS :: #force_inline proc "c" (row0, row1, row2, row3: ^__m128) { tmp0 := _mm_unpacklo_ps(row0^, row1^) tmp1 := _mm_unpacklo_ps(row2^, row3^) @@ -413,17 +511,21 @@ _MM_TRANSPOSE4_PS :: #force_inline proc "c" (row0, row1, row2, row3: ^__m128) { row3^ = _mm_movelh_ps(tmp3, tmp1) } +@(enable_target_feature="sse") _mm_stream_ps :: #force_inline proc "c" (addr: [^]f32, a: __m128) { intrinsics.non_temporal_store((^__m128)(addr), a) } when ODIN_ARCH == .amd64 { + @(enable_target_feature="sse") _mm_cvtss_si64 :: #force_inline proc "c"(a: __m128) -> i64 { return cvtss2si64(a) } + @(enable_target_feature="sse") _mm_cvttss_si64 :: #force_inline proc "c"(a: __m128) -> i64 { return cvttss2si64(a) } + @(enable_target_feature="sse") _mm_cvtsi64_ss :: #force_inline proc "c"(a: __m128, b: i64) -> __m128 { return cvtsi642ss(a, b) } diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index cb2e61f46..d15df8120 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -4,103 +4,135 @@ package simd_x86 import "core:intrinsics" import "core:simd" +@(enable_target_feature="sse2") _mm_pause :: #force_inline proc "c" () { pause() } +@(enable_target_feature="sse2") _mm_clflush :: #force_inline proc "c" (p: rawptr) { clflush(p) } +@(enable_target_feature="sse2") _mm_lfence :: #force_inline proc "c" () { lfence() } +@(enable_target_feature="sse2") _mm_mfence :: #force_inline proc "c" () { mfence() } +@(enable_target_feature="sse2") _mm_add_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add(transmute(i8x16)a, transmute(i8x16)b) } +@(enable_target_feature="sse2") _mm_add_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_add_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sse2") _mm_add_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add(transmute(i64x2)a, transmute(i64x2)b) } +@(enable_target_feature="sse2") _mm_adds_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add_sat(transmute(i8x16)a, transmute(i8x16)b) } +@(enable_target_feature="sse2") _mm_adds_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add_sat(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_adds_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add_sat(transmute(u8x16)a, transmute(u8x16)b) } +@(enable_target_feature="sse2") _mm_adds_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add_sat(transmute(u16x8)a, transmute(u16x8)b) } +@(enable_target_feature="sse2") _mm_avg_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pavgb(transmute(u8x16)a, transmute(u8x16)b) } +@(enable_target_feature="sse2") _mm_avg_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pavgw(transmute(u16x8)a, transmute(u16x8)b) } +@(enable_target_feature="sse2") _mm_madd_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaddwd(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_max_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaxsw(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_max_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaxub(transmute(u8x16)a, transmute(u8x16)b) } +@(enable_target_feature="sse2") _mm_min_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pminsw(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_min_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pminub(transmute(u8x16)a, transmute(u8x16)b) } +@(enable_target_feature="sse2") _mm_mulhi_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmulhw(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_mulhi_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmulhuw(transmute(u16x8)a, transmute(u16x8)b) } +@(enable_target_feature="sse2") _mm_mullo_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.mul(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_mul_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmuludq(transmute(u32x4)a, transmute(u32x4)b) } +@(enable_target_feature="sse2") _mm_sad_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psadbw(transmute(u8x16)a, transmute(u8x16)b) } +@(enable_target_feature="sse2") _mm_sub_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i8x16)a, transmute(i8x16)b) } +@(enable_target_feature="sse2") _mm_sub_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_sub_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sse2") _mm_sub_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i64x2)a, transmute(i64x2)b) } +@(enable_target_feature="sse2") _mm_subs_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(i8x16)a, transmute(i8x16)b) } +@(enable_target_feature="sse2") _mm_subs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_subs_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(u8x16)a, transmute(u8x16)b) } +@(enable_target_feature="sse2") _mm_subs_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(u16x8)a, transmute(u16x8)b) } @@ -108,6 +140,7 @@ _mm_subs_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { @(private) +@(enable_target_feature="sse2") _mm_slli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { shift :: IMM8 & 0xff @@ -134,6 +167,7 @@ _mm_slli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128 } @(private) +@(enable_target_feature="sse2") _mm_srli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { shift :: IMM8 return transmute(__m128i)simd.shuffle( @@ -159,203 +193,264 @@ _mm_srli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128 } +@(enable_target_feature="sse2") _mm_slli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_slli_si128_impl(a, IMM8) } +@(enable_target_feature="sse2") _mm_bslli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_slli_si128_impl(a, IMM8) } +@(enable_target_feature="sse2") _mm_bsrli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_srli_si128_impl(a, IMM8) } +@(enable_target_feature="sse2") _mm_slli_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)pslliw(transmute(i16x8)a, IMM8) } +@(enable_target_feature="sse2") _mm_sll_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psllw(transmute(i16x8)a, transmute(i16x8)count) } +@(enable_target_feature="sse2") _mm_slli_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psllid(transmute(i32x4)a, IMM8) } +@(enable_target_feature="sse2") _mm_sll_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)pslld(transmute(i32x4)a, transmute(i32x4)count) } +@(enable_target_feature="sse2") _mm_slli_epi64 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)pslliq(transmute(i64x2)a, IMM8) } +@(enable_target_feature="sse2") _mm_sll_epi64 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psllq(transmute(i64x2)a, transmute(i64x2)count) } +@(enable_target_feature="sse2") _mm_srai_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psraiw(transmute(i16x8)a. IMM8) } +@(enable_target_feature="sse2") _mm_sra_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psraw(transmute(i16x8)a, transmute(i16x8)count) } +@(enable_target_feature="sse2") _mm_srai_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psraid(transmute(i32x4)a, IMM8) } +@(enable_target_feature="sse2") _mm_sra_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrad(transmute(i32x4)a, transmute(i32x4)count) } +@(enable_target_feature="sse2") _mm_srli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_srli_si128_impl(a, IMM8) } +@(enable_target_feature="sse2") _mm_srli_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrliw(transmute(i16x8)a. IMM8) } +@(enable_target_feature="sse2") _mm_srl_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrlw(transmute(i16x8)a, transmute(i16x8)count) } +@(enable_target_feature="sse2") _mm_srli_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrlid(transmute(i32x4)a, IMM8) } +@(enable_target_feature="sse2") _mm_srl_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrld(transmute(i32x4)a, transmute(i32x4)count) } +@(enable_target_feature="sse2") _mm_srli_epi64 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrliq(transmute(i64x2)a, IMM8) } +@(enable_target_feature="sse2") _mm_srl_epi64 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrlq(transmute(i64x2)a, transmute(i64x2)count) } +@(enable_target_feature="sse2") _mm_and_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return simd.and(a, b) } +@(enable_target_feature="sse2") _mm_andnot_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return simd.and_not(b, a) } +@(enable_target_feature="sse2") _mm_or_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return simd.or(a, b) } +@(enable_target_feature="sse2") _mm_xor_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return simd.xor(a, b) } +@(enable_target_feature="sse2") _mm_cmpeq_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_eq(transmute(i8x16)a, transmute(i8x16)b) } +@(enable_target_feature="sse2") _mm_cmpeq_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_eq(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_cmpeq_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_eq(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sse2") _mm_cmpgt_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_gt(transmute(i8x16)a, transmute(i8x16)b) } +@(enable_target_feature="sse2") _mm_cmpgt_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_gt(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_cmpgt_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_gt(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sse2") _mm_cmplt_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_lt(transmute(i8x16)a, transmute(i8x16)b) } +@(enable_target_feature="sse2") _mm_cmplt_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_lt(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_cmplt_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_lt(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sse2") _mm_cvtepi32_pd :: #force_inline proc "c" (a: __m128i) -> __m128d { v := transmute(i32x4)a return cast(__m128d)simd.shuffle(v, v, 0, 1) } +@(enable_target_feature="sse2") _mm_cvtsi32_sd :: #force_inline proc "c" (a: __m128d, b: i32) -> __m128d { return simd.replace(a, 0, f64(b)) } +@(enable_target_feature="sse2") _mm_cvtepi32_ps :: #force_inline proc "c" (a: __m128i) -> __m128 { return cvtdq2ps(transmute(i32x4)a) } +@(enable_target_feature="sse2") _mm_cvtps_epi32 :: #force_inline proc "c" (a: __m128) -> __m128i { return transmute(__m128i)cvtps2dq(a) } +@(enable_target_feature="sse2") _mm_cvtsi32_si128 :: #force_inline proc "c" (a: i32) -> __m128i { return transmute(__m128i)i32x4{a, 0, 0, 0} } +@(enable_target_feature="sse2") _mm_cvtsi128_si32 :: #force_inline proc "c" (a: __m128i) -> i32 { return simd.extract(transmute(i32x4)a, 0) } +@(enable_target_feature="sse2") _mm_set_epi64x :: #force_inline proc "c" (e1, e0: i64) -> __m128i { return transmute(__m128i)i64x2{e0, e1} } +@(enable_target_feature="sse2") _mm_set_epi32 :: #force_inline proc "c" (e3, e2, e1, e0: i32) -> __m128i { return transmute(__m128i)i32x4{e0, e1, e2, e3} } +@(enable_target_feature="sse2") _mm_set_epi16 :: #force_inline proc "c" (e7, e6, e5, e4, e3, e2, e1, e0: i16) -> __m128i { return transmute(__m128i)i16x8{e0, e1, e2, e3, e4, e5, e6, e7} } +@(enable_target_feature="sse2") _mm_set_epi8 :: #force_inline proc "c" (e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0: i8) -> __m128i { return transmute(__m128i)i8x16{e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15} } +@(enable_target_feature="sse2") _mm_set1_epi64x :: #force_inline proc "c" (a: i64) -> __m128i { return _mm_set_epi64x(a, a) } +@(enable_target_feature="sse2") _mm_set1_epi32 :: #force_inline proc "c" (a: i32) -> __m128i { return _mm_set_epi32(a, a, a, a) } +@(enable_target_feature="sse2") _mm_set1_epi16 :: #force_inline proc "c" (a: i16) -> __m128i { return _mm_set_epi16(a, a, a, a, a, a, a, a) } +@(enable_target_feature="sse2") _mm_set1_epi8 :: #force_inline proc "c" (a: i8) -> __m128i { return _mm_set_epi8(a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a) } +@(enable_target_feature="sse2") _mm_setr_epi32 :: #force_inline proc "c" (e3, e2, e1, e0: i32) -> __m128i { return _mm_set_epi32(e0, e1, e2, e3) } +@(enable_target_feature="sse2") _mm_setr_epi16 :: #force_inline proc "c" (e7, e6, e5, e4, e3, e2, e1, e0: i16) -> __m128i { return _mm_set_epi16(e0, e1, e2, e3, e4, e5, e6, e7) } +@(enable_target_feature="sse2") _mm_setr_epi8 :: #force_inline proc "c" (e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0: i8) -> __m128i { return _mm_set_epi8(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15) } +@(enable_target_feature="sse2") _mm_setzero_si128 :: #force_inline proc "c" () -> __m128i { return _mm_set1_epi64x(0) } +@(enable_target_feature="sse2") _mm_loadl_epi64 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { return _mm_set_epi64x(0, intrinsics.unaligned_load((^i64)(mem_addr))) } +@(enable_target_feature="sse2") _mm_load_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { return mem_addr^ } +@(enable_target_feature="sse2") _mm_loadu_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { dst := _mm_undefined_si128() intrinsics.mem_copy_non_overlapping(&dst, mem_addr, size_of(__m128i)) return dst } +@(enable_target_feature="sse2") _mm_maskmoveu_si128 :: #force_inline proc "c" (a, mask: __m128i, mem_addr: rawptr) { maskmovdqu(transmute(i8x16)a, transmute(i8x16)mask, mem_addr) } +@(enable_target_feature="sse2") _mm_store_si128 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { mem_addr^ = a } +@(enable_target_feature="sse2") _mm_storeu_si128 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { storeudq(mem_addr, a) } +@(enable_target_feature="sse2") _mm_storel_epi64 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { a := a intrinsics.mem_copy_non_overlapping(mem_addr, &a, 8) } +@(enable_target_feature="sse2") _mm_stream_si128 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { intrinsics.non_temporal_store(mem_addr, a) } +@(enable_target_feature="sse2") _mm_stream_si32 :: #force_inline proc "c" (mem_addr: ^i32, a: i32) { intrinsics.non_temporal_store(mem_addr, a) } +@(enable_target_feature="sse2") _mm_move_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { zero := _mm_setzero_si128() return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)zero, 0, 2) @@ -364,24 +459,31 @@ _mm_move_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { +@(enable_target_feature="sse2") _mm_packs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)packsswb(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_packs_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)packssdw(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="sse2") _mm_packus_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)packuswb(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="sse2") _mm_extract_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> i32 { return i32(simd.extract(transmute(u16x8)a, IMM8)) } +@(enable_target_feature="sse2") _mm_insert_epi16 :: #force_inline proc "c" (a: __m128i, i: i32, $IMM8: u32) -> __m128i { return i32(simd.replace(transmute(u16x8)a, IMM8, i16(i))) } +@(enable_target_feature="sse2") _mm_movemask_epi8 :: #force_inline proc "c" (a: __m128i) -> i32 { return pmovmskb(transmute(i8x16)a) } +@(enable_target_feature="sse2") _mm_shuffle_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { v := transmute(i32x4)a return transmute(__m128i)simd.shuffle( @@ -393,6 +495,7 @@ _mm_shuffle_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i (IMM8 >> 6) & 0b11, ) } +@(enable_target_feature="sse2") _mm_shufflehi_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { v := transmute(i16x8)a return transmute(__m128i)simd.shuffle( @@ -408,6 +511,7 @@ _mm_shufflehi_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128 ((IMM8 >> 6) & 0b11) + 4, ) } +@(enable_target_feature="sse2") _mm_shufflelo_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { v := transmute(i16x8)a return transmute(__m128i)simd.shuffle( @@ -423,6 +527,7 @@ _mm_shufflelo_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128 7, ) } +@(enable_target_feature="sse2") _mm_unpackhi_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle( transmute(i8x16)a, @@ -430,15 +535,19 @@ _mm_unpackhi_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, ) } +@(enable_target_feature="sse2") _mm_unpackhi_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i16x8)a, transmute(i16x8)b, 4, 12, 5, 13, 6, 14, 7, 15) } +@(enable_target_feature="sse2") _mm_unpackhi_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i32x4)a, transmute(i32x4)b, 2, 6, 3, 7) } +@(enable_target_feature="sse2") _mm_unpackhi_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)b, 1, 3) } +@(enable_target_feature="sse2") _mm_unpacklo_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle( transmute(i8x16)a, @@ -446,12 +555,15 @@ _mm_unpacklo_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23, ) } +@(enable_target_feature="sse2") _mm_unpacklo_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i16x8)a, transmute(i16x8)b, 0, 8, 1, 9, 2, 10, 3, 11) } +@(enable_target_feature="sse2") _mm_unpacklo_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i32x4)a, transmute(i32x4)b, 0, 4, 1, 5) } +@(enable_target_feature="sse2") _mm_unpacklo_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)b, 0, 2) } @@ -459,57 +571,75 @@ _mm_unpacklo_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { +@(enable_target_feature="sse2") _mm_add_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(a) + _mm_cvtsd_f64(b)) } +@(enable_target_feature="sse2") _mm_add_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.add(a, b) } +@(enable_target_feature="sse2") _mm_div_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(a) / _mm_cvtsd_f64(b)) } +@(enable_target_feature="sse2") _mm_div_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.div(a, b) } +@(enable_target_feature="sse2") _mm_max_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return maxsd(a, b) } +@(enable_target_feature="sse2") _mm_max_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return maxpd(a, b) } +@(enable_target_feature="sse2") _mm_min_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return minsd(a, b) } +@(enable_target_feature="sse2") _mm_min_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return minpd(a, b) } +@(enable_target_feature="sse2") _mm_mul_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(a) * _mm_cvtsd_f64(b)) } +@(enable_target_feature="sse2") _mm_mul_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.mul(a, b) } +@(enable_target_feature="sse2") _mm_sqrt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(sqrtsd(b))) } +@(enable_target_feature="sse2") _mm_sqrt_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { return simd.sqrt(a) } +@(enable_target_feature="sse2") _mm_sub_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(a) - _mm_cvtsd_f64(b)) } +@(enable_target_feature="sse2") _mm_sub_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.sub(a, b) } +@(enable_target_feature="sse2") _mm_and_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return transmute(__m128d)_mm_and_si128(transmute(__m128i)a, transmute(__m128i)b) } +@(enable_target_feature="sse2") _mm_andnot_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return transmute(__m128d)_mm_andnot_si128(transmute(__m128i)a, transmute(__m128i)b) } +@(enable_target_feature="sse2") _mm_or_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return transmute(__m128d)_mm_or_si128(transmute(__m128i)a, transmute(__m128i)b) } +@(enable_target_feature="sse2") _mm_xor_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return transmute(__m128d)_mm_xor_si128(transmute(__m128i)a, transmute(__m128i)b) } @@ -517,111 +647,147 @@ _mm_xor_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { +@(enable_target_feature="sse2") _mm_cmpeq_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 0) } +@(enable_target_feature="sse2") _mm_cmplt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 1) } +@(enable_target_feature="sse2") _mm_cmple_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 2) } +@(enable_target_feature="sse2") _mm_cmpgt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(_mm_cmplt_sd(b, a), 1, simd.extract(a, 1)) } +@(enable_target_feature="sse2") _mm_cmpge_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(_mm_cmple_sd(b, a), 1, simd.extract(a, 1)) } +@(enable_target_feature="sse2") _mm_cmpord_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 7) } +@(enable_target_feature="sse2") _mm_cmpunord_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 3) } +@(enable_target_feature="sse2") _mm_cmpneq_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 4) } +@(enable_target_feature="sse2") _mm_cmpnlt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 5) } +@(enable_target_feature="sse2") _mm_cmpnle_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 6) } +@(enable_target_feature="sse2") _mm_cmpngt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(_mm_cmpnlt_sd(b, a), 1, simd.extract(a, 1)) } +@(enable_target_feature="sse2") _mm_cmpnge_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(_mm_cmpnle_sd(b, a), 1, simd.extract(a, 1)) } +@(enable_target_feature="sse2") _mm_cmpeq_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 0) } +@(enable_target_feature="sse2") _mm_cmplt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 1) } +@(enable_target_feature="sse2") _mm_cmple_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 2) } +@(enable_target_feature="sse2") _mm_cmpgt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_cmplt_pd(b, a) } +@(enable_target_feature="sse2") _mm_cmpge_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_cmple_pd(b, a) } +@(enable_target_feature="sse2") _mm_cmpord_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 7) } +@(enable_target_feature="sse2") _mm_cmpunord_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 3) } +@(enable_target_feature="sse2") _mm_cmpneq_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 4) } +@(enable_target_feature="sse2") _mm_cmpnlt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 5) } +@(enable_target_feature="sse2") _mm_cmpnle_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 6) } +@(enable_target_feature="sse2") _mm_cmpngt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_cmpnlt_pd(b, a) } +@(enable_target_feature="sse2") _mm_cmpnge_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_cmpnle_pd(b, a) } +@(enable_target_feature="sse2") _mm_comieq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comieqsd(a, b) } +@(enable_target_feature="sse2") _mm_comilt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comiltsd(a, b) } +@(enable_target_feature="sse2") _mm_comile_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comilesd(a, b) } +@(enable_target_feature="sse2") _mm_comigt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comigtsd(a, b) } +@(enable_target_feature="sse2") _mm_comige_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comigesd(a, b) } +@(enable_target_feature="sse2") _mm_comineq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comineqsd(a, b) } +@(enable_target_feature="sse2") _mm_ucomieq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomieqsd(a, b) } +@(enable_target_feature="sse2") _mm_ucomilt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomiltsd(a, b) } +@(enable_target_feature="sse2") _mm_ucomile_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomilesd(a, b) } +@(enable_target_feature="sse2") _mm_ucomigt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomigtsd(a, b) } +@(enable_target_feature="sse2") _mm_ucomige_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomigesd(a, b) } +@(enable_target_feature="sse2") _mm_ucomineq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomineqsd(a, b) } @@ -630,115 +796,151 @@ _mm_ucomineq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { +@(enable_target_feature="sse2") _mm_cvtpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { return cvtpd2ps(a) } +@(enable_target_feature="sse2") _mm_cvtps_pd :: #force_inline proc "c" (a: __m128) -> __m128d { return cvtps2pd(a) } +@(enable_target_feature="sse2") _mm_cvtpd_epi32 :: #force_inline proc "c" (a: __m128d) -> __m128i { return transmute(__m128i)cvtpd2dq(a) } +@(enable_target_feature="sse2") _mm_cvtsd_si32 :: #force_inline proc "c" (a: __m128d) -> i32 { return cvtsd2si(a) } +@(enable_target_feature="sse2") _mm_cvtsd_ss :: #force_inline proc "c" (a, b: __m128d) -> __m128 { return cvtsd2ss(a, b) } +@(enable_target_feature="sse2") _mm_cvtsd_f64 :: #force_inline proc "c" (a: __m128d) -> f64 { return simd.extract(a, 0) } +@(enable_target_feature="sse2") _mm_cvtss_sd :: #force_inline proc "c" (a, b: __m128) -> __m128d { return cvtss2sd(a, b) } +@(enable_target_feature="sse2") _mm_cvttpd_epi32 :: #force_inline proc "c" (a: __m128d) -> __m128i { return transmute(__m128i)cvttpd2dq(a) } +@(enable_target_feature="sse2") _mm_cvttsd_si32 :: #force_inline proc "c" (a: __m128d) -> i32 { return cvttsd2si(a) } +@(enable_target_feature="sse2") _mm_cvttps_epi32 :: #force_inline proc "c" (a: __m128) -> __m128i { return transmute(__m128i)cvttps2dq(a) } +@(enable_target_feature="sse2") _mm_set_sd :: #force_inline proc "c" (a: f64) -> __m128d { return _mm_set_pd(0.0, a) } +@(enable_target_feature="sse2") _mm_set1_pd :: #force_inline proc "c" (a: f64) -> __m128d { return _mm_set_pd(a, a) } +@(enable_target_feature="sse2") _mm_set_pd1 :: #force_inline proc "c" (a: f64) -> __m128d { return _mm_set_pd(a, a) } +@(enable_target_feature="sse2") _mm_set_pd :: #force_inline proc "c" (a: f64, b: f64) -> __m128d { return __m128d{b, a} } +@(enable_target_feature="sse2") _mm_setr_pd :: #force_inline proc "c" (a: f64, b: f64) -> __m128d { return _mm_set_pd(b, a) } +@(enable_target_feature="sse2") _mm_setzero_pd :: #force_inline proc "c" () -> __m128d { return _mm_set_pd(0.0, 0.0) } +@(enable_target_feature="sse2") _mm_movemask_pd :: #force_inline proc "c" (a: __m128d) -> i32 { return movmskpd(a) } +@(enable_target_feature="sse2") _mm_load_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { return (^__m128d)(mem_addr)^ } +@(enable_target_feature="sse2") _mm_load_sd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { return _mm_setr_pd(mem_addr^, 0.) } +@(enable_target_feature="sse2") _mm_loadh_pd :: #force_inline proc "c" (a: __m128d, mem_addr: ^f64) -> __m128d { return _mm_setr_pd(simd.extract(a, 0), mem_addr^) } +@(enable_target_feature="sse2") _mm_loadl_pd :: #force_inline proc "c" (a: __m128d, mem_addr: ^f64) -> __m128d { return _mm_setr_pd(mem_addr^, simd.extract(a, 1)) } +@(enable_target_feature="sse2") _mm_stream_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { intrinsics.non_temporal_store((^__m128d)(mem_addr), a) } +@(enable_target_feature="sse2") _mm_store_sd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { mem_addr^ = simd.extract(a, 0) } +@(enable_target_feature="sse2") _mm_store_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { (^__m128d)(mem_addr)^ = a } +@(enable_target_feature="sse2") _mm_storeu_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { storeupd(mem_addr, a) } +@(enable_target_feature="sse2") _mm_store1_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { (^__m128d)(mem_addr)^ = simd.shuffle(a, a, 0, 0) } +@(enable_target_feature="sse2") _mm_store_pd1 :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { (^__m128d)(mem_addr)^ = simd.shuffle(a, a, 0, 0) } +@(enable_target_feature="sse2") _mm_storer_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { (^__m128d)(mem_addr)^ = simd.shuffle(a, a, 1, 0) } +@(enable_target_feature="sse2") _mm_storeh_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { mem_addr^ = simd.extract(a, 1) } +@(enable_target_feature="sse2") _mm_storel_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { mem_addr^ = simd.extract(a, 0) } +@(enable_target_feature="sse2") _mm_load1_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { d := mem_addr^ return _mm_setr_pd(d, d) } +@(enable_target_feature="sse2") _mm_load_pd1 :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { return _mm_load1_pd(mem_addr) } +@(enable_target_feature="sse2") _mm_loadr_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { a := _mm_load_pd(mem_addr) return simd.shuffle(a, a, 1, 0) } +@(enable_target_feature="sse2") _mm_loadu_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { dst := _mm_undefined_pd() intrinsics.mem_copy_non_overlapping(&dst, mem_addr, size_of(__m128d)) return dst } +@(enable_target_feature="sse2") _mm_shuffle_pd :: #force_inline proc "c" (a, b: __m128d, $MASK: u32) -> __m128d { return simd.shuffle(a, b, MASK&0b1, ((MASK>>1)&0b1) + 2) } +@(enable_target_feature="sse2") _mm_move_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_setr_pd(simd.extract(b, 0), simd.extract(a, 1)) } @@ -746,71 +948,92 @@ _mm_move_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { +@(enable_target_feature="sse2") _mm_castpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { return transmute(__m128)a } +@(enable_target_feature="sse2") _mm_castpd_si128 :: #force_inline proc "c" (a: __m128d) -> __m128i { return transmute(__m128i)a } +@(enable_target_feature="sse2") _mm_castps_pd :: #force_inline proc "c" (a: __m128) -> __m128d { return transmute(__m128d)a } +@(enable_target_feature="sse2") _mm_castps_si128 :: #force_inline proc "c" (a: __m128) -> __m128i { return transmute(__m128i)a } +@(enable_target_feature="sse2") _mm_castsi128_pd :: #force_inline proc "c" (a: __m128i) -> __m128d { return transmute(__m128d)a } +@(enable_target_feature="sse2") _mm_castsi128_ps :: #force_inline proc "c" (a: __m128i) -> __m128 { return transmute(__m128)a } +@(enable_target_feature="sse2") _mm_undefined_pd :: #force_inline proc "c" () -> __m128d { return __m128d{0, 0} } +@(enable_target_feature="sse2") _mm_undefined_si128 :: #force_inline proc "c" () -> __m128i { return __m128i{0, 0} } +@(enable_target_feature="sse2") _mm_unpackhi_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.shuffle(a, b, 1, 3) } +@(enable_target_feature="sse2") _mm_unpacklo_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.shuffle(a, b, 0, 2) } when ODIN_ARCH == .amd64 { + @(enable_target_feature="sse2") _mm_cvtsd_si64 :: #force_inline proc "c" (a: __m128d) -> i64 { return cvtsd2si64(a) } + @(enable_target_feature="sse2") _mm_cvtsd_si64x :: #force_inline proc "c" (a: __m128d) -> i64 { return _mm_cvtsd_si64(a) } + @(enable_target_feature="sse2") _mm_cvttsd_si64 :: #force_inline proc "c" (a: __m128d) -> i64 { return cvttsd2si64(a) } + @(enable_target_feature="sse2") _mm_cvttsd_si64x :: #force_inline proc "c" (a: __m128d) -> i64 { return _mm_cvttsd_si64(a) } + @(enable_target_feature="sse2") _mm_stream_si64 :: #force_inline proc "c" (mem_addr: ^i64, a: i64) { intrinsics.non_temporal_store(mem_addr, a) } + @(enable_target_feature="sse2") _mm_cvtsi64_si128 :: #force_inline proc "c" (a: i64) -> __m128i { return _mm_set_epi64x(0, a) } + @(enable_target_feature="sse2") _mm_cvtsi64x_si128 :: #force_inline proc "c" (a: i64) -> __m128i { return _mm_cvtsi64_si128(a) } + @(enable_target_feature="sse2") _mm_cvtsi128_si64 :: #force_inline proc "c" (a: __m128i) -> i64 { return simd.extract(transmute(i64x2)a, 0) } + @(enable_target_feature="sse2") _mm_cvtsi128_si64x :: #force_inline proc "c" (a: __m128i) -> i64 { return _mm_cvtsi128_si64(a) } + @(enable_target_feature="sse2") _mm_cvtsi64_sd :: #force_inline proc "c" (a: __m128d, b: i64) -> __m128d { return simd.replace(a, 0, f64(b)) } + @(enable_target_feature="sse2") _mm_cvtsi64x_sd :: #force_inline proc "c" (a: __m128d, b: i64) -> __m128d { return _mm_cvtsi64_sd(a, b) } diff --git a/core/simd/x86/sse3.odin b/core/simd/x86/sse3.odin index 9766a43e6..370bfa952 100644 --- a/core/simd/x86/sse3.odin +++ b/core/simd/x86/sse3.odin @@ -4,36 +4,47 @@ package simd_x86 import "core:intrinsics" import "core:simd" +@(enable_target_feature="sse3") _mm_addsub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return addsubps(a, b) } +@(enable_target_feature="sse3") _mm_addsub_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { return addsubpd(a, b) } +@(enable_target_feature="sse3") _mm_hadd_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { return haddpd(a, b) } +@(enable_target_feature="sse3") _mm_hadd_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return haddps(a, b) } +@(enable_target_feature="sse3") _mm_hsub_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { return hsubpd(a, b) } +@(enable_target_feature="sse3") _mm_hsub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return hsubps(a, b) } +@(enable_target_feature="sse3") _mm_lddqu_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { return transmute(__m128i)lddqu(mem_addr) } +@(enable_target_feature="sse3") _mm_movedup_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { return simd.shuffle(a, a, 0, 0) } +@(enable_target_feature="sse3") _mm_loaddup_pd :: #force_inline proc "c" (mem_addr: [^]f64) -> __m128d { return _mm_load1_pd(mem_addr) } +@(enable_target_feature="sse3") _mm_movehdup_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return simd.shuffle(a, a, 1, 1, 3, 3) } +@(enable_target_feature="sse3") _mm_moveldup_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return simd.shuffle(a, a, 0, 0, 2, 2) } diff --git a/core/simd/x86/ssse3.odin b/core/simd/x86/ssse3.odin index 6c6f28008..8c677aed4 100644 --- a/core/simd/x86/ssse3.odin +++ b/core/simd/x86/ssse3.odin @@ -5,18 +5,23 @@ import "core:intrinsics" import "core:simd" _ :: simd +@(enable_target_feature="ssse3") _mm_abs_epi8 :: #force_inline proc "c" (a: __m128i) -> __m128i { return transmute(__m128i)pabsb128(transmute(i8x16)a) } +@(enable_target_feature="ssse3") _mm_abs_epi16 :: #force_inline proc "c" (a: __m128i) -> __m128i { return transmute(__m128i)pabsw128(transmute(i16x8)a) } +@(enable_target_feature="ssse3") _mm_abs_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { return transmute(__m128i)pabsd128(transmute(i32x4)a) } +@(enable_target_feature="ssse3") _mm_shuffle_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pshufb128(transmute(u8x16)a, transmute(u8x16)b) } +@(enable_target_feature="ssse3") _mm_alignr_epi8 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u32) -> __m128i { shift :: IMM8 @@ -53,36 +58,47 @@ _mm_alignr_epi8 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u32) -> __m128i } +@(enable_target_feature="ssse3") _mm_hadd_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phaddw128(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="ssse3") _mm_hadds_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phaddsw128(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="ssse3") _mm_hadd_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phaddd128(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="ssse3") _mm_hsub_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phsubw128(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="ssse3") _mm_hsubs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phsubsw128(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="ssse3") _mm_hsub_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phsubd128(transmute(i32x4)a, transmute(i32x4)b) } +@(enable_target_feature="ssse3") _mm_maddubs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaddubsw128(transmute(u8x16)a, transmute(i8x16)b) } +@(enable_target_feature="ssse3") _mm_mulhrs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmulhrsw128(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="ssse3") _mm_sign_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psignb128(transmute(i8x16)a, transmute(i8x16)b) } +@(enable_target_feature="ssse3") _mm_sign_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psignw128(transmute(i16x8)a, transmute(i16x8)b) } +@(enable_target_feature="ssse3") _mm_sign_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psignd128(transmute(i32x4)a, transmute(i32x4)b) } diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 27e09a0db..65470749f 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -256,7 +256,6 @@ struct BuildContext { String extra_linker_flags; String extra_assembler_flags; String microarch; - String target_features; BuildModeKind build_mode; bool generate_docs; i32 optimization_level; @@ -320,6 +319,10 @@ struct BuildContext { PtrMap defined_values; + BlockingMutex target_features_mutex; + StringSet target_features_set; + String target_features_string; + }; gb_global BuildContext build_context = {0}; @@ -1197,6 +1200,100 @@ void init_build_context(TargetMetrics *cross_target) { #include "microsoft_craziness.h" #endif + +Array split_by_comma(String const &list) { + isize n = 1; + for (isize i = 0; i < list.len; i++) { + if (list.text[i] == ',') { + n++; + } + } + auto res = array_make(heap_allocator(), n); + + String s = list; + for (isize i = 0; i < n; i++) { + isize m = string_index_byte(s, ','); + if (m < 0) { + res[i] = s; + break; + } + res[i] = substring(s, 0, m); + s = substring(s, m+1, s.len); + } + return res; +} + +bool check_target_feature_is_valid(TokenPos pos, String const &feature) { + // TODO(bill): check_target_feature_is_valid + return true; +} + +bool check_target_feature_is_enabled(TokenPos pos, String const &target_feature_list) { + BuildContext *bc = &build_context; + mutex_lock(&bc->target_features_mutex); + defer (mutex_unlock(&bc->target_features_mutex)); + + auto items = split_by_comma(target_feature_list); + array_free(&items); + for_array(i, items) { + String const &item = items.data[i]; + if (!check_target_feature_is_valid(pos, item)) { + error(pos, "Target feature '%.*s' is not valid", LIT(item)); + return false; + } + if (!string_set_exists(&bc->target_features_set, item)) { + error(pos, "Target feature '%.*s' is not enabled", LIT(item)); + return false; + } + } + + return true; +} + +void enable_target_feature(TokenPos pos, String const &target_feature_list) { + BuildContext *bc = &build_context; + mutex_lock(&bc->target_features_mutex); + defer (mutex_unlock(&bc->target_features_mutex)); + + auto items = split_by_comma(target_feature_list); + array_free(&items); + for_array(i, items) { + String const &item = items.data[i]; + if (!check_target_feature_is_valid(pos, item)) { + error(pos, "Target feature '%.*s' is not valid", LIT(item)); + } + } +} + + +char const *target_features_set_to_cstring(gbAllocator allocator, bool with_quotes) { + isize len = 0; + for_array(i, build_context.target_features_set.entries) { + if (i != 0) { + len += 1; + } + String feature = build_context.target_features_set.entries[i].value; + len += feature.len; + if (with_quotes) len += 2; + } + char *features = gb_alloc_array(allocator, char, len+1); + len = 0; + for_array(i, build_context.target_features_set.entries) { + if (i != 0) { + features[len++] = ','; + } + + if (with_quotes) features[len++] = '"'; + String feature = build_context.target_features_set.entries[i].value; + gb_memmove(features, feature.text, feature.len); + len += feature.len; + if (with_quotes) features[len++] = '"'; + } + features[len++] = 0; + + return features; +} + // NOTE(Jeroen): Set/create the output and other paths and report an error as appropriate. // We've previously called `parse_build_flags`, so `out_filepath` should be set. bool init_build_paths(String init_filename) { @@ -1206,6 +1303,9 @@ bool init_build_paths(String init_filename) { // NOTE(Jeroen): We're pre-allocating BuildPathCOUNT slots so that certain paths are always at the same enumerated index. array_init(&bc->build_paths, permanent_allocator(), BuildPathCOUNT); + string_set_init(&bc->target_features_set, heap_allocator(), 1024); + mutex_init(&bc->target_features_mutex); + // [BuildPathMainPackage] Turn given init path into a `Path`, which includes normalizing it into a full path. bc->build_paths[BuildPath_Main_Package] = path_from_string(ha, init_filename); @@ -1382,5 +1482,10 @@ bool init_build_paths(String init_filename) { return false; } + if (bc->target_features_string.len != 0) { + enable_target_feature({}, bc->target_features_string); + } + return true; -} \ No newline at end of file +} + diff --git a/src/check_decl.cpp b/src/check_decl.cpp index 62a1e2555..d4818892b 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -899,6 +899,18 @@ void check_proc_decl(CheckerContext *ctx, Entity *e, DeclInfo *d) { } } + if (ac.require_target_feature.len != 0 && ac.enable_target_feature.len != 0) { + error(e->token, "Attributes @(require_target_feature=...) and @(enable_target_feature=...) cannot be used together"); + } else if (ac.require_target_feature.len != 0) { + if (check_target_feature_is_enabled(e->token.pos, ac.require_target_feature)) { + e->Procedure.target_feature = ac.require_target_feature; + } else { + e->Procedure.target_feature_disabled = true; + } + } else if (ac.enable_target_feature.len != 0) { + enable_target_feature(e->token.pos, ac.enable_target_feature); + e->Procedure.target_feature = ac.enable_target_feature; + } switch (e->Procedure.optimization_mode) { case ProcedureOptimizationMode_None: diff --git a/src/checker.cpp b/src/checker.cpp index 8afc6eb14..874839ece 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -3207,6 +3207,22 @@ DECL_ATTRIBUTE_PROC(proc_decl_attribute) { } } return true; + } else if (name == "require_target_feature") { + ExactValue ev = check_decl_attribute_value(c, value); + if (ev.kind == ExactValue_String) { + ac->require_target_feature = ev.value_string; + } else { + error(elem, "Expected a string value for '%.*s'", LIT(name)); + } + return true; + } else if (name == "enable_target_feature") { + ExactValue ev = check_decl_attribute_value(c, value); + if (ev.kind == ExactValue_String) { + ac->enable_target_feature = ev.value_string; + } else { + error(elem, "Expected a string value for '%.*s'", LIT(name)); + } + return true; } return false; } diff --git a/src/checker.hpp b/src/checker.hpp index 1c9ffd8c7..8fc9e54c8 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -124,6 +124,9 @@ struct AttributeContext { String objc_name; bool objc_is_class_method; Type * objc_type; + + String require_target_feature; // required by the target micro-architecture + String enable_target_feature; // will be enabled for the procedure only }; AttributeContext make_attribute_context(String link_prefix) { diff --git a/src/entity.cpp b/src/entity.cpp index 904a630fb..76e6912b9 100644 --- a/src/entity.cpp +++ b/src/entity.cpp @@ -233,10 +233,12 @@ struct Entity { String link_name; String link_prefix; DeferredProcedure deferred_procedure; - bool is_foreign; - bool is_export; - bool generated_from_polymorphic; ProcedureOptimizationMode optimization_mode; + bool is_foreign : 1; + bool is_export : 1; + bool generated_from_polymorphic : 1; + bool target_feature_disabled : 1; + String target_feature; } Procedure; struct { Array entities; diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 7cf588853..cf7389ec1 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -1332,8 +1332,8 @@ void lb_generate_code(lbGenerator *gen) { } } - if (build_context.target_features.len != 0) { - llvm_features = alloc_cstring(permanent_allocator(), build_context.target_features); + if (build_context.target_features_set.entries.count != 0) { + llvm_features = target_features_set_to_cstring(permanent_allocator(), false); } // GB_ASSERT_MSG(LLVMTargetHasAsmBackend(target)); diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 1e3591bf1..296a7fa6b 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -169,6 +169,19 @@ lbProcedure *lb_create_procedure(lbModule *m, Entity *entity, bool ignore_body) } } + if (!entity->Procedure.target_feature_disabled && + entity->Procedure.target_feature.len != 0) { + auto features = split_by_comma(entity->Procedure.target_feature); + for_array(i, features) { + String feature = features[i]; + LLVMAttributeRef ref = LLVMCreateStringAttribute( + m->ctx, + cast(char const *)feature.text, cast(unsigned)feature.len, + "", 0); + LLVMAddAttributeAtIndex(p->value, LLVMAttributeIndex_FunctionIndex, ref); + } + } + if (entity->flags & EntityFlag_Cold) { lb_add_attribute_to_proc(m, p->value, "cold"); } diff --git a/src/main.cpp b/src/main.cpp index 13c8bd74d..ee71b91df 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1376,8 +1376,8 @@ bool parse_build_flags(Array args) { } case BuildFlag_TargetFeatures: { GB_ASSERT(value.kind == ExactValue_String); - build_context.target_features = value.value_string; - string_to_lower(&build_context.target_features); + build_context.target_features_string = value.value_string; + string_to_lower(&build_context.target_features_string); break; } case BuildFlag_RelocMode: { diff --git a/src/string.cpp b/src/string.cpp index 616761265..44eccd2d2 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -157,6 +157,15 @@ int string_compare(String const &x, String const &y) { return 0; } +isize string_index_byte(String const &s, u8 x) { + for (isize i = 0; i < s.len; i++) { + if (s.text[i] == x) { + return i; + } + } + return -1; +} + GB_COMPARE_PROC(string_cmp_proc) { String x = *(String *)a; String y = *(String *)b; From a0babefe55568c400835a5984db676d9f650a8b4 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 15:13:45 +0100 Subject: [PATCH 77/86] Fix lb_build_builtin_simd_proc --- src/llvm_backend_proc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 296a7fa6b..e3ffcaef2 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -1021,8 +1021,8 @@ lbValue lb_build_builtin_simd_proc(lbProcedure *p, Ast *expr, TypeAndValue const res.type = tv.type; lbValue arg0 = {}; if (ce->args.count > 0) arg0 = lb_build_expr(p, ce->args[0]); - lbValue arg1 = {}; if (ce->args.count > 1) arg0 = lb_build_expr(p, ce->args[1]); - lbValue arg2 = {}; if (ce->args.count > 2) arg0 = lb_build_expr(p, ce->args[2]); + lbValue arg1 = {}; if (ce->args.count > 1) arg1 = lb_build_expr(p, ce->args[1]); + lbValue arg2 = {}; if (ce->args.count > 2) arg2 = lb_build_expr(p, ce->args[2]); Type *elem = base_array_type(arg0.type); From 51707032d10d166094959d49e597b322a8d7c55d Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 15:17:02 +0100 Subject: [PATCH 78/86] Add SSE4.1 --- core/simd/x86/sse41.odin | 352 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 core/simd/x86/sse41.odin diff --git a/core/simd/x86/sse41.odin b/core/simd/x86/sse41.odin new file mode 100644 index 000000000..516e0bdc2 --- /dev/null +++ b/core/simd/x86/sse41.odin @@ -0,0 +1,352 @@ +//+build i386, amd64 +package simd_x86 + +import "core:simd" + +// SSE4 rounding constants +_MM_FROUND_TO_NEAREST_INT :: 0x00 +_MM_FROUND_TO_NEG_INF :: 0x01 +_MM_FROUND_TO_POS_INF :: 0x02 +_MM_FROUND_TO_ZERO :: 0x03 +_MM_FROUND_CUR_DIRECTION :: 0x04 +_MM_FROUND_RAISE_EXC :: 0x00 +_MM_FROUND_NO_EXC :: 0x08 +_MM_FROUND_NINT :: 0x00 +_MM_FROUND_FLOOR :: _MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEG_INF +_MM_FROUND_CEIL :: _MM_FROUND_RAISE_EXC | _MM_FROUND_TO_POS_INF +_MM_FROUND_TRUNC :: _MM_FROUND_RAISE_EXC | _MM_FROUND_TO_ZERO +_MM_FROUND_RINT :: _MM_FROUND_RAISE_EXC | _MM_FROUND_CUR_DIRECTION +_MM_FROUND_NEARBYINT :: _MM_FROUND_NO_EXC | _MM_FROUND_CUR_DIRECTION + + + +@(enable_target_feature="sse4.1") +_mm_blendv_epi8 :: #force_inline proc "c" (a, b, mask: __m128i) -> __m128i { + return transmute(__m128i)pblendvb(transmute(i8x16)a, transmute(i8x16)b, transmute(i8x16)mask) +} +@(enable_target_feature="sse4.1") +_mm_blend_epi16 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u8) -> __m128i { + return transmute(__m128i)pblendw(transmute(i16x8)a, transmute(i16x8)b, IMM8) +} +@(enable_target_feature="sse4.1") +_mm_blendv_pd :: #force_inline proc "c" (a, b, mask: __m128d) -> __m128d { + return blendvpd(a, b, mask) +} +@(enable_target_feature="sse4.1") +_mm_blendv_ps :: #force_inline proc "c" (a, b, mask: __m128) -> __m128 { + return blendvps(a, b, mask) +} +@(enable_target_feature="sse4.1") +_mm_blend_pd :: #force_inline proc "c" (a, b: __m128d, $IMM2: u8) -> __m128d { + return blendpd(a, b, IMM2) +} +@(enable_target_feature="sse4.1") +_mm_blend_ps :: #force_inline proc "c" (a, b: __m128, $IMM4: u8) -> __m128 { + return blendps(a, b, IMM4) +} +@(enable_target_feature="sse4.1") +_mm_extract_ps :: #force_inline proc "c" (a: __m128, $IMM8: u32) -> i32 { + return transmute(i32)simd.extract(a, IMM8) +} +@(enable_target_feature="sse4.1") +_mm_extract_epi8 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> i32 { + return i32(simd.extract(transmute(u8x16)a, IMM8)) +} +@(enable_target_feature="sse4.1") +_mm_extract_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> i32 { + return simd.extract(transmute(i32x4)a, IMM8) +} +@(enable_target_feature="sse4.1") +_mm_insert_ps :: #force_inline proc "c" (a, b: __m128, $IMM8: u8) -> __m128 { + return insertps(a, b, IMM8) +} +@(enable_target_feature="sse4.1") +_mm_insert_epi8 :: #force_inline proc "c" (a: __m128i, i: i32, $IMM8: u32) -> __m128i { + return transmute(__m128i)simd.replace(transmute(i8x16)a, IMM8, i8(i)) +} +@(enable_target_feature="sse4.1") +_mm_insert_epi32 :: #force_inline proc "c" (a: __m128i, i: i32, $IMM8: u32) -> __m128i { + return transmute(__m128i)simd.replace(transmute(i32x4)a, IMM8, i) +} +@(enable_target_feature="sse4.1") +_mm_max_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pmaxsb(transmute(i8x16)a, transmute(i8x16)b) +} +@(enable_target_feature="sse4.1") +_mm_max_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pmaxuw(transmute(u16x8)a, transmute(u16x8)b) +} +@(enable_target_feature="sse4.1") +_mm_max_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pmaxsd(transmute(i32x4)a, transmute(i32x4)b) +} +@(enable_target_feature="sse4.1") +_mm_max_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pmaxud(transmute(u32x4)a, transmute(u32x4)b) +} +@(enable_target_feature="sse4.1") +_mm_min_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pminsb(transmute(i8x16)a, transmute(i8x16)b) +} +@(enable_target_feature="sse4.1") +_mm_min_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pminuw(transmute(u16x8)a, transmute(u16x8)b) +} +@(enable_target_feature="sse4.1") +_mm_min_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pminsd(transmute(i32x4)a, transmute(i32x4)b) +} +@(enable_target_feature="sse4.1") +_mm_min_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pminud(transmute(u32x4)a, transmute(u32x4)b) +} +@(enable_target_feature="sse4.1") +_mm_packus_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)packusdw(transmute(i32x4)a, transmute(i32x4)b) +} +@(enable_target_feature="sse4.1") +_mm_cmpeq_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_eq(transmute(i64x2)a, transmute(i64x2)b) +} +@(enable_target_feature="sse4.1") +_mm_cvtepi8_epi16 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(i8x16)a + y := simd.shuffle(x, x, 0, 1, 2, 3, 4, 5, 6, 7) + return transmute(__m128i)i16x8(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepi8_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(i8x16)a + y := simd.shuffle(x, x, 0, 1, 2, 3) + return transmute(__m128i)i32x4(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepi8_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(i8x16)a + y := simd.shuffle(x, x, 0, 1) + return transmute(__m128i)i64x2(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepi16_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(i16x8)a + y := simd.shuffle(x, x, 0, 1, 2, 3) + return transmute(__m128i)i32x4(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepi16_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(i16x8)a + y := simd.shuffle(x, x, 0, 1) + return transmute(__m128i)i64x2(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepi32_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(i32x4)a + y := simd.shuffle(x, x, 0, 1) + return transmute(__m128i)i64x2(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepu8_epi16 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(u8x16)a + y := simd.shuffle(x, x, 0, 1, 2, 3, 4, 5, 6, 7) + return transmute(__m128i)i16x8(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepu8_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(u8x16)a + y := simd.shuffle(x, x, 0, 1, 2, 3) + return transmute(__m128i)i32x4(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepu8_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(u8x16)a + y := simd.shuffle(x, x, 0, 1) + return transmute(__m128i)i64x2(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepu16_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(u16x8)a + y := simd.shuffle(x, x, 0, 1, 2, 3) + return transmute(__m128i)i32x4(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepu16_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(u16x8)a + y := simd.shuffle(x, x, 0, 1) + return transmute(__m128i)i64x2(y) +} +@(enable_target_feature="sse4.1") +_mm_cvtepu32_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { + x := transmute(u32x4)a + y := simd.shuffle(x, x, 0, 1) + return transmute(__m128i)i64x2(y) +} +@(enable_target_feature="sse4.1") +_mm_dp_pd :: #force_inline proc "c" (a, b: __m128d, $IMM8: u8) -> __m128d { + return dppd(a, b, IMM8) +} +@(enable_target_feature="sse4.1") +_mm_dp_ps :: #force_inline proc "c" (a, b: __m128, $IMM8: u8) -> __m128 { + return dpps(a, b, IMM8) +} +@(enable_target_feature="sse4.1") +_mm_floor_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { + return simd.floor(a) +} +@(enable_target_feature="sse4.1") +_mm_floor_ps :: #force_inline proc "c" (a: __m128) -> __m128 { + return simd.floor(a) +} +@(enable_target_feature="sse4.1") +_mm_floor_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return roundsd(a, b, _MM_FROUND_FLOOR) +} +@(enable_target_feature="sse4.1") +_mm_floor_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return roundss(a, b, _MM_FROUND_FLOOR) +} +@(enable_target_feature="sse4.1") +_mm_ceil_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { + return simd.ceil(a) +} +@(enable_target_feature="sse4.1") +_mm_ceil_ps :: #force_inline proc "c" (a: __m128) -> __m128 { + return simd.ceil(a) +} +@(enable_target_feature="sse4.1") +_mm_ceil_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { + return roundsd(a, b, _MM_FROUND_CEIL) +} +@(enable_target_feature="sse4.1") +_mm_ceil_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { + return roundss(a, b, _MM_FROUND_CEIL) +} +@(enable_target_feature="sse4.1") +_mm_round_pd :: #force_inline proc "c" (a: __m128d, $ROUNDING: i32) -> __m128d { + return roundpd(a, ROUNDING) +} +@(enable_target_feature="sse4.1") +_mm_round_ps :: #force_inline proc "c" (a: __m128, $ROUNDING: i32) -> __m128 { + return roundps(a, ROUNDING) +} +@(enable_target_feature="sse4.1") +_mm_round_sd :: #force_inline proc "c" (a, b: __m128d, $ROUNDING: i32) -> __m128d { + return roundsd(a, b, ROUNDING) +} +@(enable_target_feature="sse4.1") +_mm_round_ss :: #force_inline proc "c" (a, b: __m128, $ROUNDING: i32) -> __m128 { + return roundss(a, b, ROUNDING) +} +@(enable_target_feature="sse4.1") +_mm_minpos_epu16 :: #force_inline proc "c" (a: __m128i) -> __m128i { + return transmute(__m128i)phminposuw(transmute(u16x8)a) +} +@(enable_target_feature="sse4.1") +_mm_mul_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)pmuldq(transmute(i32x4)a, transmute(i32x4)b) +} +@(enable_target_feature="sse4.1") +_mm_mullo_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { + return transmute(__m128i)simd.mul(transmute(i32x4)a, transmute(i32x4)b) +} +@(enable_target_feature="sse4.1") +_mm_mpsadbw_epu8 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u8) -> __m128i { + return transmute(__m128i)mpsadbw(transmute(u8x16)a, transmute(u8x16)b, IMM8) +} +@(enable_target_feature="sse4.1") +_mm_testz_si128 :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { + return ptestz(transmute(i64x2)a, transmute(i64x2)mask) +} +@(enable_target_feature="sse4.1") +_mm_testc_si128 :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { + return ptestc(transmute(i64x2)a, transmute(i64x2)mask) +} +@(enable_target_feature="sse4.1") +_mm_testnzc_si128 :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { + return ptestnzc(transmute(i64x2)a, transmute(i64x2)mask) +} +@(enable_target_feature="sse4.1") +_mm_test_all_zeros :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { + return _mm_testz_si128(a, mask) +} +@(enable_target_feature="sse4.1") +_mm_test_all_ones :: #force_inline proc "c" (a: __m128i) -> i32 { + return _mm_testc_si128(a, _mm_cmpeq_epi32(a, a)) +} +@(enable_target_feature="sse4.1") +_mm_test_mix_ones_zeros :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { + return _mm_testnzc_si128(a, mask) +} + + +when ODIN_ARCH == .amd64 { + @(enable_target_feature="sse4.1") + _mm_extract_epi64 :: #force_inline proc "c" (a: __m128i, $IMM1: u32) -> i64 { + return simd.extract(transmute(i64x2)a, IMM1) + } + + @(enable_target_feature="sse4.1") + _mm_insert_epi64 :: #force_inline proc "c" (a: __m128i, i: i64, $IMM1: u32) -> __m128i { + return transmute(__m128i)simd.replace(transmute(i64x2)a, IMM1, i) + } +} + + +@(private, default_calling_convention="c") +foreign _ { + @(link_name = "llvm.x86.sse41.pblendvb") + pblendvb :: proc(a, b: i8x16, mask: i8x16) -> i8x16 --- + @(link_name = "llvm.x86.sse41.blendvpd") + blendvpd :: proc(a, b, mask: __m128d) -> __m128d --- + @(link_name = "llvm.x86.sse41.blendvps") + blendvps :: proc(a, b, mask: __m128) -> __m128 --- + @(link_name = "llvm.x86.sse41.blendpd") + blendpd :: proc(a, b: __m128d, #const imm2: u8) -> __m128d --- + @(link_name = "llvm.x86.sse41.blendps") + blendps :: proc(a, b: __m128, #const imm4: u8) -> __m128 --- + @(link_name = "llvm.x86.sse41.pblendw") + pblendw :: proc(a: i16x8, b: i16x8, #const imm8: u8) -> i16x8 --- + @(link_name = "llvm.x86.sse41.insertps") + insertps :: proc(a, b: __m128, #const imm8: u8) -> __m128 --- + @(link_name = "llvm.x86.sse41.pmaxsb") + pmaxsb :: proc(a, b: i8x16) -> i8x16 --- + @(link_name = "llvm.x86.sse41.pmaxuw") + pmaxuw :: proc(a, b: u16x8) -> u16x8 --- + @(link_name = "llvm.x86.sse41.pmaxsd") + pmaxsd :: proc(a, b: i32x4) -> i32x4 --- + @(link_name = "llvm.x86.sse41.pmaxud") + pmaxud :: proc(a, b: u32x4) -> u32x4 --- + @(link_name = "llvm.x86.sse41.pminsb") + pminsb :: proc(a, b: i8x16) -> i8x16 --- + @(link_name = "llvm.x86.sse41.pminuw") + pminuw :: proc(a, b: u16x8) -> u16x8 --- + @(link_name = "llvm.x86.sse41.pminsd") + pminsd :: proc(a, b: i32x4) -> i32x4 --- + @(link_name = "llvm.x86.sse41.pminud") + pminud :: proc(a, b: u32x4) -> u32x4 --- + @(link_name = "llvm.x86.sse41.packusdw") + packusdw :: proc(a, b: i32x4) -> u16x8 --- + @(link_name = "llvm.x86.sse41.dppd") + dppd :: proc(a, b: __m128d, #const imm8: u8) -> __m128d --- + @(link_name = "llvm.x86.sse41.dpps") + dpps :: proc(a, b: __m128, #const imm8: u8) -> __m128 --- + @(link_name = "llvm.x86.sse41.round.pd") + roundpd :: proc(a: __m128d, rounding: i32) -> __m128d --- + @(link_name = "llvm.x86.sse41.round.ps") + roundps :: proc(a: __m128, rounding: i32) -> __m128 --- + @(link_name = "llvm.x86.sse41.round.sd") + roundsd :: proc(a, b: __m128d, rounding: i32) -> __m128d --- + @(link_name = "llvm.x86.sse41.round.ss") + roundss :: proc(a, b: __m128, rounding: i32) -> __m128 --- + @(link_name = "llvm.x86.sse41.phminposuw") + phminposuw :: proc(a: u16x8) -> u16x8 --- + @(link_name = "llvm.x86.sse41.pmuldq") + pmuldq :: proc(a, b: i32x4) -> i64x2 --- + @(link_name = "llvm.x86.sse41.mpsadbw") + mpsadbw :: proc(a, b: u8x16, #const imm8: u8) -> u16x8 --- + @(link_name = "llvm.x86.sse41.ptestz") + ptestz :: proc(a, mask: i64x2) -> i32 --- + @(link_name = "llvm.x86.sse41.ptestc") + ptestc :: proc(a, mask: i64x2) -> i32 --- + @(link_name = "llvm.x86.sse41.ptestnzc") + ptestnzc :: proc(a, mask: i64x2) -> i32 --- +} \ No newline at end of file From 5b42dd7707a8921321958934cf5d75ada1d8a006 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 15:27:09 +0100 Subject: [PATCH 79/86] Correct `@(require_results)` on parapoly procedures --- src/check_decl.cpp | 10 ++++++---- src/check_expr.cpp | 8 ++++++++ src/check_stmt.cpp | 10 +++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/check_decl.cpp b/src/check_decl.cpp index d4818892b..86280b6cb 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -1014,10 +1014,12 @@ void check_proc_decl(CheckerContext *ctx, Entity *e, DeclInfo *d) { } } - if (pt->result_count == 0 && ac.require_results) { - error(pl->type, "'require_results' is not needed on a procedure with no results"); - } else { - pt->require_results = ac.require_results; + if (ac.require_results) { + if (pt->result_count == 0) { + error(pl->type, "'require_results' is not needed on a procedure with no results"); + } else { + pt->require_results = true; + } } if (ac.link_name.len > 0) { diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 7fe9b8acf..f954f1583 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -442,6 +442,14 @@ bool find_or_generate_polymorphic_procedure(CheckerContext *old_c, Entity *base_ final_proc_type->Proc.is_poly_specialized = true; final_proc_type->Proc.is_polymorphic = true; + final_proc_type->Proc.variadic = src->Proc.variadic; + final_proc_type->Proc.require_results = src->Proc.require_results; + final_proc_type->Proc.c_vararg = src->Proc.c_vararg; + final_proc_type->Proc.has_named_results = src->Proc.has_named_results; + final_proc_type->Proc.diverging = src->Proc.diverging; + final_proc_type->Proc.return_by_pointer = src->Proc.return_by_pointer; + final_proc_type->Proc.optional_ok = src->Proc.optional_ok; + for (isize i = 0; i < operands.count; i++) { Operand o = operands[i]; diff --git a/src/check_stmt.cpp b/src/check_stmt.cpp index f2c830c1b..b316f940f 100644 --- a/src/check_stmt.cpp +++ b/src/check_stmt.cpp @@ -1405,12 +1405,12 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { if (kind == Expr_Stmt) { return; } - Ast *expr = strip_or_return_expr(operand.expr); + Ast *expr = strip_or_return_expr(operand.expr); if (expr->kind == Ast_CallExpr) { AstCallExpr *ce = &expr->CallExpr; - Type *t = type_of_expr(ce->proc); - if (is_type_proc(t)) { + Type *t = base_type(type_of_expr(ce->proc)); + if (t->kind == Type_Proc) { if (t->Proc.require_results) { gbString expr_str = expr_to_string(ce->proc); error(node, "'%s' requires that its results must be handled", expr_str); @@ -1421,8 +1421,8 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { } else if (expr->kind == Ast_SelectorCallExpr) { AstSelectorCallExpr *se = &expr->SelectorCallExpr; ast_node(ce, CallExpr, se->call); - Type *t = type_of_expr(ce->proc); - if (is_type_proc(t)) { + Type *t = base_type(type_of_expr(ce->proc)); + if (t->kind == Type_Proc) { if (t->Proc.require_results) { gbString expr_str = expr_to_string(ce->proc); error(node, "'%s' requires that its results must be handled", expr_str); From f3868ac932fcdae54e1009a6b39da37044add07c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 15:57:26 +0100 Subject: [PATCH 80/86] Improve missing handled results for built in procedures --- src/check_stmt.cpp | 46 +++++++++++++++++++++++++-------- src/checker.hpp | 1 + src/checker_builtin_procs.hpp | 48 +++++++++++++++++------------------ src/parser.hpp | 1 - 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/src/check_stmt.cpp b/src/check_stmt.cpp index b316f940f..f061b4961 100644 --- a/src/check_stmt.cpp +++ b/src/check_stmt.cpp @@ -1381,6 +1381,18 @@ bool all_operands_valid(Array const &operands) { return true; } +bool check_stmt_internal_builtin_proc_id(Ast *expr, BuiltinProcId *id_) { + BuiltinProcId id = BuiltinProc_Invalid; + Entity *e = entity_of_node(expr); + if (e != nullptr && e->kind == Entity_Builtin) { + if (e->Builtin.id && e->Builtin.id != BuiltinProc_DIRECTIVE) { + id = cast(BuiltinProcId)e->Builtin.id; + } + } + if (id_) *id_ = id; + return id != BuiltinProc_Invalid; +} + void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { u32 mod_flags = flags & (~Stmt_FallthroughAllowed); switch (node->kind) { @@ -1408,26 +1420,40 @@ void check_stmt_internal(CheckerContext *ctx, Ast *node, u32 flags) { Ast *expr = strip_or_return_expr(operand.expr); if (expr->kind == Ast_CallExpr) { + BuiltinProcId builtin_id = BuiltinProc_Invalid; + bool do_require = false; + AstCallExpr *ce = &expr->CallExpr; Type *t = base_type(type_of_expr(ce->proc)); if (t->kind == Type_Proc) { - if (t->Proc.require_results) { - gbString expr_str = expr_to_string(ce->proc); - error(node, "'%s' requires that its results must be handled", expr_str); - gb_string_free(expr_str); - } + do_require = t->Proc.require_results; + } else if (check_stmt_internal_builtin_proc_id(ce->proc, &builtin_id)) { + auto const &bp = builtin_procs[builtin_id]; + do_require = bp.kind == Expr_Expr && !bp.ignore_results; + } + if (do_require) { + gbString expr_str = expr_to_string(ce->proc); + error(node, "'%s' requires that its results must be handled", expr_str); + gb_string_free(expr_str); } return; } else if (expr->kind == Ast_SelectorCallExpr) { + BuiltinProcId builtin_id = BuiltinProc_Invalid; + bool do_require = false; + AstSelectorCallExpr *se = &expr->SelectorCallExpr; ast_node(ce, CallExpr, se->call); Type *t = base_type(type_of_expr(ce->proc)); if (t->kind == Type_Proc) { - if (t->Proc.require_results) { - gbString expr_str = expr_to_string(ce->proc); - error(node, "'%s' requires that its results must be handled", expr_str); - gb_string_free(expr_str); - } + do_require = t->Proc.require_results; + } else if (check_stmt_internal_builtin_proc_id(ce->proc, &builtin_id)) { + auto const &bp = builtin_procs[builtin_id]; + do_require = bp.kind == Expr_Expr && !bp.ignore_results; + } + if (do_require) { + gbString expr_str = expr_to_string(ce->proc); + error(node, "'%s' requires that its results must be handled", expr_str); + gb_string_free(expr_str); } return; } diff --git a/src/checker.hpp b/src/checker.hpp index 8fc9e54c8..f11a00532 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -60,6 +60,7 @@ struct BuiltinProc { ExprKind kind; BuiltinProcPkg pkg; bool diverging; + bool ignore_results; // ignores require results handling }; diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 2e27cc026..05f256775 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -385,26 +385,26 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("atomic_signal_fence"), 1, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("atomic_store"), 2, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("atomic_store_explicit"), 3, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_load_explicit"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_add"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_add_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_sub"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_sub_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_and"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_and_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_nand"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_nand_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_or"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_or_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_xor"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_xor_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_exchange"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_exchange_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_compare_exchange_strong"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_compare_exchange_strong_explicit"), 5, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_compare_exchange_weak"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("atomic_compare_exchange_weak_explicit"), 5, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("atomic_load"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_load_explicit"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_add"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_add_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_sub"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_sub_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_and"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_and_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_nand"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_nand_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_or"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_or_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_xor"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_xor_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_exchange"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_exchange_explicit"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_compare_exchange_strong"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_compare_exchange_strong_explicit"), 5, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_compare_exchange_weak"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("atomic_compare_exchange_weak_explicit"), 5, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, {STR_LIT("fixed_point_mul"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("fixed_point_div"), 3, false, Expr_Expr, BuiltinProcPkg_intrinsics}, @@ -477,7 +477,7 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, - {STR_LIT("syscall"), 1, true, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("syscall"), 1, true, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, {STR_LIT("x86_cpuid"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("x86_xgetbv"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, @@ -565,12 +565,12 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("__entry_point"), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, - {STR_LIT("objc_send"), 3, true, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("objc_send"), 3, true, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, {STR_LIT("objc_find_selector"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("objc_find_class"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("objc_register_selector"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, - {STR_LIT("objc_register_class"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("objc_register_selector"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, + {STR_LIT("objc_register_class"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics, false, true}, {STR_LIT("constant_utf16_cstring"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, diff --git a/src/parser.hpp b/src/parser.hpp index dc294b6ce..a648828fb 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -411,7 +411,6 @@ AST_KIND(_ExprBegin, "", bool) \ Token ellipsis; \ ProcInlining inlining; \ bool optional_ok_one; \ - i32 builtin_id; \ void *sce_temp_data; \ }) \ AST_KIND(FieldValue, "field value", struct { Token eq; Ast *field, *value; }) \ From 912d29af8321bb25d40916c2ec0b1709f6265a56 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 15:59:48 +0100 Subject: [PATCH 81/86] Add `@(require_results)` to all appropriate procedures --- core/simd/x86/abm.odin | 4 + core/simd/x86/adx.odin | 6 + core/simd/x86/pclmulqdq.odin | 2 +- core/simd/x86/rdtsc.odin | 2 + core/simd/x86/sha.odin | 14 +- core/simd/x86/sse.odin | 176 +++++++-------- core/simd/x86/sse2.odin | 406 +++++++++++++++++------------------ core/simd/x86/sse3.odin | 22 +- core/simd/x86/sse41.odin | 120 +++++------ core/simd/x86/ssse3.odin | 32 +-- 10 files changed, 398 insertions(+), 386 deletions(-) diff --git a/core/simd/x86/abm.odin b/core/simd/x86/abm.odin index f1898811f..5d7549ab3 100644 --- a/core/simd/x86/abm.odin +++ b/core/simd/x86/abm.odin @@ -3,17 +3,21 @@ package simd_x86 import "core:intrinsics" +@(require_results) _lzcnt_u32 :: #force_inline proc "c" (x: u32) -> u32 { return intrinsics.count_leading_zeros(x) } +@(require_results) _popcnt32 :: #force_inline proc "c" (x: u32) -> i32 { return i32(intrinsics.count_ones(x)) } when ODIN_ARCH == .amd64 { + @(require_results) _lzcnt_u64 :: #force_inline proc "c" (x: u64) -> u64 { return intrinsics.count_leading_zeros(x) } + @(require_results) _popcnt64 :: #force_inline proc "c" (x: u64) -> i32 { return i32(intrinsics.count_ones(x)) } diff --git a/core/simd/x86/adx.odin b/core/simd/x86/adx.odin index e73aa03a6..d03cffcff 100644 --- a/core/simd/x86/adx.odin +++ b/core/simd/x86/adx.odin @@ -1,14 +1,17 @@ //+build i386, amd64 package simd_x86 +@(require_results) _addcarry_u32 :: #force_inline proc "c" (c_in: u8, a: u32, b: u32, out: ^u32) -> u8 { x, y := llvm_addcarry_u32(c_in, a, b) out^ = y return x } +@(require_results) _addcarryx_u32 :: #force_inline proc "c" (c_in: u8, a: u32, b: u32, out: ^u32) -> u8 { return llvm_addcarryx_u32(c_in, a, b, out) } +@(require_results) _subborrow_u32 :: #force_inline proc "c" (c_in: u8, a: u32, b: u32, out: ^u32) -> u8 { x, y := llvm_subborrow_u32(c_in, a, b) out^ = y @@ -16,14 +19,17 @@ _subborrow_u32 :: #force_inline proc "c" (c_in: u8, a: u32, b: u32, out: ^u32) - } when ODIN_ARCH == .amd64 { + @(require_results) _addcarry_u64 :: #force_inline proc "c" (c_in: u8, a: u64, b: u64, out: ^u64) -> u8 { x, y := llvm_addcarry_u64(c_in, a, b) out^ = y return x } + @(require_results) _addcarryx_u64 :: #force_inline proc "c" (c_in: u8, a: u64, b: u64, out: ^u64) -> u8 { return llvm_addcarryx_u64(c_in, a, b, out) } + @(require_results) _subborrow_u64 :: #force_inline proc "c" (c_in: u8, a: u64, b: u64, out: ^u64) -> u8 { x, y := llvm_subborrow_u64(c_in, a, b) out^ = y diff --git a/core/simd/x86/pclmulqdq.odin b/core/simd/x86/pclmulqdq.odin index 8a665db03..692fb7ce1 100644 --- a/core/simd/x86/pclmulqdq.odin +++ b/core/simd/x86/pclmulqdq.odin @@ -1,7 +1,7 @@ //+build i386, amd64 package simd_x86 -@(enable_target_feature="pclmulqdq") +@(require_results, enable_target_feature="pclmulqdq") _mm_clmulepi64_si128 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u8) -> __m128i { return pclmulqdq(a, b, u8(IMM8)) } diff --git a/core/simd/x86/rdtsc.odin b/core/simd/x86/rdtsc.odin index 91dcc4ec9..54024c3f2 100644 --- a/core/simd/x86/rdtsc.odin +++ b/core/simd/x86/rdtsc.odin @@ -1,10 +1,12 @@ //+build i386, amd64 package simd_x86 +@(require_results) _rdtsc :: #force_inline proc "c" () -> u64 { return rdtsc() } +@(require_results) __rdtscp :: #force_inline proc "c" (aux: ^u32) -> u64 { return rdtscp(aux) } diff --git a/core/simd/x86/sha.odin b/core/simd/x86/sha.odin index 90f1d72ce..f015f4b8a 100644 --- a/core/simd/x86/sha.odin +++ b/core/simd/x86/sha.odin @@ -1,31 +1,31 @@ //+build i386, amd64 package simd_x86 -@(enable_target_feature="sha") +@(require_results, enable_target_feature="sha") _mm_sha1msg1_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha1msg1(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sha") +@(require_results, enable_target_feature="sha") _mm_sha1msg2_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha1msg2(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sha") +@(require_results, enable_target_feature="sha") _mm_sha1nexte_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha1nexte(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sha") +@(require_results, enable_target_feature="sha") _mm_sha1rnds4_epu32 :: #force_inline proc "c" (a, b: __m128i, $FUNC: u32) -> __m128i where 0 <= FUNC, FUNC <= 3 { return transmute(__m128i)sha1rnds4(transmute(i32x4)a, transmute(i32x4)b, u8(FUNC & 0xff)) } -@(enable_target_feature="sha") +@(require_results, enable_target_feature="sha") _mm_sha256msg1_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha256msg1(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sha") +@(require_results, enable_target_feature="sha") _mm_sha256msg2_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)sha256msg2(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sha") +@(require_results, enable_target_feature="sha") _mm_sha256rnds2_epu32 :: #force_inline proc "c" (a, b, k: __m128i) -> __m128i { return transmute(__m128i)sha256rnds2(transmute(i32x4)a, transmute(i32x4)b, transmute(i32x4)k) } diff --git a/core/simd/x86/sse.odin b/core/simd/x86/sse.odin index 6d8939b1b..3efdeccba 100644 --- a/core/simd/x86/sse.odin +++ b/core/simd/x86/sse.odin @@ -43,299 +43,299 @@ _MM_FLUSH_ZERO_ON :: 0x8000 _MM_FLUSH_ZERO_OFF :: 0x0000 -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_add_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return addss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_add_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.add(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_sub_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return subss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_sub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.sub(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_mul_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return mulss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_mul_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.mul(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_div_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return divss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_div_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.div(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_sqrt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return sqrtss(a) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_sqrt_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return sqrtps(a) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_rcp_ss :: #force_inline proc "c" (a: __m128) -> __m128 { return rcpss(a) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_rcp_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return rcpps(a) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_rsqrt_ss :: #force_inline proc "c" (a: __m128) -> __m128 { return rsqrtss(a) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_rsqrt_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return rsqrtps(a) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_min_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return minss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_min_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return minps(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_max_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return maxss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_max_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return maxps(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_and_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return transmute(__m128)simd.and(transmute(__m128i)a, transmute(__m128i)b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_andnot_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return transmute(__m128)simd.and_not(transmute(__m128i)a, transmute(__m128i)b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_or_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return transmute(__m128)simd.or(transmute(__m128i)a, transmute(__m128i)b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_xor_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return transmute(__m128)simd.xor(transmute(__m128i)a, transmute(__m128i)b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpeq_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 0) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmplt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 1) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmple_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 2) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpgt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, cmpss(b, a, 1), 4, 1, 2, 3) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpge_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, cmpss(b, a, 2), 4, 1, 2, 3) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpneq_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 4) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpnlt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 5) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpnle_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 6) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpngt_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, cmpss(b, a, 5), 4, 1, 2, 3) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpnge_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, cmpss(b, a, 6), 4, 1, 2, 3) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpord_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 7) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpunord_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpss(a, b, 3) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpeq_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 0) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmplt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 1) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmple_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 2) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpgt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 1) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpge_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 2) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpneq_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 4) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpnlt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 5) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpnle_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(a, b, 6) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpngt_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 5) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpnge_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 6) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpord_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 7) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cmpunord_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return cmpps(b, a, 3) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_comieq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comieq_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_comilt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comilt_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_comile_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comile_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_comigt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comigt_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_comige_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comige_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_comineq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return comineq_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_ucomieq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomieq_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_ucomilt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomilt_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_ucomile_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomile_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_ucomigt_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomigt_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_ucomige_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomige_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_ucomineq_ss :: #force_inline proc "c" (a, b: __m128) -> b32 { return ucomineq_ss(a, b) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cvtss_si32 :: #force_inline proc "c" (a: __m128) -> i32 { return cvtss2si(a) } _mm_cvt_ss2si :: _mm_cvtss_si32 _mm_cvttss_si32 :: _mm_cvtss_si32 -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cvtss_f32 :: #force_inline proc "c" (a: __m128) -> f32 { return simd.extract(a, 0) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_cvtsi32_ss :: #force_inline proc "c" (a: __m128, b: i32) -> __m128 { return cvtsi2ss(a, b) } _mm_cvt_si2ss :: _mm_cvtsi32_ss -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_set_ss :: #force_inline proc "c" (a: f32) -> __m128 { return __m128{a, 0, 0, 0} } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_set1_ps :: #force_inline proc "c" (a: f32) -> __m128 { return __m128(a) } _mm_set_ps1 :: _mm_set1_ps -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_set_ps :: #force_inline proc "c" (a, b, c, d: f32) -> __m128 { return __m128{d, c, b, a} } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_setr_ps :: #force_inline proc "c" (a, b, c, d: f32) -> __m128 { return __m128{a, b, c, d} } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_setzero_ps :: #force_inline proc "c" () -> __m128 { return __m128{0, 0, 0, 0} } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_shuffle_ps :: #force_inline proc "c" (a, b: __m128, $MASK: u32) -> __m128 { return simd.shuffle( a, b, @@ -346,58 +346,58 @@ _mm_shuffle_ps :: #force_inline proc "c" (a, b: __m128, $MASK: u32) -> __m128 { } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_unpackhi_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 2, 6, 3, 7) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_unpacklo_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 0, 4, 1, 5) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_movehl_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 6, 7, 2, 3) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_movelh_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 0, 1, 4, 5) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_movemask_ps :: #force_inline proc "c" (a: __m128) -> u32 { return movmskps(a) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_load_ss :: #force_inline proc "c" (p: ^f32) -> __m128 { return __m128{p^, 0, 0, 0} } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_load1_ps :: #force_inline proc "c" (p: ^f32) -> __m128 { a := p^ return __m128(a) } _mm_load_ps1 :: _mm_load1_ps -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_load_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { return (^__m128)(p)^ } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_loadu_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { dst := _mm_undefined_ps() intrinsics.mem_copy_non_overlapping(&dst, p, size_of(__m128)) return dst } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_loadr_ps :: #force_inline proc "c" (p: [^]f32) -> __m128 { return simd.lanes_reverse(_mm_load_ps(p)) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_loadu_si64 :: #force_inline proc "c" (mem_addr: rawptr) -> __m128i { a := intrinsics.unaligned_load((^i64)(mem_addr)) return __m128i{a, 0} @@ -431,7 +431,7 @@ _mm_storer_ps :: #force_inline proc "c" (p: [^]f32, a: __m128) { } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_move_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return simd.shuffle(a, b, 4, 1, 2, 3) } @@ -441,7 +441,7 @@ _mm_sfence :: #force_inline proc "c" () { sfence() } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_getcsr :: #force_inline proc "c" () -> (result: u32) { stmxcsr(&result) return result @@ -453,19 +453,19 @@ _mm_setcsr :: #force_inline proc "c" (val: u32) { ldmxcsr(&val) } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _MM_GET_EXCEPTION_MASK :: #force_inline proc "c" () -> u32 { return _mm_getcsr() & _MM_MASK_MASK } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _MM_GET_EXCEPTION_STATE :: #force_inline proc "c" () -> u32 { return _mm_getcsr() & _MM_EXCEPT_MASK } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _MM_GET_FLUSH_ZERO_MODE :: #force_inline proc "c" () -> u32 { return _mm_getcsr() & _MM_FLUSH_ZERO_MASK } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _MM_GET_ROUNDING_MODE :: #force_inline proc "c" () -> u32 { return _mm_getcsr() & _MM_ROUND_MASK } @@ -493,7 +493,7 @@ _mm_prefetch :: #force_inline proc "c" (p: rawptr, $STRATEGY: u32) { } -@(enable_target_feature="sse") +@(require_results, enable_target_feature="sse") _mm_undefined_ps :: #force_inline proc "c" () -> __m128 { return _mm_set1_ps(0) } @@ -517,15 +517,15 @@ _mm_stream_ps :: #force_inline proc "c" (addr: [^]f32, a: __m128) { } when ODIN_ARCH == .amd64 { - @(enable_target_feature="sse") + @(require_results, enable_target_feature="sse") _mm_cvtss_si64 :: #force_inline proc "c"(a: __m128) -> i64 { return cvtss2si64(a) } - @(enable_target_feature="sse") + @(require_results, enable_target_feature="sse") _mm_cvttss_si64 :: #force_inline proc "c"(a: __m128) -> i64 { return cvttss2si64(a) } - @(enable_target_feature="sse") + @(require_results, enable_target_feature="sse") _mm_cvtsi64_ss :: #force_inline proc "c"(a: __m128, b: i64) -> __m128 { return cvtsi642ss(a, b) } diff --git a/core/simd/x86/sse2.odin b/core/simd/x86/sse2.odin index d15df8120..f33bd2195 100644 --- a/core/simd/x86/sse2.odin +++ b/core/simd/x86/sse2.odin @@ -21,118 +21,118 @@ _mm_mfence :: #force_inline proc "c" () { mfence() } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_add_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_add_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_add_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_add_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add(transmute(i64x2)a, transmute(i64x2)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_adds_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add_sat(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_adds_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add_sat(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_adds_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add_sat(transmute(u8x16)a, transmute(u8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_adds_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.add_sat(transmute(u16x8)a, transmute(u16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_avg_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pavgb(transmute(u8x16)a, transmute(u8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_avg_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pavgw(transmute(u16x8)a, transmute(u16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_madd_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaddwd(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_max_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaxsw(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_max_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaxub(transmute(u8x16)a, transmute(u8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_min_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pminsw(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_min_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pminub(transmute(u8x16)a, transmute(u8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_mulhi_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmulhw(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_mulhi_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmulhuw(transmute(u16x8)a, transmute(u16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_mullo_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.mul(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_mul_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmuludq(transmute(u32x4)a, transmute(u32x4)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sad_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psadbw(transmute(u8x16)a, transmute(u8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sub_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sub_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sub_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sub_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub(transmute(i64x2)a, transmute(i64x2)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_subs_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_subs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_subs_epu8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(u8x16)a, transmute(u8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_subs_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.sub_sat(transmute(u16x8)a, transmute(u16x8)b) } @@ -140,7 +140,7 @@ _mm_subs_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { @(private) -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_slli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { shift :: IMM8 & 0xff @@ -167,7 +167,7 @@ _mm_slli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128 } @(private) -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { shift :: IMM8 return transmute(__m128i)simd.shuffle( @@ -193,233 +193,233 @@ _mm_srli_si128_impl :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128 } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_slli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_slli_si128_impl(a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_bslli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_slli_si128_impl(a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_bsrli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_srli_si128_impl(a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_slli_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)pslliw(transmute(i16x8)a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sll_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psllw(transmute(i16x8)a, transmute(i16x8)count) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_slli_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psllid(transmute(i32x4)a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sll_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)pslld(transmute(i32x4)a, transmute(i32x4)count) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_slli_epi64 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)pslliq(transmute(i64x2)a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sll_epi64 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psllq(transmute(i64x2)a, transmute(i64x2)count) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srai_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psraiw(transmute(i16x8)a. IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sra_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psraw(transmute(i16x8)a, transmute(i16x8)count) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srai_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psraid(transmute(i32x4)a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sra_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrad(transmute(i32x4)a, transmute(i32x4)count) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srli_si128 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return _mm_srli_si128_impl(a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srli_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrliw(transmute(i16x8)a. IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srl_epi16 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrlw(transmute(i16x8)a, transmute(i16x8)count) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srli_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrlid(transmute(i32x4)a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srl_epi32 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrld(transmute(i32x4)a, transmute(i32x4)count) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srli_epi64 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { return transmute(__m128i)psrliq(transmute(i64x2)a, IMM8) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_srl_epi64 :: #force_inline proc "c" (a, count: __m128i) -> __m128i { return transmute(__m128i)psrlq(transmute(i64x2)a, transmute(i64x2)count) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_and_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return simd.and(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_andnot_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return simd.and_not(b, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_or_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return simd.or(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_xor_si128 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return simd.xor(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpeq_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_eq(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpeq_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_eq(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpeq_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_eq(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpgt_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_gt(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpgt_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_gt(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpgt_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_gt(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmplt_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_lt(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmplt_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_lt(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmplt_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_lt(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtepi32_pd :: #force_inline proc "c" (a: __m128i) -> __m128d { v := transmute(i32x4)a return cast(__m128d)simd.shuffle(v, v, 0, 1) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtsi32_sd :: #force_inline proc "c" (a: __m128d, b: i32) -> __m128d { return simd.replace(a, 0, f64(b)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtepi32_ps :: #force_inline proc "c" (a: __m128i) -> __m128 { return cvtdq2ps(transmute(i32x4)a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtps_epi32 :: #force_inline proc "c" (a: __m128) -> __m128i { return transmute(__m128i)cvtps2dq(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtsi32_si128 :: #force_inline proc "c" (a: i32) -> __m128i { return transmute(__m128i)i32x4{a, 0, 0, 0} } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtsi128_si32 :: #force_inline proc "c" (a: __m128i) -> i32 { return simd.extract(transmute(i32x4)a, 0) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set_epi64x :: #force_inline proc "c" (e1, e0: i64) -> __m128i { return transmute(__m128i)i64x2{e0, e1} } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set_epi32 :: #force_inline proc "c" (e3, e2, e1, e0: i32) -> __m128i { return transmute(__m128i)i32x4{e0, e1, e2, e3} } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set_epi16 :: #force_inline proc "c" (e7, e6, e5, e4, e3, e2, e1, e0: i16) -> __m128i { return transmute(__m128i)i16x8{e0, e1, e2, e3, e4, e5, e6, e7} } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set_epi8 :: #force_inline proc "c" (e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0: i8) -> __m128i { return transmute(__m128i)i8x16{e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15} } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set1_epi64x :: #force_inline proc "c" (a: i64) -> __m128i { return _mm_set_epi64x(a, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set1_epi32 :: #force_inline proc "c" (a: i32) -> __m128i { return _mm_set_epi32(a, a, a, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set1_epi16 :: #force_inline proc "c" (a: i16) -> __m128i { return _mm_set_epi16(a, a, a, a, a, a, a, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set1_epi8 :: #force_inline proc "c" (a: i8) -> __m128i { return _mm_set_epi8(a, a, a, a, a, a, a, a, a, a, a, a, a, a, a, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_setr_epi32 :: #force_inline proc "c" (e3, e2, e1, e0: i32) -> __m128i { return _mm_set_epi32(e0, e1, e2, e3) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_setr_epi16 :: #force_inline proc "c" (e7, e6, e5, e4, e3, e2, e1, e0: i16) -> __m128i { return _mm_set_epi16(e0, e1, e2, e3, e4, e5, e6, e7) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_setr_epi8 :: #force_inline proc "c" (e15, e14, e13, e12, e11, e10, e9, e8, e7, e6, e5, e4, e3, e2, e1, e0: i8) -> __m128i { return _mm_set_epi8(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_setzero_si128 :: #force_inline proc "c" () -> __m128i { return _mm_set1_epi64x(0) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_loadl_epi64 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { return _mm_set_epi64x(0, intrinsics.unaligned_load((^i64)(mem_addr))) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_load_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { return mem_addr^ } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_loadu_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { dst := _mm_undefined_si128() intrinsics.mem_copy_non_overlapping(&dst, mem_addr, size_of(__m128i)) @@ -450,7 +450,7 @@ _mm_stream_si128 :: #force_inline proc "c" (mem_addr: ^__m128i, a: __m128i) { _mm_stream_si32 :: #force_inline proc "c" (mem_addr: ^i32, a: i32) { intrinsics.non_temporal_store(mem_addr, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_move_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { zero := _mm_setzero_si128() return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)zero, 0, 2) @@ -459,31 +459,31 @@ _mm_move_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_packs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)packsswb(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_packs_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)packssdw(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_packus_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)packuswb(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_extract_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> i32 { return i32(simd.extract(transmute(u16x8)a, IMM8)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_insert_epi16 :: #force_inline proc "c" (a: __m128i, i: i32, $IMM8: u32) -> __m128i { return i32(simd.replace(transmute(u16x8)a, IMM8, i16(i))) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_movemask_epi8 :: #force_inline proc "c" (a: __m128i) -> i32 { return pmovmskb(transmute(i8x16)a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_shuffle_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { v := transmute(i32x4)a return transmute(__m128i)simd.shuffle( @@ -495,7 +495,7 @@ _mm_shuffle_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i (IMM8 >> 6) & 0b11, ) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_shufflehi_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { v := transmute(i16x8)a return transmute(__m128i)simd.shuffle( @@ -511,7 +511,7 @@ _mm_shufflehi_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128 ((IMM8 >> 6) & 0b11) + 4, ) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_shufflelo_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128i { v := transmute(i16x8)a return transmute(__m128i)simd.shuffle( @@ -527,7 +527,7 @@ _mm_shufflelo_epi16 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> __m128 7, ) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpackhi_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle( transmute(i8x16)a, @@ -535,19 +535,19 @@ _mm_unpackhi_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31, ) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpackhi_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i16x8)a, transmute(i16x8)b, 4, 12, 5, 13, 6, 14, 7, 15) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpackhi_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i32x4)a, transmute(i32x4)b, 2, 6, 3, 7) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpackhi_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)b, 1, 3) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpacklo_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle( transmute(i8x16)a, @@ -555,15 +555,15 @@ _mm_unpacklo_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23, ) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpacklo_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i16x8)a, transmute(i16x8)b, 0, 8, 1, 9, 2, 10, 3, 11) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpacklo_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i32x4)a, transmute(i32x4)b, 0, 4, 1, 5) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpacklo_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.shuffle(transmute(i64x2)a, transmute(i64x2)b, 0, 2) } @@ -571,75 +571,75 @@ _mm_unpacklo_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_add_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(a) + _mm_cvtsd_f64(b)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_add_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.add(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_div_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(a) / _mm_cvtsd_f64(b)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_div_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.div(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_max_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return maxsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_max_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return maxpd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_min_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return minsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_min_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return minpd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_mul_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(a) * _mm_cvtsd_f64(b)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_mul_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.mul(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sqrt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(sqrtsd(b))) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sqrt_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { return simd.sqrt(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sub_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(a, 0, _mm_cvtsd_f64(a) - _mm_cvtsd_f64(b)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_sub_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.sub(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_and_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return transmute(__m128d)_mm_and_si128(transmute(__m128i)a, transmute(__m128i)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_andnot_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return transmute(__m128d)_mm_andnot_si128(transmute(__m128i)a, transmute(__m128i)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_or_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return transmute(__m128d)_mm_or_si128(transmute(__m128i)a, transmute(__m128i)b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_xor_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return transmute(__m128d)_mm_xor_si128(transmute(__m128i)a, transmute(__m128i)b) } @@ -647,147 +647,147 @@ _mm_xor_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpeq_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 0) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmplt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 1) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmple_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 2) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpgt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(_mm_cmplt_sd(b, a), 1, simd.extract(a, 1)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpge_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(_mm_cmple_sd(b, a), 1, simd.extract(a, 1)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpord_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 7) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpunord_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 3) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpneq_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 4) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpnlt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 5) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpnle_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmpsd(a, b, 6) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpngt_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(_mm_cmpnlt_sd(b, a), 1, simd.extract(a, 1)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpnge_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.replace(_mm_cmpnle_sd(b, a), 1, simd.extract(a, 1)) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpeq_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 0) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmplt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 1) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmple_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 2) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpgt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_cmplt_pd(b, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpge_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_cmple_pd(b, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpord_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 7) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpunord_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 3) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpneq_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 4) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpnlt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 5) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpnle_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return cmppd(a, b, 6) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpngt_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_cmpnlt_pd(b, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cmpnge_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_cmpnle_pd(b, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_comieq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comieqsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_comilt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comiltsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_comile_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comilesd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_comigt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comigtsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_comige_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comigesd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_comineq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return comineqsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_ucomieq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomieqsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_ucomilt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomiltsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_ucomile_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomilesd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_ucomigt_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomigtsd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_ucomige_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomigesd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_ucomineq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { return ucomineqsd(a, b) } @@ -796,87 +796,87 @@ _mm_ucomineq_sd :: #force_inline proc "c" (a, b: __m128d) -> i32 { -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { return cvtpd2ps(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtps_pd :: #force_inline proc "c" (a: __m128) -> __m128d { return cvtps2pd(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtpd_epi32 :: #force_inline proc "c" (a: __m128d) -> __m128i { return transmute(__m128i)cvtpd2dq(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtsd_si32 :: #force_inline proc "c" (a: __m128d) -> i32 { return cvtsd2si(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtsd_ss :: #force_inline proc "c" (a, b: __m128d) -> __m128 { return cvtsd2ss(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtsd_f64 :: #force_inline proc "c" (a: __m128d) -> f64 { return simd.extract(a, 0) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvtss_sd :: #force_inline proc "c" (a, b: __m128) -> __m128d { return cvtss2sd(a, b) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvttpd_epi32 :: #force_inline proc "c" (a: __m128d) -> __m128i { return transmute(__m128i)cvttpd2dq(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvttsd_si32 :: #force_inline proc "c" (a: __m128d) -> i32 { return cvttsd2si(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_cvttps_epi32 :: #force_inline proc "c" (a: __m128) -> __m128i { return transmute(__m128i)cvttps2dq(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set_sd :: #force_inline proc "c" (a: f64) -> __m128d { return _mm_set_pd(0.0, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set1_pd :: #force_inline proc "c" (a: f64) -> __m128d { return _mm_set_pd(a, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set_pd1 :: #force_inline proc "c" (a: f64) -> __m128d { return _mm_set_pd(a, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_set_pd :: #force_inline proc "c" (a: f64, b: f64) -> __m128d { return __m128d{b, a} } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_setr_pd :: #force_inline proc "c" (a: f64, b: f64) -> __m128d { return _mm_set_pd(b, a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_setzero_pd :: #force_inline proc "c" () -> __m128d { return _mm_set_pd(0.0, 0.0) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_movemask_pd :: #force_inline proc "c" (a: __m128d) -> i32 { return movmskpd(a) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_load_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { return (^__m128d)(mem_addr)^ } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_load_sd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { return _mm_setr_pd(mem_addr^, 0.) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_loadh_pd :: #force_inline proc "c" (a: __m128d, mem_addr: ^f64) -> __m128d { return _mm_setr_pd(simd.extract(a, 0), mem_addr^) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_loadl_pd :: #force_inline proc "c" (a: __m128d, mem_addr: ^f64) -> __m128d { return _mm_setr_pd(mem_addr^, simd.extract(a, 1)) } @@ -916,31 +916,31 @@ _mm_storeh_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { _mm_storel_pd :: #force_inline proc "c" (mem_addr: ^f64, a: __m128d) { mem_addr^ = simd.extract(a, 0) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_load1_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { d := mem_addr^ return _mm_setr_pd(d, d) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_load_pd1 :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { return _mm_load1_pd(mem_addr) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_loadr_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { a := _mm_load_pd(mem_addr) return simd.shuffle(a, a, 1, 0) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_loadu_pd :: #force_inline proc "c" (mem_addr: ^f64) -> __m128d { dst := _mm_undefined_pd() intrinsics.mem_copy_non_overlapping(&dst, mem_addr, size_of(__m128d)) return dst } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_shuffle_pd :: #force_inline proc "c" (a, b: __m128d, $MASK: u32) -> __m128d { return simd.shuffle(a, b, MASK&0b1, ((MASK>>1)&0b1) + 2) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_move_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return _mm_setr_pd(simd.extract(b, 0), simd.extract(a, 1)) } @@ -948,64 +948,64 @@ _mm_move_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_castpd_ps :: #force_inline proc "c" (a: __m128d) -> __m128 { return transmute(__m128)a } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_castpd_si128 :: #force_inline proc "c" (a: __m128d) -> __m128i { return transmute(__m128i)a } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_castps_pd :: #force_inline proc "c" (a: __m128) -> __m128d { return transmute(__m128d)a } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_castps_si128 :: #force_inline proc "c" (a: __m128) -> __m128i { return transmute(__m128i)a } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_castsi128_pd :: #force_inline proc "c" (a: __m128i) -> __m128d { return transmute(__m128d)a } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_castsi128_ps :: #force_inline proc "c" (a: __m128i) -> __m128 { return transmute(__m128)a } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_undefined_pd :: #force_inline proc "c" () -> __m128d { return __m128d{0, 0} } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_undefined_si128 :: #force_inline proc "c" () -> __m128i { return __m128i{0, 0} } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpackhi_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.shuffle(a, b, 1, 3) } -@(enable_target_feature="sse2") +@(require_results, enable_target_feature="sse2") _mm_unpacklo_pd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return simd.shuffle(a, b, 0, 2) } when ODIN_ARCH == .amd64 { - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvtsd_si64 :: #force_inline proc "c" (a: __m128d) -> i64 { return cvtsd2si64(a) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvtsd_si64x :: #force_inline proc "c" (a: __m128d) -> i64 { return _mm_cvtsd_si64(a) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvttsd_si64 :: #force_inline proc "c" (a: __m128d) -> i64 { return cvttsd2si64(a) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvttsd_si64x :: #force_inline proc "c" (a: __m128d) -> i64 { return _mm_cvttsd_si64(a) } @@ -1013,27 +1013,27 @@ when ODIN_ARCH == .amd64 { _mm_stream_si64 :: #force_inline proc "c" (mem_addr: ^i64, a: i64) { intrinsics.non_temporal_store(mem_addr, a) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvtsi64_si128 :: #force_inline proc "c" (a: i64) -> __m128i { return _mm_set_epi64x(0, a) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvtsi64x_si128 :: #force_inline proc "c" (a: i64) -> __m128i { return _mm_cvtsi64_si128(a) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvtsi128_si64 :: #force_inline proc "c" (a: __m128i) -> i64 { return simd.extract(transmute(i64x2)a, 0) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvtsi128_si64x :: #force_inline proc "c" (a: __m128i) -> i64 { return _mm_cvtsi128_si64(a) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvtsi64_sd :: #force_inline proc "c" (a: __m128d, b: i64) -> __m128d { return simd.replace(a, 0, f64(b)) } - @(enable_target_feature="sse2") + @(require_results, enable_target_feature="sse2") _mm_cvtsi64x_sd :: #force_inline proc "c" (a: __m128d, b: i64) -> __m128d { return _mm_cvtsi64_sd(a, b) } diff --git a/core/simd/x86/sse3.odin b/core/simd/x86/sse3.odin index 370bfa952..7a3073c18 100644 --- a/core/simd/x86/sse3.odin +++ b/core/simd/x86/sse3.odin @@ -4,47 +4,47 @@ package simd_x86 import "core:intrinsics" import "core:simd" -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_addsub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return addsubps(a, b) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_addsub_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { return addsubpd(a, b) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_hadd_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { return haddpd(a, b) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_hadd_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return haddps(a, b) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_hsub_pd :: #force_inline proc "c" (a: __m128d, b: __m128d) -> __m128d { return hsubpd(a, b) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_hsub_ps :: #force_inline proc "c" (a, b: __m128) -> __m128 { return hsubps(a, b) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_lddqu_si128 :: #force_inline proc "c" (mem_addr: ^__m128i) -> __m128i { return transmute(__m128i)lddqu(mem_addr) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_movedup_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { return simd.shuffle(a, a, 0, 0) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_loaddup_pd :: #force_inline proc "c" (mem_addr: [^]f64) -> __m128d { return _mm_load1_pd(mem_addr) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_movehdup_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return simd.shuffle(a, a, 1, 1, 3, 3) } -@(enable_target_feature="sse3") +@(require_results, enable_target_feature="sse3") _mm_moveldup_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return simd.shuffle(a, a, 0, 0, 2, 2) } diff --git a/core/simd/x86/sse41.odin b/core/simd/x86/sse41.odin index 516e0bdc2..b35be33f2 100644 --- a/core/simd/x86/sse41.odin +++ b/core/simd/x86/sse41.odin @@ -20,271 +20,271 @@ _MM_FROUND_NEARBYINT :: _MM_FROUND_NO_EXC | _MM_FROUND_CUR_DIRECTION -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_blendv_epi8 :: #force_inline proc "c" (a, b, mask: __m128i) -> __m128i { return transmute(__m128i)pblendvb(transmute(i8x16)a, transmute(i8x16)b, transmute(i8x16)mask) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_blend_epi16 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u8) -> __m128i { return transmute(__m128i)pblendw(transmute(i16x8)a, transmute(i16x8)b, IMM8) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_blendv_pd :: #force_inline proc "c" (a, b, mask: __m128d) -> __m128d { return blendvpd(a, b, mask) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_blendv_ps :: #force_inline proc "c" (a, b, mask: __m128) -> __m128 { return blendvps(a, b, mask) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_blend_pd :: #force_inline proc "c" (a, b: __m128d, $IMM2: u8) -> __m128d { return blendpd(a, b, IMM2) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_blend_ps :: #force_inline proc "c" (a, b: __m128, $IMM4: u8) -> __m128 { return blendps(a, b, IMM4) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_extract_ps :: #force_inline proc "c" (a: __m128, $IMM8: u32) -> i32 { return transmute(i32)simd.extract(a, IMM8) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_extract_epi8 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> i32 { return i32(simd.extract(transmute(u8x16)a, IMM8)) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_extract_epi32 :: #force_inline proc "c" (a: __m128i, $IMM8: u32) -> i32 { return simd.extract(transmute(i32x4)a, IMM8) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_insert_ps :: #force_inline proc "c" (a, b: __m128, $IMM8: u8) -> __m128 { return insertps(a, b, IMM8) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_insert_epi8 :: #force_inline proc "c" (a: __m128i, i: i32, $IMM8: u32) -> __m128i { return transmute(__m128i)simd.replace(transmute(i8x16)a, IMM8, i8(i)) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_insert_epi32 :: #force_inline proc "c" (a: __m128i, i: i32, $IMM8: u32) -> __m128i { return transmute(__m128i)simd.replace(transmute(i32x4)a, IMM8, i) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_max_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaxsb(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_max_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaxuw(transmute(u16x8)a, transmute(u16x8)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_max_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaxsd(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_max_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaxud(transmute(u32x4)a, transmute(u32x4)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_min_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pminsb(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_min_epu16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pminuw(transmute(u16x8)a, transmute(u16x8)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_min_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pminsd(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_min_epu32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pminud(transmute(u32x4)a, transmute(u32x4)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_packus_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)packusdw(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cmpeq_epi64 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.lanes_eq(transmute(i64x2)a, transmute(i64x2)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepi8_epi16 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(i8x16)a y := simd.shuffle(x, x, 0, 1, 2, 3, 4, 5, 6, 7) return transmute(__m128i)i16x8(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepi8_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(i8x16)a y := simd.shuffle(x, x, 0, 1, 2, 3) return transmute(__m128i)i32x4(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepi8_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(i8x16)a y := simd.shuffle(x, x, 0, 1) return transmute(__m128i)i64x2(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepi16_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(i16x8)a y := simd.shuffle(x, x, 0, 1, 2, 3) return transmute(__m128i)i32x4(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepi16_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(i16x8)a y := simd.shuffle(x, x, 0, 1) return transmute(__m128i)i64x2(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepi32_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(i32x4)a y := simd.shuffle(x, x, 0, 1) return transmute(__m128i)i64x2(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepu8_epi16 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(u8x16)a y := simd.shuffle(x, x, 0, 1, 2, 3, 4, 5, 6, 7) return transmute(__m128i)i16x8(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepu8_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(u8x16)a y := simd.shuffle(x, x, 0, 1, 2, 3) return transmute(__m128i)i32x4(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepu8_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(u8x16)a y := simd.shuffle(x, x, 0, 1) return transmute(__m128i)i64x2(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepu16_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(u16x8)a y := simd.shuffle(x, x, 0, 1, 2, 3) return transmute(__m128i)i32x4(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepu16_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(u16x8)a y := simd.shuffle(x, x, 0, 1) return transmute(__m128i)i64x2(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_cvtepu32_epi64 :: #force_inline proc "c" (a: __m128i) -> __m128i { x := transmute(u32x4)a y := simd.shuffle(x, x, 0, 1) return transmute(__m128i)i64x2(y) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_dp_pd :: #force_inline proc "c" (a, b: __m128d, $IMM8: u8) -> __m128d { return dppd(a, b, IMM8) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_dp_ps :: #force_inline proc "c" (a, b: __m128, $IMM8: u8) -> __m128 { return dpps(a, b, IMM8) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_floor_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { return simd.floor(a) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_floor_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return simd.floor(a) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_floor_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return roundsd(a, b, _MM_FROUND_FLOOR) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_floor_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return roundss(a, b, _MM_FROUND_FLOOR) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_ceil_pd :: #force_inline proc "c" (a: __m128d) -> __m128d { return simd.ceil(a) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_ceil_ps :: #force_inline proc "c" (a: __m128) -> __m128 { return simd.ceil(a) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_ceil_sd :: #force_inline proc "c" (a, b: __m128d) -> __m128d { return roundsd(a, b, _MM_FROUND_CEIL) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_ceil_ss :: #force_inline proc "c" (a, b: __m128) -> __m128 { return roundss(a, b, _MM_FROUND_CEIL) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_round_pd :: #force_inline proc "c" (a: __m128d, $ROUNDING: i32) -> __m128d { return roundpd(a, ROUNDING) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_round_ps :: #force_inline proc "c" (a: __m128, $ROUNDING: i32) -> __m128 { return roundps(a, ROUNDING) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_round_sd :: #force_inline proc "c" (a, b: __m128d, $ROUNDING: i32) -> __m128d { return roundsd(a, b, ROUNDING) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_round_ss :: #force_inline proc "c" (a, b: __m128, $ROUNDING: i32) -> __m128 { return roundss(a, b, ROUNDING) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_minpos_epu16 :: #force_inline proc "c" (a: __m128i) -> __m128i { return transmute(__m128i)phminposuw(transmute(u16x8)a) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_mul_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmuldq(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_mullo_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)simd.mul(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_mpsadbw_epu8 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u8) -> __m128i { return transmute(__m128i)mpsadbw(transmute(u8x16)a, transmute(u8x16)b, IMM8) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_testz_si128 :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { return ptestz(transmute(i64x2)a, transmute(i64x2)mask) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_testc_si128 :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { return ptestc(transmute(i64x2)a, transmute(i64x2)mask) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_testnzc_si128 :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { return ptestnzc(transmute(i64x2)a, transmute(i64x2)mask) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_test_all_zeros :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { return _mm_testz_si128(a, mask) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_test_all_ones :: #force_inline proc "c" (a: __m128i) -> i32 { return _mm_testc_si128(a, _mm_cmpeq_epi32(a, a)) } -@(enable_target_feature="sse4.1") +@(require_results, enable_target_feature="sse4.1") _mm_test_mix_ones_zeros :: #force_inline proc "c" (a: __m128i, mask: __m128i) -> i32 { return _mm_testnzc_si128(a, mask) } when ODIN_ARCH == .amd64 { - @(enable_target_feature="sse4.1") + @(require_results, enable_target_feature="sse4.1") _mm_extract_epi64 :: #force_inline proc "c" (a: __m128i, $IMM1: u32) -> i64 { return simd.extract(transmute(i64x2)a, IMM1) } - @(enable_target_feature="sse4.1") + @(require_results, enable_target_feature="sse4.1") _mm_insert_epi64 :: #force_inline proc "c" (a: __m128i, i: i64, $IMM1: u32) -> __m128i { return transmute(__m128i)simd.replace(transmute(i64x2)a, IMM1, i) } diff --git a/core/simd/x86/ssse3.odin b/core/simd/x86/ssse3.odin index 8c677aed4..f11ef6774 100644 --- a/core/simd/x86/ssse3.odin +++ b/core/simd/x86/ssse3.odin @@ -5,23 +5,23 @@ import "core:intrinsics" import "core:simd" _ :: simd -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_abs_epi8 :: #force_inline proc "c" (a: __m128i) -> __m128i { return transmute(__m128i)pabsb128(transmute(i8x16)a) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_abs_epi16 :: #force_inline proc "c" (a: __m128i) -> __m128i { return transmute(__m128i)pabsw128(transmute(i16x8)a) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_abs_epi32 :: #force_inline proc "c" (a: __m128i) -> __m128i { return transmute(__m128i)pabsd128(transmute(i32x4)a) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_shuffle_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pshufb128(transmute(u8x16)a, transmute(u8x16)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_alignr_epi8 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u32) -> __m128i { shift :: IMM8 @@ -58,47 +58,47 @@ _mm_alignr_epi8 :: #force_inline proc "c" (a, b: __m128i, $IMM8: u32) -> __m128i } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_hadd_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phaddw128(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_hadds_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phaddsw128(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_hadd_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phaddd128(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_hsub_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phsubw128(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_hsubs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phsubsw128(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_hsub_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)phsubd128(transmute(i32x4)a, transmute(i32x4)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_maddubs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmaddubsw128(transmute(u8x16)a, transmute(i8x16)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_mulhrs_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)pmulhrsw128(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_sign_epi8 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psignb128(transmute(i8x16)a, transmute(i8x16)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_sign_epi16 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psignw128(transmute(i16x8)a, transmute(i16x8)b) } -@(enable_target_feature="ssse3") +@(require_results, enable_target_feature="ssse3") _mm_sign_epi32 :: #force_inline proc "c" (a, b: __m128i) -> __m128i { return transmute(__m128i)psignd128(transmute(i32x4)a, transmute(i32x4)b) } From 68222cb8ab1dc4a8581677cfeb872690d885f3ea Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 16:06:31 +0100 Subject: [PATCH 82/86] Add SSE4.2 --- core/simd/x86/sse42.odin | 149 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 core/simd/x86/sse42.odin diff --git a/core/simd/x86/sse42.odin b/core/simd/x86/sse42.odin new file mode 100644 index 000000000..62b4f0478 --- /dev/null +++ b/core/simd/x86/sse42.odin @@ -0,0 +1,149 @@ +//+build i386, amd64 +package simd_x86 + +import "core:simd" + +_SIDD_UBYTE_OPS :: 0b0000_0000 +_SIDD_UWORD_OPS :: 0b0000_0001 +_SIDD_SBYTE_OPS :: 0b0000_0010 +_SIDD_SWORD_OPS :: 0b0000_0011 + +_SIDD_CMP_EQUAL_ANY :: 0b0000_0000 +_SIDD_CMP_RANGES :: 0b0000_0100 +_SIDD_CMP_EQUAL_EACH :: 0b0000_1000 +_SIDD_CMP_EQUAL_ORDERED :: 0b0000_1100 + +_SIDD_POSITIVE_POLARITY :: 0b0000_0000 +_SIDD_NEGATIVE_POLARITY :: 0b0001_0000 +_SIDD_MASKED_POSITIVE_POLARITY :: 0b0010_0000 +_SIDD_MASKED_NEGATIVE_POLARITY :: 0b0011_0000 + +_SIDD_LEAST_SIGNIFICANT :: 0b0000_0000 +_SIDD_MOST_SIGNIFICANT :: 0b0100_0000 + +_SIDD_BIT_MASK :: 0b0000_0000 +_SIDD_UNIT_MASK :: 0b0100_0000 + +@(require_results, enable_target_feature="sse4.2") +_mm_cmpistrm :: #force_inline proc "c" (a: __m128i, b: __m128i, $IMM8: i8) -> __m128i { + return transmute(__m128i)pcmpistrm128(transmute(i8x16)a, transmute(i8x16)b, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpistri :: #force_inline proc "c" (a: __m128i, b: __m128i, $IMM8: i8) -> i32 { + return pcmpistri128(transmute(i8x16)a, transmute(i8x16)b, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpistrz :: #force_inline proc "c" (a: __m128i, b: __m128i, $IMM8: i8) -> i32 { + return pcmpistriz128(transmute(i8x16)a, transmute(i8x16)b, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpistrc :: #force_inline proc "c" (a: __m128i, b: __m128i, $IMM8: i8) -> i32 { + return pcmpistric128(transmute(i8x16)a, transmute(i8x16)b, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpistrs :: #force_inline proc "c" (a: __m128i, b: __m128i, $IMM8: i8) -> i32 { + return pcmpistris128(transmute(i8x16)a, transmute(i8x16)b, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpistro :: #force_inline proc "c" (a: __m128i, b: __m128i, $IMM8: i8) -> i32 { + return pcmpistrio128(transmute(i8x16)a, transmute(i8x16)b, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpistra :: #force_inline proc "c" (a: __m128i, b: __m128i, $IMM8: i8) -> i32 { + return pcmpistria128(transmute(i8x16)a, transmute(i8x16)b, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpestrm :: #force_inline proc "c" (a: __m128i, la: i32, b: __m128i, lb: i32, $IMM8: i8) -> __m128i { + return transmute(__m128i)pcmpestrm128(transmute(i8x16)a, la, transmute(i8x16)b, lb, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpestri :: #force_inline proc "c" (a: __m128i, la: i32, b: __m128i, lb: i32, $IMM8: i8) -> i32 { + return pcmpestri128(transmute(i8x16)a, la, transmute(i8x16)b, lb, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpestrz :: #force_inline proc "c" (a: __m128i, la: i32, b: __m128i, lb: i32, $IMM8: i8) -> i32 { + return pcmpestriz128(transmute(i8x16)a, la, transmute(i8x16)b, lb, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpestrc :: #force_inline proc "c" (a: __m128i, la: i32, b: __m128i, lb: i32, $IMM8: i8) -> i32 { + return pcmpestric128(transmute(i8x16)a, la, transmute(i8x16)b, lb, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpestrs :: #force_inline proc "c" (a: __m128i, la: i32, b: __m128i, lb: i32, $IMM8: i8) -> i32 { + return pcmpestris128(transmute(i8x16)a, la, transmute(i8x16)b, lb, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpestro :: #force_inline proc "c" (a: __m128i, la: i32, b: __m128i, lb: i32, $IMM8: i8) -> i32 { + return pcmpestrio128(transmute(i8x16)a, la, transmute(i8x16)b, lb, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpestra :: #force_inline proc "c" (a: __m128i, la: i32, b: __m128i, lb: i32, $IMM8: i8) -> i32 { + return pcmpestria128(transmute(i8x16)a, la, transmute(i8x16)b, lb, IMM8) +} +@(require_results, enable_target_feature="sse4.2") +_mm_crc32_u8 :: #force_inline proc "c" (crc: u32, v: u8) -> u32 { + return crc32_32_8(crc, v) +} +@(require_results, enable_target_feature="sse4.2") +_mm_crc32_u16 :: #force_inline proc "c" (crc: u32, v: u16) -> u32 { + return crc32_32_16(crc, v) +} +@(require_results, enable_target_feature="sse4.2") +_mm_crc32_u32 :: #force_inline proc "c" (crc: u32, v: u32) -> u32 { + return crc32_32_32(crc, v) +} +@(require_results, enable_target_feature="sse4.2") +_mm_cmpgt_epi64 :: #force_inline proc "c" (a: __m128i, b: __m128i) -> __m128i { + return transmute(__m128i)simd.lanes_gt(transmute(i64x2)a, transmute(i64x2)b) +} + +when ODIN_ARCH == .amd64 { + @(require_results, enable_target_feature="sse4.2") + _mm_crc32_u64 :: #force_inline proc "c" (crc: u64, v: u64) -> u64 { + return crc32_64_64(crc, v) + } +} + +@(private, default_calling_convention="c") +foreign _ { + // SSE 4.2 string and text comparison ops + @(link_name="llvm.x86.sse42.pcmpestrm128") + pcmpestrm128 :: proc(a: i8x16, la: i32, b: i8x16, lb: i32, #const imm8: i8) -> u8x16 --- + @(link_name="llvm.x86.sse42.pcmpestri128") + pcmpestri128 :: proc(a: i8x16, la: i32, b: i8x16, lb: i32, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpestriz128") + pcmpestriz128 :: proc(a: i8x16, la: i32, b: i8x16, lb: i32, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpestric128") + pcmpestric128 :: proc(a: i8x16, la: i32, b: i8x16, lb: i32, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpestris128") + pcmpestris128 :: proc(a: i8x16, la: i32, b: i8x16, lb: i32, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpestrio128") + pcmpestrio128 :: proc(a: i8x16, la: i32, b: i8x16, lb: i32, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpestria128") + pcmpestria128 :: proc(a: i8x16, la: i32, b: i8x16, lb: i32, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpistrm128") + pcmpistrm128 :: proc(a, b: i8x16, #const imm8: i8) -> i8x16 --- + @(link_name="llvm.x86.sse42.pcmpistri128") + pcmpistri128 :: proc(a, b: i8x16, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpistriz128") + pcmpistriz128 :: proc(a, b: i8x16, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpistric128") + pcmpistric128 :: proc(a, b: i8x16, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpistris128") + pcmpistris128 :: proc(a, b: i8x16, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpistrio128") + pcmpistrio128 :: proc(a, b: i8x16, #const imm8: i8) -> i32 --- + @(link_name="llvm.x86.sse42.pcmpistria128") + pcmpistria128 :: proc(a, b: i8x16, #const imm8: i8) -> i32 --- + // SSE 4.2 CRC instructions + @(link_name="llvm.x86.sse42.crc32.32.8") + crc32_32_8 :: proc(crc: u32, v: u8) -> u32 --- + @(link_name="llvm.x86.sse42.crc32.32.16") + crc32_32_16 :: proc(crc: u32, v: u16) -> u32 --- + @(link_name="llvm.x86.sse42.crc32.32.32") + crc32_32_32 :: proc(crc: u32, v: u32) -> u32 --- + + // AMD64 Only + @(link_name="llvm.x86.sse42.crc32.64.64") + crc32_64_64 :: proc(crc: u64, v: u64) -> u64 --- +} From 4e49d24df980e3f4b853d62af01ed68cb2c622d2 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 16:08:06 +0100 Subject: [PATCH 83/86] Add enable_target_feature to ABM --- core/simd/x86/abm.odin | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/simd/x86/abm.odin b/core/simd/x86/abm.odin index 5d7549ab3..79b806242 100644 --- a/core/simd/x86/abm.odin +++ b/core/simd/x86/abm.odin @@ -3,21 +3,21 @@ package simd_x86 import "core:intrinsics" -@(require_results) +@(require_results, enable_target_feature="lzcnt") _lzcnt_u32 :: #force_inline proc "c" (x: u32) -> u32 { return intrinsics.count_leading_zeros(x) } -@(require_results) +@(require_results, enable_target_feature="popcnt") _popcnt32 :: #force_inline proc "c" (x: u32) -> i32 { return i32(intrinsics.count_ones(x)) } when ODIN_ARCH == .amd64 { - @(require_results) + @(require_results, enable_target_feature="lzcnt") _lzcnt_u64 :: #force_inline proc "c" (x: u64) -> u64 { return intrinsics.count_leading_zeros(x) } - @(require_results) + @(require_results, enable_target_feature="popcnt") _popcnt64 :: #force_inline proc "c" (x: u64) -> i32 { return i32(intrinsics.count_ones(x)) } From cb10af08cb612e7d73abe3174e3e4ce0aa162ecc Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 30 May 2022 16:42:32 +0100 Subject: [PATCH 84/86] Correct intrinsics.odin for documentation --- core/intrinsics/intrinsics.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 89f9a5f20..72effaca9 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -279,8 +279,8 @@ wasm_memory_atomic_wait32 :: proc(ptr: ^u32, expected: u32, timeout_ns: i64) - wasm_memory_atomic_notify32 :: proc(ptr: ^u32, waiters: u32) -> (waiters_woken_up: u32) --- // x86 Targets (i386, amd64) -cpuid :: proc(ax, cx: u32) -> (eax, ebc, ecx, edx: u32) --- -xgetbv :: proc(cx: u32) -> (eax, edx: u32) --- +x86_cpuid :: proc(ax, cx: u32) -> (eax, ebc, ecx, edx: u32) --- +x86_xgetbv :: proc(cx: u32) -> (eax, edx: u32) --- // Darwin targets only From a7840d50e2a1f6683607ab2b0dae895952eda103 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 31 May 2022 00:01:23 +0100 Subject: [PATCH 85/86] Correct documentation --- core/intrinsics/intrinsics.odin | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index 72effaca9..9994a1914 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -10,10 +10,10 @@ soa_struct :: proc($N: int, $T: typeid) -> type/#soa[N]T // Volatile volatile_load :: proc(dst: ^$T) -> T --- -volatile_store :: proc(dst: ^$T, val: T) -> T --- +volatile_store :: proc(dst: ^$T, val: T) --- -nontemporal_load :: proc(dst: ^$T) -> T --- -nontemporal_store :: proc(dst: ^$T, val: T) -> T --- +non_temporal_load :: proc(dst: ^$T) -> T --- +non_temporal_store :: proc(dst: ^$T, val: T) --- // Trapping debug_trap :: proc() --- From 516f6647b46c69a67139154c02c74b436cd4b999 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 31 May 2022 11:00:41 +0100 Subject: [PATCH 86/86] Fix intrinsics.non_temporal_{load, store} --- src/llvm_backend_proc.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index e3ffcaef2..75ca77641 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -2117,10 +2117,14 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, LLVMValueRef instr = LLVMBuildStore(p->builder, val.value, dst.value); switch (id) { - case BuiltinProc_volatile_store: LLVMSetVolatile(instr, true); break; case BuiltinProc_non_temporal_store: - // TODO(bill): BuiltinProc_non_temporal_store + { + unsigned kind_id = LLVMGetMDKindIDInContext(p->module->ctx, "nontemporal", 11); + LLVMMetadataRef node = LLVMValueAsMetadata(LLVMConstInt(lb_type(p->module, t_u32), 1, false)); + LLVMSetMetadata(instr, kind_id, LLVMMetadataAsValue(p->module->ctx, node)); + } break; + case BuiltinProc_volatile_store: LLVMSetVolatile(instr, true); break; case BuiltinProc_atomic_store: LLVMSetOrdering(instr, LLVMAtomicOrderingSequentiallyConsistent); break; case BuiltinProc_atomic_store_explicit: LLVMSetOrdering(instr, llvm_atomic_ordering_from_odin(ce->args[2])); break; } @@ -2138,10 +2142,15 @@ lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValue const &tv, LLVMValueRef instr = LLVMBuildLoad(p->builder, dst.value, ""); switch (id) { - case BuiltinProc_volatile_load: LLVMSetVolatile(instr, true); break; case BuiltinProc_non_temporal_load: - // TODO(bill): BuiltinProc_non_temporal_load + { + unsigned kind_id = LLVMGetMDKindIDInContext(p->module->ctx, "nontemporal", 11); + LLVMMetadataRef node = LLVMValueAsMetadata(LLVMConstInt(lb_type(p->module, t_u32), 1, false)); + LLVMSetMetadata(instr, kind_id, LLVMMetadataAsValue(p->module->ctx, node)); + } break; + break; + case BuiltinProc_volatile_load: LLVMSetVolatile(instr, true); break; case BuiltinProc_atomic_load: LLVMSetOrdering(instr, LLVMAtomicOrderingSequentiallyConsistent); break; case BuiltinProc_atomic_load_explicit: LLVMSetOrdering(instr, llvm_atomic_ordering_from_odin(ce->args[1])); break; }