From cb0704d51c6a170bca8206a9bb3e9796c71c6341 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 21 Jul 2024 21:58:25 -0400 Subject: [PATCH 01/43] Add `core:text/regex` --- core/text/regex/common/common.odin | 27 + core/text/regex/common/debugging.odin | 25 + core/text/regex/compiler/compiler.odin | 538 +++++++++++++++ core/text/regex/compiler/debugging.odin | 84 +++ core/text/regex/compiler/doc.odin | 9 + core/text/regex/doc.odin | 75 ++ core/text/regex/optimizer/doc.odin | 58 ++ core/text/regex/optimizer/optimizer.odin | 522 ++++++++++++++ core/text/regex/parser/debugging.odin | 103 +++ core/text/regex/parser/doc.odin | 10 + core/text/regex/parser/parser.odin | 580 ++++++++++++++++ core/text/regex/regex.odin | 434 ++++++++++++ core/text/regex/tokenizer/tokenizer.odin | 349 ++++++++++ core/text/regex/virtual_machine/doc.odin | 175 +++++ core/text/regex/virtual_machine/util.odin | 73 ++ .../virtual_machine/virtual_machine.odin | 638 ++++++++++++++++++ 16 files changed, 3700 insertions(+) create mode 100644 core/text/regex/common/common.odin create mode 100644 core/text/regex/common/debugging.odin create mode 100644 core/text/regex/compiler/compiler.odin create mode 100644 core/text/regex/compiler/debugging.odin create mode 100644 core/text/regex/compiler/doc.odin create mode 100644 core/text/regex/doc.odin create mode 100644 core/text/regex/optimizer/doc.odin create mode 100644 core/text/regex/optimizer/optimizer.odin create mode 100644 core/text/regex/parser/debugging.odin create mode 100644 core/text/regex/parser/doc.odin create mode 100644 core/text/regex/parser/parser.odin create mode 100644 core/text/regex/regex.odin create mode 100644 core/text/regex/tokenizer/tokenizer.odin create mode 100644 core/text/regex/virtual_machine/doc.odin create mode 100644 core/text/regex/virtual_machine/util.odin create mode 100644 core/text/regex/virtual_machine/virtual_machine.odin diff --git a/core/text/regex/common/common.odin b/core/text/regex/common/common.odin new file mode 100644 index 000000000..f53f043a1 --- /dev/null +++ b/core/text/regex/common/common.odin @@ -0,0 +1,27 @@ +// This package helps break dependency cycles. +package regex_common + +// VM limitations +MAX_CAPTURE_GROUPS :: 10 +MAX_PROGRAM_SIZE :: int(max(i16)) +MAX_CLASSES :: int(max(u8)) + +Flag :: enum u8 { + // Global: try to match the pattern anywhere in the string. + Global, + // Multiline: treat `^` and `$` as if they also match newlines. + Multiline, + // Case Insensitive: treat `a-z` as if it was also `A-Z`. + Case_Insensitive, + // Ignore Whitespace: bypass unescaped whitespace outside of classes. + Ignore_Whitespace, + // Unicode: let the compiler and virtual machine know to expect Unicode strings. + Unicode, + + // No Capture: avoid saving capture group data entirely. + No_Capture, + // No Optimization: do not pass the pattern through the optimizer; for debugging. + No_Optimization, +} + +Flags :: bit_set[Flag; u8] diff --git a/core/text/regex/common/debugging.odin b/core/text/regex/common/debugging.odin new file mode 100644 index 000000000..062c314cc --- /dev/null +++ b/core/text/regex/common/debugging.odin @@ -0,0 +1,25 @@ +package regex_common + +@require import "core:os" +import "core:io" +import "core:strings" + +ODIN_DEBUG_REGEX :: #config(ODIN_DEBUG_REGEX, false) + +when ODIN_DEBUG_REGEX { + debug_stream := os.stream_from_handle(os.stderr) +} + +write_padded_hex :: proc(w: io.Writer, #any_int n, zeroes: int) { + sb := strings.builder_make() + defer strings.builder_destroy(&sb) + + sbw := strings.to_writer(&sb) + io.write_int(sbw, n, 0x10) + + io.write_string(w, "0x") + for _ in 0.. bool #no_bounds_check { + assert(q != nil) + assert(w != nil) + + if q == w { + return true + } + + if len(q.runes) != len(w.runes) || len(q.ranges) != len(w.ranges) { + return false + } + + for r, i in q.runes { + if r != w.runes[i] { + return false + } + } + + for r, i in q.ranges { + if r.lower != w.ranges[i].lower || r.upper != w.ranges[i].upper { + return false + } + } + + return true +} + +map_all_classes :: proc(tree: Node, collection: ^[dynamic]Rune_Class_Data) { + if tree == nil { + return + } + + switch specific in tree { + case ^Node_Rune: break + case ^Node_Wildcard: break + case ^Node_Anchor: break + case ^Node_Word_Boundary: break + case ^Node_Match_All_And_Escape: break + + case ^Node_Concatenation: + for subnode in specific.nodes { + map_all_classes(subnode, collection) + } + + case ^Node_Repeat_Zero: + map_all_classes(specific.inner, collection) + case ^Node_Repeat_Zero_Non_Greedy: + map_all_classes(specific.inner, collection) + case ^Node_Repeat_One: + map_all_classes(specific.inner, collection) + case ^Node_Repeat_One_Non_Greedy: + map_all_classes(specific.inner, collection) + case ^Node_Repeat_N: + map_all_classes(specific.inner, collection) + case ^Node_Optional: + map_all_classes(specific.inner, collection) + case ^Node_Optional_Non_Greedy: + map_all_classes(specific.inner, collection) + case ^Node_Group: + map_all_classes(specific.inner, collection) + + case ^Node_Alternation: + map_all_classes(specific.left, collection) + map_all_classes(specific.right, collection) + + case ^Node_Rune_Class: + unseen := true + for &value in collection { + if classes_are_exact(&specific.data, &value) { + unseen = false + break + } + } + + if unseen { + append(collection, specific.data) + } + } +} + +append_raw :: #force_inline proc(code: ^Program, data: $T) { + // NOTE: This is system-dependent endian. + for b in transmute([size_of(T)]byte)data { + append(code, cast(Opcode)b) + } +} +inject_raw :: #force_inline proc(code: ^Program, start: int, data: $T) { + // NOTE: This is system-dependent endian. + for b, i in transmute([size_of(T)]byte)data { + inject_at(code, start + i, cast(Opcode)b) + } +} + +@require_results +generate_code :: proc(c: ^Compiler, node: Node) -> (code: Program) { + if node == nil { + return + } + + // NOTE: For Jump/Split arguments, we write as i16 and will reinterpret + // this later when relative jumps are turned into absolute jumps. + + switch specific in node { + // Atomic Nodes: + case ^Node_Rune: + if .Unicode not_in c.flags || specific.data < unicode.MAX_LATIN1 { + append(&code, Opcode.Byte) + append(&code, cast(Opcode)specific.data) + } else { + append(&code, Opcode.Rune) + append_raw(&code, specific.data) + } + + case ^Node_Rune_Class: + if specific.negating { + append(&code, Opcode.Rune_Class_Negated) + } else { + append(&code, Opcode.Rune_Class) + } + + index := -1 + for &data, i in c.class_data { + if classes_are_exact(&data, &specific.data) { + index = i + break + } + } + assert(index != -1, "Unable to find collected Rune_Class_Data index.") + + append(&code, Opcode(index)) + + case ^Node_Wildcard: + append(&code, Opcode.Wildcard) + + case ^Node_Anchor: + if .Multiline in c.flags { + append(&code, Opcode.Multiline_Open) + append(&code, Opcode.Multiline_Close) + } else { + if specific.start { + c.anchor_start_seen = true + append(&code, Opcode.Assert_Start) + } else { + append(&code, Opcode.Assert_End) + } + } + case ^Node_Word_Boundary: + if specific.non_word { + append(&code, Opcode.Assert_Non_Word_Boundary) + } else { + append(&code, Opcode.Assert_Word_Boundary) + } + + // Compound Nodes: + case ^Node_Group: + code = generate_code(c, specific.inner) + + if specific.capture && .No_Capture not_in c.flags { + inject_at(&code, 0, Opcode.Save) + inject_at(&code, 1, Opcode(2 * specific.capture_id)) + + append(&code, Opcode.Save) + append(&code, Opcode(2 * specific.capture_id + 1)) + } + + case ^Node_Alternation: + left := generate_code(c, specific.left) + right := generate_code(c, specific.right) + + left_len := len(left) + + // Avoiding duplicate allocation by reusing `left`. + code = left + + inject_at(&code, 0, Opcode.Split) + inject_raw(&code, size_of(byte) , i16(SPLIT_SIZE)) + inject_raw(&code, size_of(byte) + size_of(i16), i16(SPLIT_SIZE + left_len + JUMP_SIZE)) + + append(&code, Opcode.Jump) + append_raw(&code, i16(len(right) + JUMP_SIZE)) + + for opcode in right { + append(&code, opcode) + } + + case ^Node_Concatenation: + for subnode in specific.nodes { + subnode_code := generate_code(c, subnode) + for opcode in subnode_code { + append(&code, opcode) + } + } + + case ^Node_Repeat_Zero: + code = generate_code(c, specific.inner) + original_len := len(code) + + inject_at(&code, 0, Opcode.Split) + inject_raw(&code, size_of(byte) , i16(SPLIT_SIZE)) + inject_raw(&code, size_of(byte) + size_of(i16), i16(SPLIT_SIZE + original_len + JUMP_SIZE)) + + append(&code, Opcode.Jump) + append_raw(&code, i16(-original_len - SPLIT_SIZE)) + + case ^Node_Repeat_Zero_Non_Greedy: + code = generate_code(c, specific.inner) + original_len := len(code) + + inject_at(&code, 0, Opcode.Split) + inject_raw(&code, size_of(byte) , i16(SPLIT_SIZE + original_len + JUMP_SIZE)) + inject_raw(&code, size_of(byte) + size_of(i16), i16(SPLIT_SIZE)) + + append(&code, Opcode.Jump) + append_raw(&code, i16(-original_len - SPLIT_SIZE)) + + case ^Node_Repeat_One: + code = generate_code(c, specific.inner) + original_len := len(code) + + append(&code, Opcode.Split) + append_raw(&code, i16(-original_len)) + append_raw(&code, i16(SPLIT_SIZE)) + + case ^Node_Repeat_One_Non_Greedy: + code = generate_code(c, specific.inner) + original_len := len(code) + + append(&code, Opcode.Split) + append_raw(&code, i16(SPLIT_SIZE)) + append_raw(&code, i16(-original_len)) + + case ^Node_Repeat_N: + inside := generate_code(c, specific.inner) + original_len := len(inside) + + if specific.lower == specific.upper { // {N} + // e{N} ... evaluates to ... e^N + for i := 0; i < specific.upper; i += 1 { + for opcode in inside { + append(&code, opcode) + } + } + + } else if specific.lower == -1 && specific.upper > 0 { // {,M} + // e{,M} ... evaluates to ... e?^M + for i := 0; i < specific.upper; i += 1 { + append(&code, Opcode.Split) + append_raw(&code, i16(SPLIT_SIZE)) + append_raw(&code, i16(SPLIT_SIZE + original_len)) + for opcode in inside { + append(&code, opcode) + } + } + + } else if specific.lower >= 0 && specific.upper == -1 { // {N,} + // e{N,} ... evaluates to ... e^N e* + for i := 0; i < specific.lower; i += 1 { + for opcode in inside { + append(&code, opcode) + } + } + + append(&code, Opcode.Split) + append_raw(&code, i16(SPLIT_SIZE)) + append_raw(&code, i16(SPLIT_SIZE + original_len + JUMP_SIZE)) + + for opcode in inside { + append(&code, opcode) + } + + append(&code, Opcode.Jump) + append_raw(&code, i16(-original_len - SPLIT_SIZE)) + + } else if specific.lower >= 0 && specific.upper > 0 { + // e{N,M} evaluates to ... e^N e?^(M-N) + for i := 0; i < specific.lower; i += 1 { + for opcode in inside { + append(&code, opcode) + } + } + for i := 0; i < specific.upper - specific.lower; i += 1 { + append(&code, Opcode.Split) + append_raw(&code, i16(SPLIT_SIZE + original_len)) + append_raw(&code, i16(SPLIT_SIZE)) + for opcode in inside { + append(&code, opcode) + } + } + + } else { + panic("RegEx compiler received invalid repetition group.") + } + + case ^Node_Optional: + code = generate_code(c, specific.inner) + original_len := len(code) + + inject_at(&code, 0, Opcode.Split) + inject_raw(&code, size_of(byte) , i16(SPLIT_SIZE)) + inject_raw(&code, size_of(byte) + size_of(i16), i16(SPLIT_SIZE + original_len)) + + case ^Node_Optional_Non_Greedy: + code = generate_code(c, specific.inner) + original_len := len(code) + + inject_at(&code, 0, Opcode.Split) + inject_raw(&code, size_of(byte) , i16(SPLIT_SIZE + original_len)) + inject_raw(&code, size_of(byte) + size_of(i16), i16(SPLIT_SIZE)) + + case ^Node_Match_All_And_Escape: + append(&code, Opcode.Match_All_And_Escape) + } + + return +} + +@require_results +compile :: proc(tree: Node, flags: common.Flags) -> (code: Program, class_data: [dynamic]Rune_Class_Data, err: Error) { + if tree == nil { + if .No_Capture not_in flags { + append(&code, Opcode.Save); append(&code, Opcode(0x00)) + append(&code, Opcode.Save); append(&code, Opcode(0x01)) + append(&code, Opcode.Match) + } else { + append(&code, Opcode.Match_And_Exit) + } + return + } + + c: Compiler + c.flags = flags + + map_all_classes(tree, &class_data) + if len(class_data) >= common.MAX_CLASSES { + err = .Too_Many_Classes + return + } + c.class_data = class_data + + code = generate_code(&c, tree) + + pc_open := 0 + + add_global: if .Global in flags { + // Check if the opening to the pattern is predictable. + // If so, use one of the optimized Wait opcodes. + iter := virtual_machine.Opcode_Iterator{ code[:], 0 } + seek_loop: for opcode, pc in virtual_machine.iterate_opcodes(&iter) { + #partial switch opcode { + case .Byte: + inject_at(&code, pc_open, Opcode.Wait_For_Byte) + pc_open += size_of(Opcode) + inject_at(&code, pc_open, Opcode(code[pc + size_of(Opcode) + pc_open])) + pc_open += size_of(u8) + break add_global + + case .Rune: + operand := (cast(^rune)&code[pc+1])^ + inject_at(&code, pc_open, Opcode.Wait_For_Rune) + pc_open += size_of(Opcode) + inject_raw(&code, pc_open, operand) + pc_open += size_of(rune) + break add_global + + case .Rune_Class: + inject_at(&code, pc_open, Opcode.Wait_For_Rune_Class) + pc_open += size_of(Opcode) + inject_at(&code, pc_open, Opcode(code[pc + size_of(Opcode) + pc_open])) + pc_open += size_of(u8) + break add_global + + case .Rune_Class_Negated: + inject_at(&code, pc_open, Opcode.Wait_For_Rune_Class_Negated) + pc_open += size_of(Opcode) + inject_at(&code, pc_open, Opcode(code[pc + size_of(Opcode) + pc_open])) + pc_open += size_of(u8) + break add_global + + case .Save: + continue + case: + break seek_loop + } + } + + // `.*?` + inject_at(&code, pc_open, Opcode.Split) + pc_open += size_of(byte) + inject_raw(&code, pc_open, i16(SPLIT_SIZE + size_of(byte) + JUMP_SIZE)) + pc_open += size_of(i16) + inject_raw(&code, pc_open, i16(SPLIT_SIZE)) + pc_open += size_of(i16) + + inject_at(&code, pc_open, Opcode.Wildcard) + pc_open += size_of(byte) + + inject_at(&code, pc_open, Opcode.Jump) + pc_open += size_of(byte) + inject_raw(&code, pc_open, i16(-size_of(byte) - SPLIT_SIZE)) + pc_open += size_of(i16) + + } + + if .No_Capture not_in flags { + // `(` + inject_at(&code, pc_open, Opcode.Save) + inject_at(&code, pc_open + size_of(byte), Opcode(0x00)) + + // `)` + append(&code, Opcode.Save); append(&code, Opcode(0x01)) + + append(&code, Opcode.Match) + } else { + append(&code, Opcode.Match_And_Exit) + } + + if len(code) >= common.MAX_PROGRAM_SIZE { + err = .Program_Too_Big + return + } + + // NOTE: No further opcode addition beyond this point, as we've already + // checked the program size. Removal or transformation is fine. + + // Post-Compile Optimizations: + + // * Jump Extension + // + // A:RelJmp(1) -> B:RelJmp(2) => A:RelJmp(2) + if .No_Optimization not_in flags { + for passes_left := 1; passes_left > 0; passes_left -= 1 { + do_another_pass := false + + iter := virtual_machine.Opcode_Iterator{ code[:], 0 } + for opcode, pc in virtual_machine.iterate_opcodes(&iter) { + #partial switch opcode { + case .Jump: + jmp := cast(^i16)&code[pc+size_of(Opcode)] + if code[cast(i16)pc+jmp^] == .Jump { + next_jmp := (cast(^i16)&code[cast(i16)pc+jmp^+size_of(Opcode)])^ + jmp^ = jmp^ + next_jmp + do_another_pass = true + } + case .Split: + jmp_x := cast(^i16)&code[pc+size_of(Opcode)] + if code[cast(i16)pc+jmp_x^] == .Jump { + next_jmp := (cast(^i16)&code[cast(i16)pc+jmp_x^+size_of(Opcode)])^ + jmp_x^ = jmp_x^ + next_jmp + do_another_pass = true + } + jmp_y := cast(^i16)&code[pc+size_of(Opcode)+size_of(i16)] + if code[cast(i16)pc+jmp_y^] == .Jump { + next_jmp := (cast(^i16)&code[cast(i16)pc+jmp_y^+size_of(Opcode)])^ + jmp_y^ = jmp_y^ + next_jmp + do_another_pass = true + } + } + } + + if do_another_pass { + passes_left += 1 + } + } + } + + // * Relative Jump to Absolute Jump + // + // RelJmp{PC +/- N} => AbsJmp{M} + iter := virtual_machine.Opcode_Iterator{ code[:], 0 } + for opcode, pc in virtual_machine.iterate_opcodes(&iter) { + // NOTE: The virtual machine implementation depends on this. + #partial switch opcode { + case .Jump: + jmp := cast(^u16)&code[pc+size_of(Opcode)] + jmp^ = jmp^ + cast(u16)pc + case .Split: + jmp_x := cast(^u16)&code[pc+size_of(Opcode)] + jmp_x^ = jmp_x^ + cast(u16)pc + jmp_y := cast(^u16)&code[pc+size_of(Opcode)+size_of(i16)] + jmp_y^ = jmp_y^ + cast(u16)pc + } + } + + return +} diff --git a/core/text/regex/compiler/debugging.odin b/core/text/regex/compiler/debugging.odin new file mode 100644 index 000000000..1ef3e6d78 --- /dev/null +++ b/core/text/regex/compiler/debugging.odin @@ -0,0 +1,84 @@ +package regex_compiler + +import "core:io" +import "core:text/regex/common" +import "core:text/regex/virtual_machine" + +get_jump_targets :: proc(code: []Opcode) -> (jump_targets: map[int]int) { + iter := virtual_machine.Opcode_Iterator{ code, 0 } + for opcode, pc in virtual_machine.iterate_opcodes(&iter) { + #partial switch opcode { + case .Jump: + jmp := cast(int)(cast(^u16)&code[pc+1])^ + jump_targets[jmp] = pc + case .Split: + jmp_x := cast(int)(cast(^u16)&code[pc+1])^ + jmp_y := cast(int)(cast(^u16)&code[pc+3])^ + jump_targets[jmp_x] = pc + jump_targets[jmp_y] = pc + } + } + return +} + +trace :: proc(w: io.Writer, code: []Opcode) { + jump_targets := get_jump_targets(code) + defer delete(jump_targets) + + iter := virtual_machine.Opcode_Iterator{ code, 0 } + for opcode, pc in virtual_machine.iterate_opcodes(&iter) { + if src, ok := jump_targets[pc]; ok { + io.write_string(w, "--") + common.write_padded_hex(w, src, 4) + io.write_string(w, "--> ") + } else { + io.write_string(w, " ") + } + + io.write_string(w, "[PC: ") + common.write_padded_hex(w, pc, 4) + io.write_string(w, "] ") + io.write_string(w, virtual_machine.opcode_to_name(opcode)) + io.write_byte(w, ' ') + + #partial switch opcode { + case .Byte: + operand := cast(rune)code[pc+1] + io.write_encoded_rune(w, operand) + case .Rune: + operand := (cast(^rune)&code[pc+1])^ + io.write_encoded_rune(w, operand) + case .Rune_Class, .Rune_Class_Negated: + operand := cast(u8)code[pc+1] + common.write_padded_hex(w, operand, 2) + case .Jump: + jmp := (cast(^u16)&code[pc+1])^ + io.write_string(w, "-> $") + common.write_padded_hex(w, jmp, 4) + case .Split: + jmp_x := (cast(^u16)&code[pc+1])^ + jmp_y := (cast(^u16)&code[pc+3])^ + io.write_string(w, "=> $") + common.write_padded_hex(w, jmp_x, 4) + io.write_string(w, ", $") + common.write_padded_hex(w, jmp_y, 4) + case .Save: + operand := cast(u8)code[pc+1] + common.write_padded_hex(w, operand, 2) + case .Wait_For_Byte: + operand := cast(rune)code[pc+1] + io.write_encoded_rune(w, operand) + case .Wait_For_Rune: + operand := (cast(^rune)&code[pc+1])^ + io.write_encoded_rune(w, operand) + case .Wait_For_Rune_Class: + operand := cast(u8)code[pc+1] + common.write_padded_hex(w, operand, 2) + case .Wait_For_Rune_Class_Negated: + operand := cast(u8)code[pc+1] + common.write_padded_hex(w, operand, 2) + } + + io.write_byte(w, '\n') + } +} diff --git a/core/text/regex/compiler/doc.odin b/core/text/regex/compiler/doc.odin new file mode 100644 index 000000000..8c876d837 --- /dev/null +++ b/core/text/regex/compiler/doc.odin @@ -0,0 +1,9 @@ +/* +package regex_compiler implements a bytecode compiler for the virtual machine +included alongside it. + +Operands larger than u8 are written in system endian order. + +More details can be found in the documentation for the virtual machine. +*/ +package regex_compiler diff --git a/core/text/regex/doc.odin b/core/text/regex/doc.odin new file mode 100644 index 000000000..8899e1af6 --- /dev/null +++ b/core/text/regex/doc.odin @@ -0,0 +1,75 @@ +/* +package regex implements a complete suite for using Regular Expressions to +match and capture text. + +Regular expressions are used to describe how a piece of text can match to +another, using a pattern language. + +Odin's regex library implements the following features: + + Alternation: `apple|cherry` + Classes: `[0-9_]` + Wildcards: `.` + Repeat, optional: `a*` + Repeat, at least once: `a+` + Optional: `a?` + Group Capture: `([0-9])` + Group Non-Capture: `(?:[0-9])` + Start & End Anchors: `^hello$` + Word Boundaries: `\bhello\b` + Non-Word Boundaries: `hello\B` + +These specifiers can be composed together, such as an optional group: +`(?:hello)?` + +This package also supports the non-greedy variants of the repeating and +optional specifiers by appending a `?` to them. + + + + ``Some people, when confronted with a problem, think + "I know, I'll use regular expressions." Now they have two problems.'' + + - Jamie Zawinski + + +Regular expressions have gathered a reputation over the decades for often being +chosen as the wrong tool for the job. Here, we will clarify a few cases in +which RegEx might be good or bad. + + +**When is it a good time to use RegEx?** + +- You don't know at compile-time what patterns of text the program will need to + match when it's running. +- As an example, you are making a client which can be configured by the user to + trigger on certain text patterns received from a server. +- For another example, you need a way for users of a text editor to compose + matching strings that are more intricate than a simple substring lookup. +- The text you're matching against is small (< 64 KiB) and your patterns aren't + overly complicated with branches (alternations, repeats, and optionals). +- If none of the above general impressions apply but your project doesn't + warrant long-term maintenance. + +**When is it a bad time to use RegEx?** + +- You know at compile-time the grammar you're parsing; a hand-made parser has + the potential to be more maintainable and readable. +- The grammar you're parsing has certain validation steps that lend itself to + forming complicated expressions, such as e-mail addresses, URIs, dates, + postal codes, credit cards, et cetera. Using RegEx to validate these + structures is almost always a bad sign. +- The text you're matching against is big (> 1 MiB); you would be better served + by first dividing the text into manageable chunks and using some heuristic to + locate the most likely location of a match before applying RegEx against it. +- You value high performance and low memory usage; RegEx will always have a + certain overhead which increases with the complexity of the pattern. + + +The implementation of this package has been optimized, but it will never be as +thoroughly performant as a hand-made parser. In comparison, there are just too +many intermediate steps, assumptions, and generalizations in what it takes to +handle a regular expression. + +*/ +package regex diff --git a/core/text/regex/optimizer/doc.odin b/core/text/regex/optimizer/doc.odin new file mode 100644 index 000000000..7f2c84c8d --- /dev/null +++ b/core/text/regex/optimizer/doc.odin @@ -0,0 +1,58 @@ +/* +package regex_optimizer implements an optimizer which acts upon the AST of a +parsed regular expression pattern, transforming it in-place without moving to a +compilation step. + +Where possible, it aims to reduce branching as much as possible in the +expression by reducing usage of `|`. + + +Here is a summary of the optimizations that it will do: + +* Class Simplification : `[aab]` => `[ab]` + `[aa]` => `[a]` + +* Class Reduction : `[a]` => `a` +* Range Construction : `[abc]` => `[a-c]` +* Rune Merging into Range : `[aa-c]` => `[a-c]` + +* Range Merging : `[a-cc-e]` => `[a-e]` + `[a-cd-e]` => `[a-e]` + `[a-cb-e]` => `[a-e]` + +* Alternation to Optional : `a|` => `a?` +* Alternation to Optional Non-Greedy : `|a` => `a??` +* Alternation Reduction : `a|a` => `a` +* Alternation to Class : `a|b` => `[ab]` +* Class Union : `[a0]|[b1]` => `[a0b1]` + `[a-b]|c` => `[a-bc]` + `a|[b-c]` => `[b-ca]` + +* Wildcard Reduction : `a|.` => `.` + `.|a` => `.` + `[ab]|.` => `.` + `.|[ab]` => `.` + +* Common Suffix Elimination : `blueberry|strawberry` => `(?:blue|straw)berry` +* Common Prefix Elimination : `abi|abe` => `ab(?:i|e)` + +* Composition: Consume All to Anchored End + `.*$` => + `.+$` => `.` + + +Possible future improvements: + +- Change the AST of alternations to be a list instead of a tree, so that + constructions such as `(ab|bb|cb)` can be considered in whole by the affix + elimination optimizations. + +- Introduce specialized opcodes for certain classes of repetition. + +- Add Common Infix Elimination. + +- Measure the precise finite minimum and maximum of a pattern, if available, + and check against that on any strings before running the virtual machine. + +*/ +package regex_optimizer diff --git a/core/text/regex/optimizer/optimizer.odin b/core/text/regex/optimizer/optimizer.odin new file mode 100644 index 000000000..fbb65cf79 --- /dev/null +++ b/core/text/regex/optimizer/optimizer.odin @@ -0,0 +1,522 @@ +package regex_optimizer + +import "base:intrinsics" +@require import "core:io" +import "core:slice" +import "core:text/regex/common" +import "core:text/regex/parser" + +Rune_Class_Range :: parser.Rune_Class_Range + +Node :: parser.Node +Node_Rune :: parser.Node_Rune +Node_Rune_Class :: parser.Node_Rune_Class +Node_Wildcard :: parser.Node_Wildcard +Node_Concatenation :: parser.Node_Concatenation +Node_Alternation :: parser.Node_Alternation +Node_Repeat_Zero :: parser.Node_Repeat_Zero +Node_Repeat_Zero_Non_Greedy :: parser.Node_Repeat_Zero_Non_Greedy +Node_Repeat_One :: parser.Node_Repeat_One +Node_Repeat_One_Non_Greedy :: parser.Node_Repeat_One_Non_Greedy +Node_Repeat_N :: parser.Node_Repeat_N +Node_Optional :: parser.Node_Optional +Node_Optional_Non_Greedy :: parser.Node_Optional_Non_Greedy +Node_Group :: parser.Node_Group +Node_Anchor :: parser.Node_Anchor +Node_Word_Boundary :: parser.Node_Word_Boundary +Node_Match_All_And_Escape :: parser.Node_Match_All_And_Escape + + +class_range_sorter :: proc(i, j: Rune_Class_Range) -> bool { + return i.lower < j.lower +} + +optimize_subtree :: proc(tree: Node, flags: common.Flags) -> (result: Node, changes: int) { + if tree == nil { + return nil, 0 + } + + result = tree + + switch specific in tree { + // No direct optimization possible on these nodes: + case ^Node_Rune: break + case ^Node_Wildcard: break + case ^Node_Anchor: break + case ^Node_Word_Boundary: break + case ^Node_Match_All_And_Escape: break + + case ^Node_Concatenation: + // * Composition: Consume All to Anchored End + // + // DO: `.*$` => + // DO: `.+$` => `.` + if .Multiline not_in flags && len(specific.nodes) >= 2 { + i := len(specific.nodes) - 2 + wrza: { + subnode := specific.nodes[i].(^Node_Repeat_Zero) or_break wrza + _ = subnode.inner.(^Node_Wildcard) or_break wrza + next_node := specific.nodes[i+1].(^Node_Anchor) or_break wrza + if next_node.start == false { + specific.nodes[i] = new(Node_Match_All_And_Escape) + ordered_remove(&specific.nodes, i + 1) + changes += 1 + break + } + } + wroa: { + subnode := specific.nodes[i].(^Node_Repeat_One) or_break wroa + subsubnode := subnode.inner.(^Node_Wildcard) or_break wroa + next_node := specific.nodes[i+1].(^Node_Anchor) or_break wroa + if next_node.start == false { + specific.nodes[i] = subsubnode + specific.nodes[i+1] = new(Node_Match_All_And_Escape) + changes += 1 + break + } + } + } + + // Only recursive optimizations: + for i := 0; i < len(specific.nodes); i += 1 { + subnode, subnode_changes := optimize_subtree(specific.nodes[i], flags) + changes += subnode_changes + if subnode == nil { + ordered_remove(&specific.nodes, i) + i -= 1 + changes += 1 + } else { + specific.nodes[i] = subnode + } + } + + if len(specific.nodes) == 1 { + result = specific.nodes[0] + changes += 1 + } else if len(specific.nodes) == 0 { + return nil, changes + 1 + } + + case ^Node_Repeat_Zero: + specific.inner, changes = optimize_subtree(specific.inner, flags) + if specific.inner == nil { + return nil, changes + 1 + } + case ^Node_Repeat_Zero_Non_Greedy: + specific.inner, changes = optimize_subtree(specific.inner, flags) + if specific.inner == nil { + return nil, changes + 1 + } + case ^Node_Repeat_One: + specific.inner, changes = optimize_subtree(specific.inner, flags) + if specific.inner == nil { + return nil, changes + 1 + } + case ^Node_Repeat_One_Non_Greedy: + specific.inner, changes = optimize_subtree(specific.inner, flags) + if specific.inner == nil { + return nil, changes + 1 + } + case ^Node_Repeat_N: + specific.inner, changes = optimize_subtree(specific.inner, flags) + if specific.inner == nil { + return nil, changes + 1 + } + case ^Node_Optional: + specific.inner, changes = optimize_subtree(specific.inner, flags) + if specific.inner == nil { + return nil, changes + 1 + } + case ^Node_Optional_Non_Greedy: + specific.inner, changes = optimize_subtree(specific.inner, flags) + if specific.inner == nil { + return nil, changes + 1 + } + + case ^Node_Group: + specific.inner, changes = optimize_subtree(specific.inner, flags) + + if specific.inner == nil { + return nil, changes + 1 + } + + if !specific.capture { + result = specific.inner + changes += 1 + } + + // Full optimization: + case ^Node_Rune_Class: + // * Class Simplification + // + // DO: `[aab]` => `[ab]` + // DO: `[aa]` => `[a]` + runes_seen: map[rune]bool + + for r in specific.runes { + runes_seen[r] = true + } + + if len(runes_seen) != len(specific.runes) { + clear(&specific.runes) + for key in runes_seen { + append(&specific.runes, key) + } + changes += 1 + } + + // * Class Reduction + // + // DO: `[a]` => `a` + if !specific.negating && len(specific.runes) == 1 && len(specific.ranges) == 0 { + only_rune := specific.runes[0] + + node := new(Node_Rune) + node.data = only_rune + + return node, changes + 1 + } + + // * Range Construction + // + // DO: `[abc]` => `[a-c]` + slice.sort(specific.runes[:]) + if len(specific.runes) > 1 { + new_range: Rune_Class_Range + new_range.lower = specific.runes[0] + new_range.upper = specific.runes[0] + + for i := 1; i < len(specific.runes); i += 1 { + r := specific.runes[i] + if new_range.lower == -1 { + new_range = { r, r } + continue + } + + if r == new_range.lower - 1 { + new_range.lower -= 1 + ordered_remove(&specific.runes, i) + i -= 1 + changes += 1 + } else if r == new_range.upper + 1 { + new_range.upper += 1 + ordered_remove(&specific.runes, i) + i -= 1 + changes += 1 + } else if new_range.lower != new_range.upper { + append(&specific.ranges, new_range) + new_range = { -1, -1 } + changes += 1 + } + } + + if new_range.lower != new_range.upper { + append(&specific.ranges, new_range) + changes += 1 + } + } + + // * Rune Merging into Range + // + // DO: `[aa-c]` => `[a-c]` + for range in specific.ranges { + for i := 0; i < len(specific.runes); i += 1 { + r := specific.runes[i] + if range.lower <= r && r <= range.upper { + ordered_remove(&specific.runes, i) + i -= 1 + changes += 1 + } + } + } + + // * Range Merging + // + // DO: `[a-cc-e]` => `[a-e]` + // DO: `[a-cd-e]` => `[a-e]` + // DO: `[a-cb-e]` => `[a-e]` + slice.sort_by(specific.ranges[:], class_range_sorter) + for i := 0; i < len(specific.ranges) - 1; i += 1 { + for j := i + 1; j < len(specific.ranges); j += 1 { + left_range := &specific.ranges[i] + right_range := specific.ranges[j] + + if left_range.upper == right_range.lower || + left_range.upper == right_range.lower - 1 || + left_range.lower <= right_range.lower && right_range.lower <= left_range.upper { + left_range.upper = max(left_range.upper, right_range.upper) + ordered_remove(&specific.ranges, j) + j -= 1 + changes += 1 + } else { + break + } + } + } + + if len(specific.ranges) == 0 { + specific.ranges = {} + } + if len(specific.runes) == 0 { + specific.runes = {} + } + + // * NOP + // + // DO: `[]` => + if len(specific.ranges) + len(specific.runes) == 0 { + return nil, 1 + } + + slice.sort(specific.runes[:]) + slice.sort_by(specific.ranges[:], class_range_sorter) + + case ^Node_Alternation: + // Perform recursive optimization first. + left_changes, right_changes: int + specific.left, left_changes = optimize_subtree(specific.left, flags) + specific.right, right_changes = optimize_subtree(specific.right, flags) + changes += left_changes + right_changes + + // * Alternation to Optional + // + // DO: `a|` => `a?` + if specific.left != nil && specific.right == nil { + node := new(Node_Optional) + node.inner = specific.left + return node, 1 + } + + // * Alternation to Optional Non-Greedy + // + // DO: `|a` => `a??` + if specific.right != nil && specific.left == nil { + node := new(Node_Optional_Non_Greedy) + node.inner = specific.right + return node, 1 + } + + // * NOP + // + // DO: `|` => + if specific.left == nil && specific.right == nil { + return nil, 1 + } + + left_rune, left_is_rune := specific.left.(^Node_Rune) + right_rune, right_is_rune := specific.right.(^Node_Rune) + + if left_is_rune && right_is_rune { + if left_rune.data == right_rune.data { + // * Alternation Reduction + // + // DO: `a|a` => `a` + return left_rune, 1 + } else { + // * Alternation to Class + // + // DO: `a|b` => `[ab]` + node := new(Node_Rune_Class) + append(&node.runes, left_rune.data) + append(&node.runes, right_rune.data) + return node, 1 + } + } + + left_wildcard, left_is_wildcard := specific.left.(^Node_Wildcard) + right_wildcard, right_is_wildcard := specific.right.(^Node_Wildcard) + + // * Class Union + // + // DO: `[a0]|[b1]` => `[a0b1]` + left_class, left_is_class := specific.left.(^Node_Rune_Class) + right_class, right_is_class := specific.right.(^Node_Rune_Class) + if left_is_class && right_is_class { + for r in right_class.runes { + append(&left_class.runes, r) + } + for range in right_class.ranges { + append(&left_class.ranges, range) + } + return left_class, 1 + } + + // * Class Union + // + // DO: `[a-b]|c` => `[a-bc]` + if left_is_class && right_is_rune { + append(&left_class.runes, right_rune.data) + return left_class, 1 + } + + // * Class Union + // + // DO: `a|[b-c]` => `[b-ca]` + if left_is_rune && right_is_class { + append(&right_class.runes, left_rune.data) + return right_class, 1 + } + + // * Wildcard Reduction + // + // DO: `a|.` => `.` + if left_is_rune && right_is_wildcard { + return right_wildcard, 1 + } + + // * Wildcard Reduction + // + // DO: `.|a` => `.` + if left_is_wildcard && right_is_rune { + return left_wildcard, 1 + } + + // * Wildcard Reduction + // + // DO: `[ab]|.` => `.` + if left_is_class && right_is_wildcard { + return right_wildcard, 1 + } + + // * Wildcard Reduction + // + // DO: `.|[ab]` => `.` + if left_is_wildcard && right_is_class { + return left_wildcard, 1 + } + + left_concatenation, left_is_concatenation := specific.left.(^Node_Concatenation) + right_concatenation, right_is_concatenation := specific.right.(^Node_Concatenation) + + // * Common Suffix Elimination + // + // DO: `blueberry|strawberry` => `(?:blue|straw)berry` + if left_is_concatenation && right_is_concatenation { + // Remember that a concatenation could contain any node, not just runes. + left_len := len(left_concatenation.nodes) + right_len := len(right_concatenation.nodes) + least_len := min(left_len, right_len) + same_len := 0 + for i := 1; i <= least_len; i += 1 { + left_subrune, left_is_subrune := left_concatenation.nodes[left_len - i].(^Node_Rune) + right_subrune, right_is_subrune := right_concatenation.nodes[right_len - i].(^Node_Rune) + + if !left_is_subrune || !right_is_subrune { + // One of the nodes isn't a rune; there's nothing more we can do. + break + } + + if left_subrune.data == right_subrune.data { + same_len += 1 + } else { + // No more similarities. + break + } + } + + if same_len > 0 { + // Dissolve this alternation into a concatenation. + cat_node := new(Node_Concatenation) + group_node := new(Node_Group) + append(&cat_node.nodes, group_node) + + // Turn the concatenation into the common suffix. + for i := left_len - same_len; i < left_len; i += 1 { + append(&cat_node.nodes, left_concatenation.nodes[i]) + } + + // Construct the group of alternating prefixes. + for i := same_len; i > 0; i -= 1 { + pop(&left_concatenation.nodes) + pop(&right_concatenation.nodes) + } + + // (Re-using this alternation node.) + alter_node := specific + alter_node.left = left_concatenation + alter_node.right = right_concatenation + group_node.inner = alter_node + + return cat_node, 1 + } + } + + // * Common Prefix Elimination + // + // DO: `abi|abe` => `ab(?:i|e)` + if left_is_concatenation && right_is_concatenation { + // Try to identify a common prefix. + // Remember that a concatenation could contain any node, not just runes. + least_len := min(len(left_concatenation.nodes), len(right_concatenation.nodes)) + same_len := 0 + for i := 0; i < least_len; i += 1 { + left_subrune, left_is_subrune := left_concatenation.nodes[i].(^Node_Rune) + right_subrune, right_is_subrune := right_concatenation.nodes[i].(^Node_Rune) + + if !left_is_subrune || !right_is_subrune { + // One of the nodes isn't a rune; there's nothing more we can do. + break + } + + if left_subrune.data == right_subrune.data { + same_len = i + 1 + } else { + // No more similarities. + break + } + } + + if same_len > 0 { + cat_node := new(Node_Concatenation) + for i := 0; i < same_len; i += 1 { + append(&cat_node.nodes, left_concatenation.nodes[i]) + } + for i := same_len; i > 0; i -= 1 { + ordered_remove(&left_concatenation.nodes, 0) + ordered_remove(&right_concatenation.nodes, 0) + } + + group_node := new(Node_Group) + // (Re-using this alternation node.) + alter_node := specific + alter_node.left = left_concatenation + alter_node.right = right_concatenation + group_node.inner = alter_node + + append(&cat_node.nodes, group_node) + return cat_node, 1 + } + } + } + + return +} + +optimize :: proc(tree: Node, flags: common.Flags) -> (result: Node, changes: int) { + result = tree + new_changes := 0 + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "AST before Optimizer: ") + parser.write_node(common.debug_stream, tree) + io.write_byte(common.debug_stream, '\n') + } + + // Keep optimizing until no more changes are seen. + for { + result, new_changes = optimize_subtree(result, flags) + changes += new_changes + if new_changes == 0 { + break + } + } + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "AST after Optimizer: ") + parser.write_node(common.debug_stream, result) + io.write_byte(common.debug_stream, '\n') + } + + + return +} diff --git a/core/text/regex/parser/debugging.odin b/core/text/regex/parser/debugging.odin new file mode 100644 index 000000000..4d531965c --- /dev/null +++ b/core/text/regex/parser/debugging.odin @@ -0,0 +1,103 @@ +package regex_parser + +import "core:io" + +write_node :: proc(w: io.Writer, node: Node) { + switch specific in node { + case ^Node_Rune: + io.write_rune(w, specific.data) + + case ^Node_Rune_Class: + io.write_byte(w, '[') + if specific.negating { + io.write_byte(w, '^') + } + for r in specific.data.runes { + io.write_rune(w, r) + } + for range in specific.data.ranges { + io.write_rune(w, range.lower) + io.write_byte(w, '-') + io.write_rune(w, range.upper) + } + io.write_byte(w, ']') + + case ^Node_Wildcard: + io.write_byte(w, '.') + + case ^Node_Concatenation: + io.write_rune(w, '「') + for subnode, i in specific.nodes { + if i != 0 { + io.write_rune(w, '⋅') + } + write_node(w, subnode) + } + io.write_rune(w, '」') + + case ^Node_Repeat_Zero: + write_node(w, specific.inner) + io.write_byte(w, '*') + case ^Node_Repeat_Zero_Non_Greedy: + write_node(w, specific.inner) + io.write_string(w, "*?") + case ^Node_Repeat_One: + write_node(w, specific.inner) + io.write_byte(w, '+') + case ^Node_Repeat_One_Non_Greedy: + write_node(w, specific.inner) + io.write_string(w, "+?") + + case ^Node_Repeat_N: + write_node(w, specific.inner) + if specific.lower == 0 && specific.upper == -1 { + io.write_byte(w, '*') + } else if specific.lower == 1 && specific.upper == -1 { + io.write_byte(w, '+') + } else { + io.write_byte(w, '{') + io.write_int(w, specific.lower) + io.write_byte(w, ',') + io.write_int(w, specific.upper) + io.write_byte(w, '}') + } + + case ^Node_Alternation: + io.write_rune(w, '《') + write_node(w, specific.left) + io.write_byte(w, '|') + write_node(w, specific.right) + io.write_rune(w, '》') + + case ^Node_Optional: + io.write_rune(w, '〈') + write_node(w, specific.inner) + io.write_byte(w, '?') + io.write_rune(w, '〉') + case ^Node_Optional_Non_Greedy: + io.write_rune(w, '〈') + write_node(w, specific.inner) + io.write_string(w, "??") + io.write_rune(w, '〉') + + case ^Node_Group: + io.write_byte(w, '(') + if !specific.capture { + io.write_string(w, "?:") + } + write_node(w, specific.inner) + io.write_byte(w, ')') + + case ^Node_Anchor: + io.write_byte(w, '^' if specific.start else '$') + + case ^Node_Word_Boundary: + io.write_string(w, `\B` if specific.non_word else `\b`) + + case ^Node_Match_All_And_Escape: + io.write_string(w, "《.*$》") + + case nil: + io.write_string(w, "") + } +} diff --git a/core/text/regex/parser/doc.odin b/core/text/regex/parser/doc.odin new file mode 100644 index 000000000..f518e518d --- /dev/null +++ b/core/text/regex/parser/doc.odin @@ -0,0 +1,10 @@ +/* +package regex_parser implements a Pratt parser, also known as a Top-Down +Operator Precedence parser, for parsing tokenized regular expression patterns. + +References: +- https://dl.acm.org/doi/10.1145/512927.512931 +- https://tdop.github.io/ +- http://crockford.com/javascript/tdop/tdop.html +*/ +package regex_parser diff --git a/core/text/regex/parser/parser.odin b/core/text/regex/parser/parser.odin new file mode 100644 index 000000000..1958ee399 --- /dev/null +++ b/core/text/regex/parser/parser.odin @@ -0,0 +1,580 @@ +package regex_parser + +import "base:intrinsics" +import "core:strconv" +import "core:strings" +import "core:text/regex/common" +import "core:text/regex/tokenizer" +import "core:unicode" +import "core:unicode/utf8" + +Token :: tokenizer.Token +Token_Kind :: tokenizer.Token_Kind +Tokenizer :: tokenizer.Tokenizer + +Rune_Class_Range :: struct { + lower, upper: rune, +} +Rune_Class_Data :: struct { + runes: [dynamic]rune, + ranges: [dynamic]Rune_Class_Range, +} + + +Node_Rune :: struct { + data: rune, +} + +Node_Rune_Class :: struct { + negating: bool, + using data: Rune_Class_Data, +} + +Node_Wildcard :: struct {} + +Node_Alternation :: struct { + left, right: Node, +} + +Node_Concatenation :: struct { + nodes: [dynamic]Node, +} + +Node_Repeat_Zero :: struct { + inner: Node, +} +Node_Repeat_Zero_Non_Greedy :: struct { + inner: Node, +} +Node_Repeat_One :: struct { + inner: Node, +} +Node_Repeat_One_Non_Greedy :: struct { + inner: Node, +} + +Node_Repeat_N :: struct { + inner: Node, + lower, upper: int, +} + +Node_Optional :: struct { + inner: Node, +} +Node_Optional_Non_Greedy :: struct { + inner: Node, +} + +Node_Group :: struct { + inner: Node, + capture_id: int, + capture: bool, +} + +Node_Anchor :: struct { + start: bool, +} +Node_Word_Boundary :: struct { + non_word: bool, +} + +Node_Match_All_And_Escape :: struct {} + +Node :: union { + ^Node_Rune, + ^Node_Rune_Class, + ^Node_Wildcard, + ^Node_Concatenation, + ^Node_Alternation, + ^Node_Repeat_Zero, + ^Node_Repeat_Zero_Non_Greedy, + ^Node_Repeat_One, + ^Node_Repeat_One_Non_Greedy, + ^Node_Repeat_N, + ^Node_Optional, + ^Node_Optional_Non_Greedy, + ^Node_Group, + ^Node_Anchor, + ^Node_Word_Boundary, + + // Optimized nodes (not created by the Parser): + ^Node_Match_All_And_Escape, +} + + +left_binding_power :: proc(kind: Token_Kind) -> int { + #partial switch kind { + case .Alternate: return 1 + case .Concatenate: return 2 + case .Repeat_Zero, .Repeat_One, + .Repeat_Zero_Non_Greedy, .Repeat_One_Non_Greedy, + .Repeat_N: return 3 + case .Optional, + .Optional_Non_Greedy: return 4 + case .Open_Paren, + .Open_Paren_Non_Capture: return 9 + } + return 0 +} + + +Expected_Token :: struct { + pos: int, + kind: Token_Kind, +} + +Invalid_Repetition :: struct { + pos: int, +} + +Invalid_Token :: struct { + pos: int, + kind: Token_Kind, +} + +Invalid_Unicode :: struct { + pos: int, +} + +Too_Many_Capture_Groups :: struct { + pos: int, +} + +Unexpected_EOF :: struct { + pos: int, +} + +Error :: union { + Expected_Token, + Invalid_Repetition, + Invalid_Token, + Invalid_Unicode, + Too_Many_Capture_Groups, + Unexpected_EOF, +} + + +Parser :: struct { + flags: common.Flags, + t: Tokenizer, + + cur_token: Token, + + groups: int, +} + + +@require_results +advance :: proc(p: ^Parser) -> Error { + p.cur_token = tokenizer.scan(&p.t) + if p.cur_token.kind == .Invalid { + return Invalid_Unicode { pos = 0 } + } + return nil +} + +expect :: proc(p: ^Parser, kind: Token_Kind) -> (err: Error) { + if p.cur_token.kind == kind { + advance(p) or_return + return + } + + return Expected_Token{ + pos = p.t.offset, + kind = kind, + } +} + +null_denotation :: proc(p: ^Parser, token: Token) -> (result: Node, err: Error) { + #partial switch token.kind { + case .Rune: + r: rune + for ru in token.text { + r = ru + break + } + assert(r != 0, "Parsed an empty Rune token.") + + if .Case_Insensitive in p.flags { + lower := unicode.to_lower(r) + upper := unicode.to_upper(r) + if lower != upper { + node := new(Node_Rune_Class) + append(&node.runes, lower) + append(&node.runes, upper) + return node, nil + } + } + + node := new(Node_Rune) + node ^= { r } + return node, nil + + case .Rune_Class: + if len(token.text) == 0 { + return nil, nil + } + + node := new(Node_Rune_Class) + + for i := 0; i < len(token.text); /**/ { + r, size := utf8.decode_rune(token.text[i:]) + if i == 0 && r == '^' { + node.negating = true + i += size + continue + } + i += size + + assert(size > 0, "RegEx tokenizer passed an incomplete Rune_Class to the parser.") + + if r == '\\' { + next_r, next_size := utf8.decode_rune(token.text[i:]) + i += next_size + assert(next_size > 0, "RegEx tokenizer passed an incomplete Rune_Class to the parser.") + + // @MetaCharacter + // NOTE: These must be kept in sync with the tokenizer. + switch next_r { + case 'f': append(&node.runes, '\f') + case 'n': append(&node.runes, '\n') + case 'r': append(&node.runes, '\r') + case 't': append(&node.runes, '\t') + + case 'd': + append(&node.ranges, Rune_Class_Range{ '0', '9' }) + case 's': + append(&node.runes, '\t') + append(&node.runes, '\n') + append(&node.runes, '\f') + append(&node.runes, '\r') + append(&node.runes, ' ') + case 'w': + append(&node.ranges, Rune_Class_Range{ '0', '9' }) + append(&node.ranges, Rune_Class_Range{ 'A', 'Z' }) + append(&node.runes, '_') + append(&node.ranges, Rune_Class_Range{ 'a', 'z' }) + case 'D': + append(&node.ranges, Rune_Class_Range{ 0, '0' - 1 }) + append(&node.ranges, Rune_Class_Range{ '9' + 1, max(rune) }) + case 'S': + append(&node.ranges, Rune_Class_Range{ 0, '\t' - 1 }) + // \t and \n are adjacent. + append(&node.runes, '\x0b') // Vertical Tab + append(&node.ranges, Rune_Class_Range{ '\r' + 1, ' ' - 1 }) + append(&node.ranges, Rune_Class_Range{ ' ' + 1, max(rune) }) + case 'W': + append(&node.ranges, Rune_Class_Range{ 0, '0' - 1 }) + append(&node.ranges, Rune_Class_Range{ '9' + 1, 'A' - 1 }) + append(&node.ranges, Rune_Class_Range{ 'Z' + 1, '_' - 1 }) + append(&node.ranges, Rune_Class_Range{ '_' + 1, 'a' - 1 }) + append(&node.ranges, Rune_Class_Range{ 'z' + 1, max(rune) }) + case: + append(&node.runes, next_r) + } + continue + } + + if r == '-' && len(node.runes) > 0 { + next_r, next_size := utf8.decode_rune(token.text[i:]) + if next_size > 0 { + last := pop(&node.runes) + i += next_size + + append(&node.ranges, Rune_Class_Range{ last, next_r }) + continue + } + } + + append(&node.runes, r) + } + + if .Case_Insensitive in p.flags { + length := len(node.runes) + #no_bounds_check for i := 0; i < length; i += 1 { + r := node.runes[i] + lower := unicode.to_lower(r) + upper := unicode.to_upper(r) + + if lower != upper { + if lower != r { + append(&node.runes, lower) + } else { + append(&node.runes, upper) + } + } + } + + length = len(node.ranges) + #no_bounds_check for i := 0; i < length; i += 1 { + range := &node.ranges[i] + + min_lower := unicode.to_lower(range.lower) + max_lower := unicode.to_lower(range.upper) + + min_upper := unicode.to_upper(range.lower) + max_upper := unicode.to_upper(range.upper) + + if min_lower != min_upper && max_lower != max_upper { + range.lower = min_lower + range.upper = max_lower + append(&node.ranges, Rune_Class_Range{ min_upper, max_upper }) + } + } + } + + result = node + + case .Wildcard: + node := new(Node_Wildcard) + result = node + + case .Open_Paren: + // Because of the recursive nature of the token parser, we take the + // group number first instead of afterwards, in order to construct + // group matches from the outside in. + p.groups += 1 + if p.groups == common.MAX_CAPTURE_GROUPS { + return nil, Too_Many_Capture_Groups{ pos = token.pos } + } + this_group := p.groups + + node := new(Node_Group) + node.capture = true + node.capture_id = this_group + + node.inner = parse_expression(p, 0) or_return + expect(p, .Close_Paren) or_return + result = node + case .Open_Paren_Non_Capture: + node := new(Node_Group) + node.inner = parse_expression(p, 0) or_return + expect(p, .Close_Paren) or_return + result = node + case .Close_Paren: + node := new(Node_Rune) + node ^= { ')' } + return node, nil + + case .Anchor_Start: + node := new(Node_Anchor) + node.start = true + result = node + case .Anchor_End: + node := new(Node_Anchor) + result = node + case .Word_Boundary: + node := new(Node_Word_Boundary) + result = node + case .Non_Word_Boundary: + node := new(Node_Word_Boundary) + node.non_word = true + result = node + + case .Alternate: + // A unary alternation with a left-side empty path, i.e. `|a`. + right, right_err := parse_expression(p, left_binding_power(.Alternate)) + #partial switch specific in right_err { + case Unexpected_EOF: + // This token is a NOP, i.e. `|`. + break + case nil: + break + case: + return nil, right_err + } + + node := new(Node_Alternation) + node.right = right + result = node + + case .EOF: + return nil, Unexpected_EOF{ pos = token.pos } + + case: + return nil, Invalid_Token{ pos = token.pos, kind = token.kind } + } + + return +} + +left_denotation :: proc(p: ^Parser, token: Token, left: Node) -> (result: Node, err: Error) { + #partial switch token.kind { + case .Alternate: + if p.cur_token.kind == .Close_Paren { + // `(a|)` + // parse_expression will fail, so intervene here. + node := new(Node_Alternation) + node.left = left + return node, nil + } + + right, right_err := parse_expression(p, left_binding_power(.Alternate)) + + #partial switch specific in right_err { + case nil: + break + case Unexpected_EOF: + // EOF is okay in an alternation; it's an edge case in the way of + // expressing an optional such as `a|`. + break + case: + return nil, right_err + } + + node := new(Node_Alternation) + node.left = left + node.right = right + result = node + + case .Concatenate: + right := parse_expression(p, left_binding_power(.Concatenate)) or_return + + // There should be no need to check if right is Node_Concatenation, due + // to how the parsing direction works. + #partial switch specific in left { + case ^Node_Concatenation: + append(&specific.nodes, right) + result = specific + case: + node := new(Node_Concatenation) + append(&node.nodes, left) + append(&node.nodes, right) + result = node + } + + case .Repeat_Zero: + node := new(Node_Repeat_Zero) + node.inner = left + result = node + case .Repeat_Zero_Non_Greedy: + node := new(Node_Repeat_Zero_Non_Greedy) + node.inner = left + result = node + case .Repeat_One: + node := new(Node_Repeat_One) + node.inner = left + result = node + case .Repeat_One_Non_Greedy: + node := new(Node_Repeat_One_Non_Greedy) + node.inner = left + result = node + + case .Repeat_N: + node := new(Node_Repeat_N) + node.inner = left + + comma := strings.index_byte(token.text, ',') + + switch comma { + case -1: // {N} + exact, ok := strconv.parse_u64_of_base(token.text, base = 10) + if !ok { + return nil, Invalid_Repetition{ pos = token.pos } + } + if exact == 0 { + return nil, Invalid_Repetition{ pos = token.pos } + } + + node.lower = cast(int)exact + node.upper = cast(int)exact + + case 0: // {,M} + upper, ok := strconv.parse_u64_of_base(token.text[1:], base = 10) + if !ok { + return nil, Invalid_Repetition{ pos = token.pos } + } + if upper == 0 { + return nil, Invalid_Repetition{ pos = token.pos } + } + + node.lower = -1 + node.upper = cast(int)upper + + case len(token.text) - 1: // {N,} + lower, ok := strconv.parse_u64_of_base(token.text[:comma], base = 10) + if !ok { + return nil, Invalid_Repetition{ pos = token.pos } + } + + node.lower = cast(int)lower + node.upper = -1 + + case: // {N,M} + lower, lower_ok := strconv.parse_u64_of_base(token.text[:comma], base = 10) + if !lower_ok { + return nil, Invalid_Repetition{ pos = token.pos } + } + upper, upper_ok := strconv.parse_u64_of_base(token.text[comma+1:], base = 10) + if !upper_ok { + return nil, Invalid_Repetition{ pos = token.pos } + } + if lower > upper { + return nil, Invalid_Repetition{ pos = token.pos } + } + if upper == 0 { + return nil, Invalid_Repetition{ pos = token.pos } + } + + node.lower = cast(int)lower + node.upper = cast(int)upper + } + + result = node + + case .Optional: + node := new(Node_Optional) + node.inner = left + result = node + case .Optional_Non_Greedy: + node := new(Node_Optional_Non_Greedy) + node.inner = left + result = node + + case .EOF: + return nil, Unexpected_EOF{ pos = token.pos } + + case: + return nil, Invalid_Token{ pos = token.pos, kind = token.kind } + } + + return +} + +parse_expression :: proc(p: ^Parser, rbp: int) -> (result: Node, err: Error) { + token := p.cur_token + + advance(p) or_return + left := null_denotation(p, token) or_return + + token = p.cur_token + for rbp < left_binding_power(token.kind) { + advance(p) or_return + left = left_denotation(p, token, left) or_return + token = p.cur_token + } + + return left, nil +} + +parse :: proc(str: string, flags: common.Flags) -> (result: Node, err: Error) { + if len(str) == 0 { + node := new(Node_Group) + return node, nil + } + + p: Parser + p.flags = flags + + tokenizer.init(&p.t, str, flags) + + p.cur_token = tokenizer.scan(&p.t) + if p.cur_token.kind == .Invalid { + return nil, Invalid_Unicode { pos = 0 } + } + + node := parse_expression(&p, 0) or_return + result = node + + return +} diff --git a/core/text/regex/regex.odin b/core/text/regex/regex.odin new file mode 100644 index 000000000..1736f2305 --- /dev/null +++ b/core/text/regex/regex.odin @@ -0,0 +1,434 @@ +package regex + +import "core:text/regex/common" +import "core:text/regex/compiler" +import "core:text/regex/optimizer" +import "core:text/regex/parser" +import "core:text/regex/virtual_machine" + +Flag :: common.Flag +Flags :: common.Flags +Parser_Error :: parser.Error +Compiler_Error :: compiler.Error + +Creation_Error :: enum { + None, + Bad_Delimiter, + Expected_Delimiter, + Unknown_Flag, +} + +Error :: union #shared_nil { + Parser_Error, + Compiler_Error, + Creation_Error, +} + +Capture :: struct { + pos: [][2]int, + groups: []string, +} + +Regular_Expression :: struct { + original_pattern: string, + flags: Flags, + class_data: []virtual_machine.Rune_Class_Data, + program: []virtual_machine.Opcode `fmt:"-"`, +} + + +@(rodata) +Flag_To_Letter := #sparse[Flag]u8 { + .Global = 'g', + .Multiline = 'm', + .Case_Insensitive = 'i', + .Ignore_Whitespace = 'x', + .Unicode = 'u', + .No_Capture = 'n', + .No_Optimization = '-', +} + +/* +Create a regular expression from a string pattern and a set of flags. + +*Allocates Using Provided Allocators* + +Inputs: +- pattern: The pattern to compile. +- flags: A `bit_set` of RegEx flags. +- permanent_allocator: The allocator to use for the final regular expression. (default: context.allocator) +- temporary_allocator: The allocator to use for the intermediate compilation stages. (default: context.temp_allocator) + +Returns: +- result: The regular expression. +- err: An error, if one occurred. +*/ +@require_results +create :: proc( + pattern: string, + flags: Flags = {}, + permanent_allocator := context.allocator, + temporary_allocator := context.temp_allocator, +) -> (result: Regular_Expression, err: Error) { + + // For the sake of speed and simplicity, we first run all the intermediate + // processes such as parsing and compilation through the temporary + // allocator. + program: [dynamic]virtual_machine.Opcode = --- + class_data: [dynamic]parser.Rune_Class_Data = --- + { + context.allocator = temporary_allocator + + ast := parser.parse(pattern, flags) or_return + + if .No_Optimization not_in flags { + ast, _ = optimizer.optimize(ast, flags) + } + + program, class_data = compiler.compile(ast, flags) or_return + } + + // When that's successful, re-allocate all at once with the permanent + // allocator so everything can be tightly packed. + context.allocator = permanent_allocator + + result.original_pattern = pattern + result.flags = flags + + if len(class_data) > 0 { + result.class_data = make([]virtual_machine.Rune_Class_Data, len(class_data)) + } + for data, i in class_data { + if len(data.runes) > 0 { + result.class_data[i].runes = make([]rune, len(data.runes)) + copy(result.class_data[i].runes, data.runes[:]) + } + if len(data.ranges) > 0 { + result.class_data[i].ranges = make([]virtual_machine.Rune_Class_Range, len(data.ranges)) + copy(result.class_data[i].ranges, data.ranges[:]) + } + } + + result.program = make([]virtual_machine.Opcode, len(program)) + copy(result.program, program[:]) + + return +} + +/* +Create a regular expression from a delimited string pattern, such as one +provided by users of a program or those found in a configuration file. + +They are in the form of: + + [DELIMITER] [regular expression] [DELIMITER] [flags] + +For example, the following strings are valid: + + /hellope/i + #hellope#i + •hellope•i + つhellopeつi + +The delimiter is determined by the very first rune in the string. +The only restriction is that the delimiter cannot be `\`, as that rune is used +to escape the delimiter if found in the middle of the string. + +All runes after the closing delimiter will be parsed as flags: + +- 'g': Global +- 'm': Multiline +- 'i': Case_Insensitive +- 'x': Ignore_Whitespace +- 'u': Unicode +- 'n': No_Capture +- '-': No_Optimization + + +*Allocates Using Provided Allocators* + +Inputs: +- pattern: The delimited pattern with optional flags to compile. +- str: The string to match against. +- permanent_allocator: The allocator to use for the final regular expression. (default: context.allocator) +- temporary_allocator: The allocator to use for the intermediate compilation stages. (default: context.temp_allocator) + +Returns: +- result: The regular expression. +- err: An error, if one occurred. +*/ +@require_results +create_by_user :: proc( + pattern: string, + permanent_allocator := context.allocator, + temporary_allocator := context.temp_allocator, +) -> (result: Regular_Expression, err: Error) { + + if len(pattern) == 0 { + err = .Expected_Delimiter + return + } + + delimiter: rune + start := -1 + end := -1 + + flags: Flags + + escaping: bool + parse_loop: for r, i in pattern { + if delimiter == 0 { + if r == '\\' { + err = .Bad_Delimiter + return + } + delimiter = r + continue parse_loop + } + + if start == -1 { + start = i + } + + if escaping { + escaping = false + continue parse_loop + } + + switch r { + case '\\': + escaping = true + case delimiter: + end = i + break parse_loop + } + } + + if end == -1 { + err = .Expected_Delimiter + return + } + + // `start` is also the size of the delimiter, which is why it's being added + // to `end` here. + for r in pattern[start + end:] { + switch r { + case 'g': flags += { .Global } + case 'm': flags += { .Multiline } + case 'i': flags += { .Case_Insensitive } + case 'x': flags += { .Ignore_Whitespace } + case 'u': flags += { .Unicode } + case 'n': flags += { .No_Capture } + case '-': flags += { .No_Optimization } + case: + err = .Unknown_Flag + return + } + } + + return create(pattern[start:end], flags, permanent_allocator, temporary_allocator) +} + +/* +Match a regular expression against a string and allocate the results into the +returned `capture` structure. + +The resulting capture strings will be slices to the string `str`, not wholly +copied strings, so they won't need to be individually deleted. + +*Allocates Using Provided Allocators* + +Inputs: +- regex: The regular expression. +- str: The string to match against. +- permanent_allocator: The allocator to use for the capture results. (default: context.allocator) +- temporary_allocator: The allocator to use for the virtual machine. (default: context.temp_allocator) + +Returns: +- capture: The capture groups found in the string. +- success: True if the regex matched the string. +*/ +@require_results +match_and_allocate_capture :: proc( + regex: Regular_Expression, + str: string, + permanent_allocator := context.allocator, + temporary_allocator := context.temp_allocator, +) -> (capture: Capture, success: bool) { + + saved: ^[2 * common.MAX_CAPTURE_GROUPS]int + + { + context.allocator = temporary_allocator + + vm := virtual_machine.create(regex.program, str) + vm.class_data = regex.class_data + + if .Unicode in regex.flags { + saved, success = virtual_machine.run(&vm, true) + } else { + saved, success = virtual_machine.run(&vm, false) + } + } + + if saved != nil { + context.allocator = permanent_allocator + + num_groups := 0 + for i := 0; i < len(saved); i += 2 { + a, b := saved[i], saved[i + 1] + if a == -1 || b == -1 { + continue + } + num_groups += 1 + } + + if num_groups > 0 { + capture.groups = make([]string, num_groups) + capture.pos = make([][2]int, num_groups) + n := 0 + + #no_bounds_check for i := 0; i < len(saved); i += 2 { + a, b := saved[i], saved[i + 1] + if a == -1 || b == -1 { + continue + } + + capture.groups[n] = str[a:b] + capture.pos[n] = {a, b} + n += 1 + } + } + } + + return +} + +/* +Match a regular expression against a string and save the capture results into +the provided `capture` structure. + +The resulting capture strings will be slices to the string `str`, not wholly +copied strings, so they won't need to be individually deleted. + +*Allocates Using Provided Allocator* + +Inputs: +- regex: The regular expression. +- str: The string to match against. +- capture: A pointer to a Capture structure with `groups` and `pos` already allocated. +- temporary_allocator: The allocator to use for the virtual machine. (default: context.temp_allocator) + +Returns: +- num_groups: The number of capture groups set into `capture`. +- success: True if the regex matched the string. +*/ +@require_results +match_with_preallocated_capture :: proc( + regex: Regular_Expression, + str: string, + capture: ^Capture, + temporary_allocator := context.temp_allocator, +) -> (num_groups: int, success: bool) { + + assert(capture != nil, "Pre-allocated RegEx capture must not be nil.") + assert(len(capture.groups) >= common.MAX_CAPTURE_GROUPS, + "Pre-allocated RegEx capture `groups` must be at least 10 elements long.") + assert(len(capture.pos) >= common.MAX_CAPTURE_GROUPS, + "Pre-allocated RegEx capture `pos` must be at least 10 elements long.") + + saved: ^[2 * common.MAX_CAPTURE_GROUPS]int + + { + context.allocator = temporary_allocator + + vm := virtual_machine.create(regex.program, str) + vm.class_data = regex.class_data + + if .Unicode in regex.flags { + saved, success = virtual_machine.run(&vm, true) + } else { + saved, success = virtual_machine.run(&vm, false) + } + } + + if saved != nil { + n := 0 + + #no_bounds_check for i := 0; i < len(saved); i += 2 { + a, b := saved[i], saved[i + 1] + if a == -1 || b == -1 { + continue + } + + capture.groups[n] = str[a:b] + capture.pos[n] = {a, b} + n += 1 + } + } + + return +} + +match :: proc { + match_and_allocate_capture, + match_with_preallocated_capture, +} + +/* +Allocate a `Capture` in advance for use with `match`. This can save some time +if you plan on performing several matches at once and only need the results +between matches. + +Inputs: +- allocator: (default: context.allocator) + +Returns: +- result: The `Capture` with the maximum number of groups allocated. +*/ +@require_results +preallocate_capture :: proc(allocator := context.allocator) -> (result: Capture) { + context.allocator = allocator + result.pos = make([][2]int, common.MAX_CAPTURE_GROUPS) + result.groups = make([]string, common.MAX_CAPTURE_GROUPS) + return +} + +/* +Free all data allocated by the `create*` procedures. + +*Frees Using Provided Allocator* + +Inputs: +- regex: A regular expression. +- allocator: (default: context.allocator) +*/ +destroy_regex :: proc(regex: Regular_Expression, allocator := context.allocator) { + context.allocator = allocator + delete(regex.program) + for data in regex.class_data { + delete(data.runes) + delete(data.ranges) + } + delete(regex.class_data) +} + +/* +Free all data allocated by the `match_and_allocate_capture` procedure. + +*Frees Using Provided Allocator* + +Inputs: +- capture: A Capture. +- allocator: (default: context.allocator) +*/ +destroy_capture :: proc(capture: Capture, allocator := context.allocator) { + context.allocator = allocator + delete(capture.groups) + delete(capture.pos) +} + +destroy :: proc { + destroy_regex, + destroy_capture, +} diff --git a/core/text/regex/tokenizer/tokenizer.odin b/core/text/regex/tokenizer/tokenizer.odin new file mode 100644 index 000000000..2702c5434 --- /dev/null +++ b/core/text/regex/tokenizer/tokenizer.odin @@ -0,0 +1,349 @@ +package regex_tokenizer + +import "core:text/regex/common" +import "core:unicode/utf8" + +Token_Kind :: enum { + Invalid, + EOF, + + Rune, + Wildcard, + + Alternate, + + Concatenate, + + Repeat_Zero, + Repeat_Zero_Non_Greedy, + Repeat_One, + Repeat_One_Non_Greedy, + + Repeat_N, + + Optional, + Optional_Non_Greedy, + + Rune_Class, + + Open_Paren, + Open_Paren_Non_Capture, + Close_Paren, + + Anchor_Start, + Anchor_End, + + Word_Boundary, + Non_Word_Boundary, +} + +Token :: struct { + kind: Token_Kind, + text: string, + pos: int, +} + +Tokenizer :: struct { + flags: common.Flags, + src: string, + + ch: rune, + offset: int, + read_offset: int, + + last_token_kind: Token_Kind, + held_token: Token, + error_state: Error, + paren_depth: int, +} + +Error :: enum { + None, + Illegal_Null_Character, + Illegal_Codepoint, + Illegal_Byte_Order_Mark, +} + +init :: proc(t: ^Tokenizer, str: string, flags: common.Flags) { + t.src = str + t.flags = flags + t.error_state = advance_rune(t) +} + +peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte { + if t.read_offset+offset < len(t.src) { + return t.src[t.read_offset+offset] + } + return 0 +} + +advance_rune :: proc(t: ^Tokenizer) -> (err: Error) { + if t.error_state != nil { + return t.error_state + } + + if t.read_offset < len(t.src) { + t.offset = t.read_offset + r, w := rune(t.src[t.read_offset]), 1 + switch { + case r == 0: + err = .Illegal_Null_Character + case r >= utf8.RUNE_SELF: + r, w = utf8.decode_rune(t.src[t.read_offset:]) + if r == utf8.RUNE_ERROR && w == 1 { + err = .Illegal_Codepoint + } else if r == utf8.RUNE_BOM && t.offset > 0 { + err = .Illegal_Byte_Order_Mark + } + } + t.read_offset += w + t.ch = r + } else { + t.offset = len(t.src) + t.ch = -1 + } + + t.error_state = err + + return +} + +@require_results +scan_class :: proc(t: ^Tokenizer) -> (str: string, ok: bool) { + start := t.read_offset + + for { + advance_rune(t) + if t.ch == -1 || t.error_state != nil { + return "", false + } + + if t.ch == '\\' { + advance_rune(t) + continue + } + + if t.ch == ']' { + return t.src[start:t.offset], true + } + } + + unreachable() +} + +@require_results +scan_repeat :: proc(t: ^Tokenizer) -> (str: string, ok: bool) { + start := t.read_offset + + for { + advance_rune(t) + if t.ch == -1 { + return "", false + } + if t.ch == '}' { + return t.src[start:t.offset], true + } + } + + unreachable() +} + +@require_results +scan_non_greedy :: proc(t: ^Tokenizer) -> bool { + if peek_byte(t) == '?' { + advance_rune(t) + return true + } + + return false +} + +scan_comment :: proc(t: ^Tokenizer) { + for { + advance_rune(t) + switch t.ch { + case -1: + return + case '\n': + // UNIX newline. + advance_rune(t) + return + case '\r': + // Mac newline. + advance_rune(t) + if t.ch == '\n' { + // Windows newline. + advance_rune(t) + } + return + } + } +} + +@require_results +scan_non_capture_group :: proc(t: ^Tokenizer) -> bool { + if peek_byte(t) == '?' && peek_byte(t, 1) == ':' { + advance_rune(t) + advance_rune(t) + return true + } + + return false +} + +@require_results +scan :: proc(t: ^Tokenizer) -> (token: Token) { + kind: Token_Kind + lit: string + pos := t.offset + + defer { + t.last_token_kind = token.kind + } + + if t.error_state != nil { + t.error_state = nil + return { .Invalid, "", pos } + } + + if t.held_token != {} { + popped := t.held_token + t.held_token = {} + + return popped + } + + ch_loop: for { + switch t.ch { + case -1: + return { .EOF, "", pos } + + case '\\': + advance_rune(t) + + if t.ch == -1 { + return { .EOF, "", pos } + } + + pos = t.offset + + // @MetaCharacter + // NOTE: These must be kept in sync with the compiler. + DIGIT_CLASS :: "0-9" + SPACE_CLASS :: "\t\n\f\r " + WORD_CLASS :: "0-9A-Z_a-z" + + switch t.ch { + case 'b': kind = .Word_Boundary + case 'B': kind = .Non_Word_Boundary + + case 'f': kind = .Rune; lit = "\f" + case 'n': kind = .Rune; lit = "\n" + case 'r': kind = .Rune; lit = "\r" + case 't': kind = .Rune; lit = "\t" + + case 'd': kind = .Rune_Class; lit = DIGIT_CLASS + case 's': kind = .Rune_Class; lit = SPACE_CLASS + case 'w': kind = .Rune_Class; lit = WORD_CLASS + case 'D': kind = .Rune_Class; lit = "^" + DIGIT_CLASS + case 'S': kind = .Rune_Class; lit = "^" + SPACE_CLASS + case 'W': kind = .Rune_Class; lit = "^" + WORD_CLASS + case: + kind = .Rune + lit = t.src[t.offset:t.read_offset] + } + + case '.': + kind = .Wildcard + + case '|': kind = .Alternate + + case '*': kind = .Repeat_Zero_Non_Greedy if scan_non_greedy(t) else .Repeat_Zero + case '+': kind = .Repeat_One_Non_Greedy if scan_non_greedy(t) else .Repeat_One + case '?': kind = .Optional_Non_Greedy if scan_non_greedy(t) else .Optional + + case '[': + if text, ok := scan_class(t); ok { + kind = .Rune_Class + lit = text + } else { + return { .EOF, "", pos } + } + + case '{': + if text, ok := scan_repeat(t); ok { + kind = .Repeat_N + lit = text + } else { + return { .EOF, "", pos } + } + + case '(': + kind = .Open_Paren_Non_Capture if scan_non_capture_group(t) else .Open_Paren + t.paren_depth += 1 + case ')': + kind = .Close_Paren + t.paren_depth -= 1 + + case '^': kind = .Anchor_Start + case '$': + kind = .Anchor_End + + case: + if .Ignore_Whitespace in t.flags { + switch t.ch { + case ' ', '\r', '\n', '\t', '\f': + advance_rune(t) + continue ch_loop + case: + break + } + } + if t.ch == '#' && t.paren_depth == 0 { + scan_comment(t) + continue ch_loop + } + + kind = .Rune + lit = t.src[t.offset:t.read_offset] + } + + break ch_loop + } + + if t.error_state != nil { + t.error_state = nil + return { .Invalid, "", pos } + } + + advance_rune(t) + + // The following set of rules dictate where Concatenate tokens are + // automatically inserted. + #partial switch kind { + case + .Close_Paren, + .Alternate, + .Optional, .Optional_Non_Greedy, + .Repeat_Zero, .Repeat_Zero_Non_Greedy, + .Repeat_One, .Repeat_One_Non_Greedy, + .Repeat_N: + // Never prepend a Concatenate before these tokens. + break + case: + #partial switch t.last_token_kind { + case + .Invalid, + .Open_Paren, .Open_Paren_Non_Capture, + .Alternate: + // Never prepend a Concatenate token when the _last token_ was one + // of these. + break + case: + t.held_token = { kind, lit, pos } + return { .Concatenate, "", pos } + } + } + + return { kind, lit, pos } +} diff --git a/core/text/regex/virtual_machine/doc.odin b/core/text/regex/virtual_machine/doc.odin new file mode 100644 index 000000000..1b0694565 --- /dev/null +++ b/core/text/regex/virtual_machine/doc.odin @@ -0,0 +1,175 @@ +/* +package regex_vm implements a threaded virtual machine for interpreting +regular expressions, based on the designs described by Russ Cox and attributed +to both Ken Thompson and Rob Pike. + +The virtual machine executes all threads in lock step, i.e. the string pointer +does not advance until all threads have finished processing the current rune. +The algorithm does not look backwards. + +Threads merge when splitting or jumping to positions already visited by another +thread, based on the observation that each thread having visited one PC +(Program Counter) state will execute identically to the previous thread. + +Each thread keeps a save state of its capture groups, and thread priority is +used to allow higher precedence operations to complete first with correct save +states, such as greedy versus non-greedy repetition. + +For more information, see: https://swtch.com/~rsc/regexp/regexp2.html + + +**Implementation Details:** + +- Each opcode is 8 bits in size, and most instructions have no operands. + +- All operands larger than `u8` are read in system endian order. + +- Jump and Split instructions operate on absolute positions in `u16` operands. + +- Classes such as `[0-9]` are stored in a RegEx-specific slice of structs which + are then dereferenced by a `u8` index from the `Rune_Class` instructions. + +- Each Byte and Rune opcode have their operands stored inline after the opcode, + sized `u8` and `i32` respectively. + +- A bitmap is used to determine which PC positions are occupied by a thread to + perform merging. The bitmap is cleared with every new frame. + +- The VM supports two modes: ASCII and Unicode, decided by a compile-time + boolean constant argument provided to `run`. The procedure differs only in + string decoding. This was done for the sake of performance. + +- No allocations are ever freed; the VM expects an arena or temporary allocator + to be used in the context preceding it. + + +**Opcode Reference:** + + (0x00) Match + + The terminal opcode which ends a thread. This always comes at the end of + the program. + + (0x01) Match_And_Exit + + A modified version of Match which stops the virtual machine entirely. It is + only compiled for `No_Capture` expressions, as those expressions do not + need to determine which thread may have saved the most appropriate capture + groups. + + (0x02) Byte + + Consumes one byte from the text using its operand, which is also a byte. + + (0x03) Rune + + Consumes one Unicode codepoint from the text using its operand, which is + four bytes long in a system-dependent endian order. + + (0x04) Rune_Class + + Consumes one character (which may be an ASCII byte or Unicode codepoint, + wholly dependent on which mode the virtual machine is running in) from the + text. + + The actual data storing what runes and ranges of runes apply to the class + are stored alongside the program in the Regular_Expression structure and + the operand for this opcode is a single byte which indexes into a + collection of these data structures. + + (0x05) Rune_Class_Negated + + A modified version of Rune_Class that functions the same, save for how it + returns the opposite of what Rune_Class matches. + + (0x06) Wildcard + + Consumes one byte or one Unicode codepoint, depending on the VM mode. + + (0x07) Jump + + Sets the Program Counter of a VM thread to the operand, which is a u16. + This opcode is used to implement Alternation (coming at the end of the left + choice) and Repeat_Zero (to cause the thread to loop backwards). + + (0x08) Split + + Spawns a new thread for the X operand and causes the current thread to jump + to the Y operand. This opcode is used to implement Alternation, all the + Repeat variations, and the Optional nodes. + + Splitting threads is how the virtual machine is able to execute optional + control flow paths, letting it evaluate different possible ways to match + text. + + (0x09) Save + + Saves the current string index to a slot on the thread dictated by the + operand. These values will be used later to reconstruct capture groups. + + (0x0A) Assert_Start + + Asserts that the thread is at the beginning of a string. + + (0x0B) Assert_End + + Asserts that the thread is at the end of a string. + + (0x0C) Assert_Word_Boundary + + Asserts that the thread is on a word boundary, which can be the start or + end of the text. This examines both the current rune and the next rune. + + (0x0D) Assert_Non_Word_Boundary + + A modified version of Assert_Word_Boundary that returns the opposite value. + + (0x0E) Multiline_Open + + This opcode is compiled in only when the `Multiline` flag is present, and + it replaces both `^` and `$` text anchors. + + It asserts that either the current thread is on one of the string + boundaries, or it consumes a `\n` or `\r` character. + + If a `\r` character is consumed, the PC will be advanced to the sibling + `Multiline_Close` opcode to optionally consume a `\n` character on the next + frame. + + (0x0F) Multiline_Close + + This opcode is always present after `Multiline_Open`. + + It handles consuming the second half of a complete newline, if necessary. + For example, Windows newlines are represented by the characters `\r\n`, + whereas UNIX newlines are `\n` and Macintosh newlines are `\r`. + + (0x10) Wait_For_Byte + (0x11) Wait_For_Rune + (0x12) Wait_For_Rune_Class + (0x13) Wait_For_Rune_Class_Negated + + These opcodes are an optimization around restarting threads on failed + matches when the beginning to a pattern is predictable and the Global flag + is set. + + They will cause the VM to wait for the next rune to match before splitting, + as would happen in the un-optimized version. + + (0x14) Match_All_And_Escape + + This opcode is an optimized version of `.*$` or `.+$` that causes the + active thread to immediately work on escaping the program by following all + Jumps out to the end. + + While running through the rest of the program, the thread will trigger on + every Save instruction it passes to store the length of the string. + + This way, any time a program hits one of these `.*$` constructs, the + virtual machine can exit early, vastly improving processing times. + + Be aware, this opcode is not compiled in if the `Multiline` flag is on, as + the meaning of `$` changes with that flag. + +*/ +package regex_vm diff --git a/core/text/regex/virtual_machine/util.odin b/core/text/regex/virtual_machine/util.odin new file mode 100644 index 000000000..edf055bc7 --- /dev/null +++ b/core/text/regex/virtual_machine/util.odin @@ -0,0 +1,73 @@ +package regex_vm + +Opcode_Iterator :: struct { + code: Program, + pc: int, +} + +iterate_opcodes :: proc(iter: ^Opcode_Iterator) -> (opcode: Opcode, pc: int, ok: bool) { + if iter.pc >= len(iter.code) { + return + } + + opcode = iter.code[iter.pc] + pc = iter.pc + ok = true + + switch opcode { + case .Match: iter.pc += size_of(Opcode) + case .Match_And_Exit: iter.pc += size_of(Opcode) + case .Byte: iter.pc += size_of(Opcode) + size_of(u8) + case .Rune: iter.pc += size_of(Opcode) + size_of(rune) + case .Rune_Class: iter.pc += size_of(Opcode) + size_of(u8) + case .Rune_Class_Negated: iter.pc += size_of(Opcode) + size_of(u8) + case .Wildcard: iter.pc += size_of(Opcode) + case .Jump: iter.pc += size_of(Opcode) + size_of(u16) + case .Split: iter.pc += size_of(Opcode) + 2 * size_of(u16) + case .Save: iter.pc += size_of(Opcode) + size_of(u8) + case .Assert_Start: iter.pc += size_of(Opcode) + case .Assert_End: iter.pc += size_of(Opcode) + case .Assert_Word_Boundary: iter.pc += size_of(Opcode) + case .Assert_Non_Word_Boundary: iter.pc += size_of(Opcode) + case .Multiline_Open: iter.pc += size_of(Opcode) + case .Multiline_Close: iter.pc += size_of(Opcode) + case .Wait_For_Byte: iter.pc += size_of(Opcode) + size_of(u8) + case .Wait_For_Rune: iter.pc += size_of(Opcode) + size_of(rune) + case .Wait_For_Rune_Class: iter.pc += size_of(Opcode) + size_of(u8) + case .Wait_For_Rune_Class_Negated: iter.pc += size_of(Opcode) + size_of(u8) + case .Match_All_And_Escape: iter.pc += size_of(Opcode) + case: + panic("Invalid opcode found in RegEx program.") + } + + return +} + +opcode_to_name :: proc(opcode: Opcode) -> (str: string) { + switch opcode { + case .Match: str = "Match" + case .Match_And_Exit: str = "Match_And_Exit" + case .Byte: str = "Byte" + case .Rune: str = "Rune" + case .Rune_Class: str = "Rune_Class" + case .Rune_Class_Negated: str = "Rune_Class_Negated" + case .Wildcard: str = "Wildcard" + case .Jump: str = "Jump" + case .Split: str = "Split" + case .Save: str = "Save" + case .Assert_Start: str = "Assert_Start" + case .Assert_End: str = "Assert_End" + case .Assert_Word_Boundary: str = "Assert_Word_Boundary" + case .Assert_Non_Word_Boundary: str = "Assert_Non_Word_Boundary" + case .Multiline_Open: str = "Multiline_Open" + case .Multiline_Close: str = "Multiline_Close" + case .Wait_For_Byte: str = "Wait_For_Byte" + case .Wait_For_Rune: str = "Wait_For_Rune" + case .Wait_For_Rune_Class: str = "Wait_For_Rune_Class" + case .Wait_For_Rune_Class_Negated: str = "Wait_For_Rune_Class_Negated" + case .Match_All_And_Escape: str = "Match_All_And_Escape" + case: str = "" + } + + return +} diff --git a/core/text/regex/virtual_machine/virtual_machine.odin b/core/text/regex/virtual_machine/virtual_machine.odin new file mode 100644 index 000000000..f92b84ace --- /dev/null +++ b/core/text/regex/virtual_machine/virtual_machine.odin @@ -0,0 +1,638 @@ +package regex_vm + +@require import "core:io" +import "core:text/regex/common" +import "core:text/regex/parser" +import "core:unicode/utf8" + +Rune_Class_Range :: parser.Rune_Class_Range + +// NOTE: This structure differs intentionally from the one in `regex/parser`, +// as this data doesn't need to be a dynamic array once it hits the VM. +Rune_Class_Data :: struct { + runes: []rune, + ranges: []Rune_Class_Range, +} + +Opcode :: enum u8 { + // | [ operands ] + Match = 0x00, // | + Match_And_Exit = 0x01, // | + Byte = 0x02, // | u8 + Rune = 0x03, // | i32 + Rune_Class = 0x04, // | u8 + Rune_Class_Negated = 0x05, // | u8 + Wildcard = 0x06, // | + Jump = 0x07, // | u16 + Split = 0x08, // | u16, u16 + Save = 0x09, // | u8 + Assert_Start = 0x0A, // | + Assert_End = 0x0B, // | + Assert_Word_Boundary = 0x0C, // | + Assert_Non_Word_Boundary = 0x0D, // | + Multiline_Open = 0x0E, // | + Multiline_Close = 0x0F, // | + Wait_For_Byte = 0x10, // | u8 + Wait_For_Rune = 0x11, // | i32 + Wait_For_Rune_Class = 0x12, // | u8 + Wait_For_Rune_Class_Negated = 0x13, // | u8 + Match_All_And_Escape = 0x14, // | +} + +Thread :: struct { + pc: int, + saved: ^[2 * common.MAX_CAPTURE_GROUPS]int, +} + +Program :: []Opcode + +Machine :: struct { + // Program state + memory: string, + class_data: []Rune_Class_Data, + code: Program, + + // Thread state + top_thread: int, + threads: [^]Thread, + next_threads: [^]Thread, + + // The busy map is used to merge threads based on their program counters. + busy_map: []u64, + + // Global state + string_pointer: int, + + current_rune: rune, + current_rune_size: int, + next_rune: rune, + next_rune_size: int, +} + + +// @MetaCharacter +// NOTE: This must be kept in sync with the compiler & tokenizer. +is_word_class :: #force_inline proc "contextless" (r: rune) -> bool { + switch r { + case '0'..='9', 'A'..='Z', '_', 'a'..='z': + return true + case: + return false + } +} + +set_busy_map :: #force_inline proc "contextless" (vm: ^Machine, pc: int) -> bool #no_bounds_check { + slot := cast(u64)pc >> 6 + bit: u64 = 1 << (cast(u64)pc & 0x3F) + if vm.busy_map[slot] & bit > 0 { + return false + } + vm.busy_map[slot] |= bit + return true +} + +check_busy_map :: #force_inline proc "contextless" (vm: ^Machine, pc: int) -> bool #no_bounds_check { + slot := cast(u64)pc >> 6 + bit: u64 = 1 << (cast(u64)pc & 0x3F) + return vm.busy_map[slot] & bit > 0 +} + +add_thread :: proc(vm: ^Machine, saved: ^[2 * common.MAX_CAPTURE_GROUPS]int, pc: int) #no_bounds_check { + if check_busy_map(vm, pc) { + return + } + + saved := saved + pc := pc + + resolution_loop: for { + if !set_busy_map(vm, pc) { + return + } + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "Thread [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "] thinking about ") + io.write_string(common.debug_stream, opcode_to_name(vm.code[pc])) + io.write_rune(common.debug_stream, '\n') + } + + #partial switch vm.code[pc] { + case .Jump: + pc = cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode)])^ + continue + + case .Split: + jmp_x := cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode)])^ + jmp_y := cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode) + size_of(u16)])^ + + add_thread(vm, saved, jmp_x) + pc = jmp_y + continue + + case .Save: + new_saved := new([2 * common.MAX_CAPTURE_GROUPS]int) + new_saved ^= saved^ + saved = new_saved + + index := vm.code[pc + size_of(Opcode)] + sp := vm.string_pointer+vm.current_rune_size + saved[index] = sp + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "Thread [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "] saving state: (slot ") + io.write_int(common.debug_stream, cast(int)index) + io.write_string(common.debug_stream, " = ") + io.write_int(common.debug_stream, sp) + io.write_string(common.debug_stream, ")\n") + } + + pc += size_of(Opcode) + size_of(u8) + continue + + case .Assert_Start: + sp := vm.string_pointer+vm.current_rune_size + if sp == 0 { + pc += size_of(Opcode) + continue + } + case .Assert_End: + sp := vm.string_pointer+vm.current_rune_size + if sp == len(vm.memory) { + pc += size_of(Opcode) + continue + } + case .Multiline_Open: + sp := vm.string_pointer+vm.current_rune_size + if sp == 0 || sp == len(vm.memory) { + if vm.next_rune == '\r' || vm.next_rune == '\n' { + // The VM is currently on a newline at the string boundary, + // so consume the newline next frame. + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved } + vm.top_thread += 1 + } else { + // Skip the `Multiline_Close` opcode. + pc += 2 * size_of(Opcode) + continue + } + } else { + // Not on a string boundary. + // Try to consume a newline next frame in the other opcode loop. + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved } + vm.top_thread += 1 + } + case .Assert_Word_Boundary: + sp := vm.string_pointer+vm.current_rune_size + if sp == 0 || sp == len(vm.memory) { + pc += size_of(Opcode) + continue + } else { + last_rune_is_wc := is_word_class(vm.current_rune) + this_rune_is_wc := is_word_class(vm.next_rune) + + if last_rune_is_wc && !this_rune_is_wc || !last_rune_is_wc && this_rune_is_wc { + pc += size_of(Opcode) + continue + } + } + case .Assert_Non_Word_Boundary: + sp := vm.string_pointer+vm.current_rune_size + if sp != 0 && sp != len(vm.memory) { + last_rune_is_wc := is_word_class(vm.current_rune) + this_rune_is_wc := is_word_class(vm.next_rune) + + if last_rune_is_wc && this_rune_is_wc || !last_rune_is_wc && !this_rune_is_wc { + pc += size_of(Opcode) + continue + } + } + + case .Wait_For_Byte: + operand := cast(rune)vm.code[pc + size_of(Opcode)] + if vm.next_rune == operand { + add_thread(vm, saved, pc + size_of(Opcode) + size_of(u8)) + } + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved } + vm.top_thread += 1 + + case .Wait_For_Rune: + operand := (cast(^rune)&vm.code[pc + size_of(Opcode)])^ + if vm.next_rune == operand { + add_thread(vm, saved, pc + size_of(Opcode) + size_of(rune)) + } + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved } + vm.top_thread += 1 + + case .Wait_For_Rune_Class: + operand := cast(u8)vm.code[pc + size_of(Opcode)] + class_data := vm.class_data[operand] + next_rune := vm.next_rune + + check: { + for r in class_data.runes { + if next_rune == r { + add_thread(vm, saved, pc + size_of(Opcode) + size_of(u8)) + break check + } + } + for range in class_data.ranges { + if range.lower <= next_rune && next_rune <= range.upper { + add_thread(vm, saved, pc + size_of(Opcode) + size_of(u8)) + break check + } + } + } + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved } + vm.top_thread += 1 + + case .Wait_For_Rune_Class_Negated: + operand := cast(u8)vm.code[pc + size_of(Opcode)] + class_data := vm.class_data[operand] + next_rune := vm.next_rune + + check_negated: { + for r in class_data.runes { + if next_rune == r { + break check_negated + } + } + for range in class_data.ranges { + if range.lower <= next_rune && next_rune <= range.upper { + break check_negated + } + } + add_thread(vm, saved, pc + size_of(Opcode) + size_of(u8)) + } + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved } + vm.top_thread += 1 + + case: + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved } + vm.top_thread += 1 + } + + break resolution_loop + } + + return +} + +run :: proc(vm: ^Machine, $UNICODE_MODE: bool) -> (saved: ^[2 * common.MAX_CAPTURE_GROUPS]int, ok: bool) #no_bounds_check { + when UNICODE_MODE { + vm.next_rune, vm.next_rune_size = utf8.decode_rune_in_string(vm.memory) + } else { + if len(vm.memory) > 0 { + vm.next_rune = cast(rune)vm.memory[0] + vm.next_rune_size = 1 + } + } + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "### Adding initial thread.\n") + } + + { + starter_saved := new([2 * common.MAX_CAPTURE_GROUPS]int) + starter_saved ^= -1 + + add_thread(vm, starter_saved, 0) + } + + // `add_thread` adds to `next_threads` by default, but we need to put this + // thread in the current thread buffer. + vm.threads, vm.next_threads = vm.next_threads, vm.threads + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "### VM starting.\n") + defer io.write_string(common.debug_stream, "### VM finished.\n") + } + + for { + for i := 0; i < len(vm.busy_map); i += 1 { + vm.busy_map[i] = 0 + } + + assert(vm.string_pointer <= len(vm.memory), "VM string pointer went out of bounds.") + + current_rune := vm.next_rune + vm.current_rune = current_rune + vm.current_rune_size = vm.next_rune_size + when UNICODE_MODE { + vm.next_rune, vm.next_rune_size = utf8.decode_rune_in_string(vm.memory[vm.string_pointer+vm.current_rune_size:]) + } else { + if vm.string_pointer+size_of(u8) < len(vm.memory) { + vm.next_rune = cast(rune)vm.memory[vm.string_pointer+size_of(u8)] + vm.next_rune_size = size_of(u8) + } else { + vm.next_rune = 0 + vm.next_rune_size = 0 + } + } + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, ">>> Dispatching rune: ") + io.write_encoded_rune(common.debug_stream, current_rune) + io.write_byte(common.debug_stream, '\n') + } + + thread_count := vm.top_thread + vm.top_thread = 0 + thread_loop: for i := 0; i < thread_count; i += 1 { + t := vm.threads[i] + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "Thread [PC:") + common.write_padded_hex(common.debug_stream, t.pc, 4) + io.write_string(common.debug_stream, "] stepping on ") + io.write_string(common.debug_stream, opcode_to_name(vm.code[t.pc])) + io.write_byte(common.debug_stream, '\n') + } + + #partial opcode: switch vm.code[t.pc] { + case .Match: + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "Thread matched!\n") + } + saved = t.saved + ok = true + break thread_loop + + case .Match_And_Exit: + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "Thread matched! (Exiting)\n") + } + return nil, true + + case .Byte: + operand := cast(rune)vm.code[t.pc + size_of(Opcode)] + if current_rune == operand { + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8)) + } + + case .Rune: + operand := (cast(^rune)&vm.code[t.pc + size_of(Opcode)])^ + if current_rune == operand { + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(rune)) + } + + case .Rune_Class: + operand := cast(u8)vm.code[t.pc + size_of(Opcode)] + class_data := vm.class_data[operand] + + for r in class_data.runes { + if current_rune == r { + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8)) + break opcode + } + } + for range in class_data.ranges { + if range.lower <= current_rune && current_rune <= range.upper { + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8)) + break opcode + } + } + + case .Rune_Class_Negated: + operand := cast(u8)vm.code[t.pc + size_of(Opcode)] + class_data := vm.class_data[operand] + for r in class_data.runes { + if current_rune == r { + break opcode + } + } + for range in class_data.ranges { + if range.lower <= current_rune && current_rune <= range.upper { + break opcode + } + } + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8)) + + case .Wildcard: + add_thread(vm, t.saved, t.pc + size_of(Opcode)) + + case .Multiline_Open: + if current_rune == '\n' { + // UNIX newline. + add_thread(vm, t.saved, t.pc + 2 * size_of(Opcode)) + } else if current_rune == '\r' { + if vm.next_rune == '\n' { + // Windows newline. (1/2) + add_thread(vm, t.saved, t.pc + size_of(Opcode)) + } else { + // Mac newline. + add_thread(vm, t.saved, t.pc + 2 * size_of(Opcode)) + } + } + case .Multiline_Close: + if current_rune == '\n' { + // Windows newline. (2/2) + add_thread(vm, t.saved, t.pc + size_of(Opcode)) + } + + case .Wait_For_Byte: + operand := cast(rune)vm.code[t.pc + size_of(Opcode)] + if vm.next_rune == operand { + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8)) + } + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, t.pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = t.pc, saved = t.saved } + vm.top_thread += 1 + + case .Wait_For_Rune: + operand := (cast(^rune)&vm.code[t.pc + size_of(Opcode)])^ + if vm.next_rune == operand { + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(rune)) + } + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, t.pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = t.pc, saved = t.saved } + vm.top_thread += 1 + + case .Wait_For_Rune_Class: + operand := cast(u8)vm.code[t.pc + size_of(Opcode)] + class_data := vm.class_data[operand] + next_rune := vm.next_rune + + check: { + for r in class_data.runes { + if next_rune == r { + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8)) + break check + } + } + for range in class_data.ranges { + if range.lower <= next_rune && next_rune <= range.upper { + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8)) + break check + } + } + } + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, t.pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = t.pc, saved = t.saved } + vm.top_thread += 1 + + case .Wait_For_Rune_Class_Negated: + operand := cast(u8)vm.code[t.pc + size_of(Opcode)] + class_data := vm.class_data[operand] + next_rune := vm.next_rune + + check_negated: { + for r in class_data.runes { + if next_rune == r { + break check_negated + } + } + for range in class_data.ranges { + if range.lower <= next_rune && next_rune <= range.upper { + break check_negated + } + } + add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8)) + } + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "*** New thread added [PC:") + common.write_padded_hex(common.debug_stream, t.pc, 4) + io.write_string(common.debug_stream, "]\n") + } + vm.next_threads[vm.top_thread] = Thread{ pc = t.pc, saved = t.saved } + vm.top_thread += 1 + + case .Match_All_And_Escape: + t.pc += size_of(Opcode) + // The point of this loop is to walk out of wherever this + // opcode lives to the end of the program, while saving the + // index to the length of the string at each pass on the way. + escape_loop: for { + #partial switch vm.code[t.pc] { + case .Match, .Match_And_Exit: + break escape_loop + + case .Jump: + t.pc = cast(int)(cast(^u16)&vm.code[t.pc + size_of(Opcode)])^ + + case .Save: + index := vm.code[t.pc + size_of(Opcode)] + t.saved[index] = len(vm.memory) + t.pc += size_of(Opcode) + size_of(u8) + + case .Match_All_And_Escape: + // Layering these is fine. + t.pc += size_of(Opcode) + + // If the loop has to process any opcode not listed above, + // it means someone did something odd like `a(.*$)b`, in + // which case, just fail. Technically, the expression makes + // no sense. + case: + break opcode + } + } + + saved = t.saved + ok = true + return + + case: + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "Opcode: ") + io.write_int(common.debug_stream, cast(int)vm.code[t.pc]) + io.write_string(common.debug_stream, "\n") + } + panic("Invalid opcode in RegEx thread loop.") + } + } + + vm.threads, vm.next_threads = vm.next_threads, vm.threads + + when common.ODIN_DEBUG_REGEX { + io.write_string(common.debug_stream, "<<< Frame ended. (Threads: ") + io.write_int(common.debug_stream, vm.top_thread) + io.write_string(common.debug_stream, ")\n") + } + + if vm.string_pointer == len(vm.memory) || vm.top_thread == 0 { + break + } + + vm.string_pointer += vm.current_rune_size + } + + return +} + +opcode_count :: proc(code: Program) -> (opcodes: int) { + iter := Opcode_Iterator{ code, 0 } + for _ in iterate_opcodes(&iter) { + opcodes += 1 + } + return +} + +create :: proc(code: Program, str: string) -> (vm: Machine) { + assert(len(code) > 0, "RegEx VM has no instructions.") + + vm.memory = str + vm.code = code + + sizing := len(code) >> 6 + (1 if len(code) & 0x3F > 0 else 0) + assert(sizing > 0) + vm.busy_map = make([]u64, sizing) + + max_possible_threads := max(1, opcode_count(vm.code) - 1) + + vm.threads = make([^]Thread, max_possible_threads) + vm.next_threads = make([^]Thread, max_possible_threads) + + return +} From 730e10bd6f3bc04050a0d3c161c08a6145f61099 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 21 Jul 2024 23:17:18 -0400 Subject: [PATCH 02/43] Support printing `Regular_Expression` in `fmt` --- core/fmt/fmt.odin | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 9aa9c99dc..22ac1cc36 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -9,6 +9,7 @@ import "core:io" import "core:reflect" import "core:strconv" import "core:strings" +import "core:text/regex" import "core:time" import "core:unicode/utf8" @@ -2405,6 +2406,21 @@ fmt_named :: proc(fi: ^Info, v: any, verb: rune, info: runtime.Type_Info_Named) write_padded_number(fi, (ns), 9) io.write_string(fi.writer, " +0000 UTC", &fi.n) return + + case regex.Regular_Expression: + io.write_byte(fi.writer, '/') + for r in a.original_pattern { + if r == '/' { + io.write_string(fi.writer, `\/`) + } else { + io.write_rune(fi.writer, r) + } + } + io.write_byte(fi.writer, '/') + for flag in a.flags { + io.write_byte(fi.writer, regex.Flag_To_Letter[flag]) + } + return } } From 3e49ceb82ab907a3549451358f78bea2ae683e7e Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 21 Jul 2024 23:18:26 -0400 Subject: [PATCH 03/43] Add tests for `core:text/regex` --- tests/core/normal.odin | 1 + .../core/text/regex/test_core_text_regex.odin | 1012 +++++++++++++++++ 2 files changed, 1013 insertions(+) create mode 100644 tests/core/text/regex/test_core_text_regex.odin diff --git a/tests/core/normal.odin b/tests/core/normal.odin index 065090be3..1f34e3292 100644 --- a/tests/core/normal.odin +++ b/tests/core/normal.odin @@ -38,6 +38,7 @@ download_assets :: proc() { @(require) import "strings" @(require) import "text/i18n" @(require) import "text/match" +@(require) import "text/regex" @(require) import "thread" @(require) import "time" @(require) import "unicode" diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin new file mode 100644 index 000000000..da44e6b2d --- /dev/null +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -0,0 +1,1012 @@ +package test_core_text_regex + +import "core:fmt" +import "core:io" +import "core:log" +import "core:strings" +import "core:testing" +import "core:text/regex" +import "core:text/regex/common" +import "core:text/regex/parser" +import "core:text/regex/tokenizer" + + +check_expression_with_flags :: proc(t: ^testing.T, pattern: string, flags: regex.Flags, haystack: string, needles: ..string, loc := #caller_location) { + rex, parse_err := regex.create(pattern, flags) + if !testing.expect_value(t, parse_err, nil, loc = loc) { + log.infof("Failed test's flags were: %v", flags, location = loc) + return + } + defer regex.destroy(rex) + + capture, success := regex.match(rex, haystack) + defer { + delete(capture.groups) + delete(capture.pos) + } + + if len(needles) > 0 { + testing.expect(t, success, "match failed", loc = loc) + } + + matches_aligned := testing.expectf(t, len(needles) == len(capture.groups), + "expected %i match groups, got %i (flags: %w)", + len(needles), len(capture.groups), flags, loc = loc) + + if matches_aligned { + for needle, i in needles { + if !testing.expectf(t, capture.groups[i] == needle, + "match group %i was %q, expected %q (flags: %w)", + i, capture.groups[i], needle, flags, loc = loc) { + } + } + } else { + log.infof("match groups were: %v", capture.groups, location = loc) + } +} + +check_expression :: proc(t: ^testing.T, pattern, haystack: string, needles: ..string, extra_flags: regex.Flags = {}, loc := #caller_location) { + check_expression_with_flags(t, pattern, { .Global } + extra_flags, + haystack, ..needles, loc = loc) + check_expression_with_flags(t, pattern, { .Global, .No_Optimization } + extra_flags, + haystack, ..needles, loc = loc) + check_expression_with_flags(t, pattern, { .Global, .Unicode } + extra_flags, + haystack, ..needles, loc = loc) + check_expression_with_flags(t, pattern, { .Global, .Unicode, .No_Optimization } + extra_flags, + haystack, ..needles, loc = loc) +} + + +@test +test_concatenation :: proc(t: ^testing.T) { + check_expression(t, "abc", "abc", "abc") +} + +@test +test_rune_class :: proc(t: ^testing.T) { + EXPR :: "[abc]" + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "b", "b") + check_expression(t, EXPR, "c", "c") +} + +@test +test_rune_ranges :: proc(t: ^testing.T) { + EXPR :: "0x[0-9A-Fa-f]+" + check_expression(t, EXPR, "0x0065c816", "0x0065c816") +} + +@test +test_rune_range_terminal_dash :: proc(t: ^testing.T) { + { + EXPR :: "[a-]" + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "-", "-") + } + { + EXPR :: "[-a]" + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "-", "-") + } + { + EXPR :: "[-a-]" + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "-", "-") + } + { + EXPR :: "[-]" + check_expression(t, EXPR, "-", "-") + } + { + EXPR :: "[--]" + check_expression(t, EXPR, "-", "-") + } + { + EXPR :: "[---]" + check_expression(t, EXPR, "-", "-") + } +} + +@test +test_rune_range_escaping_class :: proc(t: ^testing.T) { + EXPR :: `[\]a\[\.]` + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "[", "[") + check_expression(t, EXPR, "]", "]") + check_expression(t, EXPR, ".", ".") + check_expression(t, EXPR, "b") +} + +@test +test_negated_rune_class :: proc(t: ^testing.T) { + EXPR :: "[^ac-d]" + check_expression(t, EXPR, "a") + check_expression(t, EXPR, "b", "b") + check_expression(t, EXPR, "e", "e") + check_expression(t, EXPR, "c") + check_expression(t, EXPR, "d") +} + +@test +test_shorthand_classes :: proc(t: ^testing.T) { + EXPR_P :: `\d\w\s` + check_expression(t, EXPR_P, "1a ", "1a ") + check_expression(t, EXPR_P, "a!1") + EXPR_N :: `\D\W\S` + check_expression(t, EXPR_N, "a!1", "a!1") + check_expression(t, EXPR_N, "1a ") +} + +@test +test_shorthand_classes_in_classes :: proc(t: ^testing.T) { + EXPR_P :: `[\d][\w][\s]` + check_expression(t, EXPR_P, "1a ", "1a ") + check_expression(t, EXPR_P, "a!1") + EXPR_NP :: `[^\d][^\w][^\s]` + check_expression(t, EXPR_NP, "a!1", "a!1") + check_expression(t, EXPR_NP, "1a ") + EXPR_N :: `[\D][\W][\S]` + check_expression(t, EXPR_N, "a!1", "a!1") + check_expression(t, EXPR_N, "1a ") + EXPR_NN :: `[^\D][^\W][^\S]` + check_expression(t, EXPR_NN, "1a ", "1a ") + check_expression(t, EXPR_NN, "a!1") +} + +@test +test_mixed_shorthand_class :: proc(t: ^testing.T) { + EXPR_P :: `[\d\s]+` + check_expression(t, EXPR_P, "0123456789 98", "0123456789 98") + check_expression(t, EXPR_P, "!@#$%^&*()_()") + EXPR_NP :: `[^\d\s]+` + check_expression(t, EXPR_NP, "!@#$%^&*()_()", "!@#$%^&*()_()") + check_expression(t, EXPR_NP, "0123456789 98") +} + +@test +test_wildcard :: proc(t: ^testing.T) { + EXPR :: "." + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, ".", ".") +} + +@test +test_alternation :: proc(t: ^testing.T) { + EXPR :: "aa|bb|cc" + check_expression(t, EXPR, "aa", "aa") + check_expression(t, EXPR, "bb", "bb") + check_expression(t, EXPR, "cc", "cc") +} + +@test +test_optional :: proc(t: ^testing.T) { + EXPR :: "a?a?a?aaa" + check_expression(t, EXPR, "aaa", "aaa") +} + +@test +test_repeat_zero :: proc(t: ^testing.T) { + EXPR :: "a*b" + check_expression(t, EXPR, "aaab", "aaab") +} + +@test +test_repeat_one :: proc(t: ^testing.T) { + EXPR :: "a+b" + check_expression(t, EXPR, "aaab", "aaab") +} + +@test +test_greedy :: proc(t: ^testing.T) { + HTML :: "" + + check_expression(t, "<.+>", HTML, HTML) + check_expression(t, "<.*>", HTML, HTML) + + check_expression(t, "aaa?", "aaa", "aaa") +} + +@test +test_non_greedy :: proc(t: ^testing.T) { + HTML :: "" + + check_expression(t, "<.+?>", HTML, "") + check_expression(t, "<.*?>", HTML, "") + + // NOTE: make a comment about optional non-greedy capture groups + check_expression(t, "aaa??", "aaa", "aa") +} + +@test +test_groups :: proc(t: ^testing.T) { + check_expression(t, "a(b)", "ab", /*|*/ "ab", "b") + check_expression(t, "(a)b", "ab", /*|*/ "ab", "a") + check_expression(t, "(a)(b)", "ab", /*|*/ "ab", "a", "b") + + check_expression(t, "(a(b))", "ab", /*|*/ "ab", "ab", "b") + check_expression(t, "((ab))", "ab", /*|*/ "ab", "ab", "ab") + check_expression(t, "((a)b)", "ab", /*|*/ "ab", "ab", "a") + + check_expression(t, "(ab)+", "ababababab", /*|*/ "ababababab", "ab") + check_expression(t, "((ab)+)", "ababababab", /*|*/ "ababababab", "ababababab", "ab") +} + +@test +test_class_group_repeat :: proc(t: ^testing.T) { + EXPR_1 :: "([0-9]:?)+" + EXPR_2 :: "([0-9]+:?)+" + check_expression(t, EXPR_1, "123:456:789", "123:456:789", "9") + check_expression(t, EXPR_2, "123:456:789", "123:456:789", "789") +} + +@test +test_non_capture_group :: proc(t: ^testing.T) { + EXPR :: "(?:a|b)c" + check_expression(t, EXPR, "ac", "ac") + check_expression(t, EXPR, "bc", "bc") + check_expression(t, EXPR, "cc") +} + +@test +test_optional_capture_group :: proc(t: ^testing.T) { + EXPR :: "^(blue|straw)?berry" + check_expression(t, EXPR, "berry", "berry") + check_expression(t, EXPR, "blueberry", "blueberry", "blue") + check_expression(t, EXPR, "strawberry", "strawberry", "straw") + check_expression(t, EXPR, "cranberry") +} + +@test +test_max_capture_groups :: proc(t: ^testing.T) { + EXPR :: "(1)(2)(3)(4)(5)(6)(7)(8)(9)" + check_expression(t, EXPR, "123456789", "123456789", + "1", "2", "3", "4", "5", "6", "7", "8", "9") +} + +@test +test_repetition :: proc(t: ^testing.T) { + { + EXPR :: "^a{3}$" + check_expression(t, EXPR, "aaa", "aaa") + check_expression(t, EXPR, "aaaa") + } + { + EXPR :: "^a{3,5}$" + check_expression(t, EXPR, "aaa", "aaa") + check_expression(t, EXPR, "aaaa", "aaaa") + check_expression(t, EXPR, "aaaaa", "aaaaa") + check_expression(t, EXPR, "aaaaaa") + } + { + EXPR :: "^(?:meow){2}$" + check_expression(t, EXPR, "meow") + check_expression(t, EXPR, "meowmeow", "meowmeow") + check_expression(t, EXPR, "meowmeowmeow") + } + { + EXPR :: "a{2,}" + check_expression(t, EXPR, "a") + check_expression(t, EXPR, "aa", "aa") + check_expression(t, EXPR, "aaa", "aaa") + } + { + EXPR :: "a{,2}" + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "aa", "aa") + check_expression(t, EXPR, "aaa", "aa") + } + { + EXPR :: "^a{3,3}$" + check_expression(t, EXPR, "aa") + check_expression(t, EXPR, "aaa", "aaa") + check_expression(t, EXPR, "aaaa") + } + { + EXPR :: "a{0,}" + check_expression(t, EXPR, "aaa", "aaa") + } +} + +@test +test_repeated_groups :: proc(t: ^testing.T) { + { + EXPR :: "(ab){3}" + check_expression(t, EXPR, "ababab", "ababab", "ab") + } + { + EXPR :: "((?:ab){3})" + check_expression(t, EXPR, "ababab", "ababab", "ababab") + } +} + +@test +test_escaped_newline :: proc(t: ^testing.T) { + EXPR :: `\n[\n]` + check_expression(t, EXPR, "\n\n", "\n\n") +} + +@test +test_anchors :: proc(t: ^testing.T) { + { + EXPR :: "^ab" + check_expression(t, EXPR, "ab", "ab") + check_expression(t, EXPR, "aab") + } + { + EXPR :: "ab$" + check_expression(t, EXPR, "ab", "ab") + check_expression(t, EXPR, "aab", "ab") + } + { + EXPR :: "^ab$" + check_expression(t, EXPR, "ab", "ab") + check_expression(t, EXPR, "aab") + } +} + +@test +test_grouped_anchors :: proc(t: ^testing.T) { + { + EXPR :: "^a|b" + check_expression(t, EXPR, "ab", "a") + check_expression(t, EXPR, "ba", "b") + } + { + EXPR :: "b|c$" + check_expression(t, EXPR, "ac", "c") + check_expression(t, EXPR, "cb", "b") + } + { + EXPR :: "^hellope$|world" + check_expression(t, EXPR, "hellope", "hellope") + check_expression(t, EXPR, "hellope world", "world") + } +} + +@test +test_empty_alternation :: proc(t: ^testing.T) { + { + EXPR :: "(?:a|)b" + check_expression(t, EXPR, "ab", "ab") + check_expression(t, EXPR, "b", "b") + } + { + EXPR :: "(?:|a)b" + check_expression(t, EXPR, "ab", "ab") + check_expression(t, EXPR, "b", "b") + } + { + EXPR :: "|b" + check_expression(t, EXPR, "b", "") + check_expression(t, EXPR, "", "") + } + { + EXPR :: "a|" + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "", "") + } + { + EXPR :: "|" + check_expression(t, EXPR, "a", "") + check_expression(t, EXPR, "", "") + } +} + +@test +test_empty_class :: proc(t: ^testing.T) { + EXPR :: "a[]b" + check_expression(t, EXPR, "ab", "ab") +} + +@test +test_dot_in_class :: proc(t: ^testing.T) { + EXPR :: `[a\..]` + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, ".", ".") + check_expression(t, EXPR, "b") +} + + +@test +test_word_boundaries :: proc(t: ^testing.T) { + STR :: "This is an island." + { + EXPR :: `\bis\b` + check_expression(t, EXPR, STR, "is") + } + { + EXPR :: `\bis\w+` + check_expression(t, EXPR, STR, "island") + } + { + EXPR :: `\w+is\b` + check_expression(t, EXPR, STR, "This") + } + { + EXPR :: `\b\w\w\b` + check_expression(t, EXPR, STR, "is") + } +} + +@test +test_non_word_boundaries :: proc(t: ^testing.T) { + { + EXPR :: `.\B.` + check_expression(t, EXPR, "ab", "ab") + check_expression(t, EXPR, " ", " ") + check_expression(t, EXPR, "a ") + check_expression(t, EXPR, " b") + } + { + EXPR :: `\B.\B` + check_expression(t, EXPR, "a") + check_expression(t, EXPR, "abc", "b") + } + { + EXPR :: `\B.+` + check_expression(t, EXPR, "abc", "bc") + } + { + EXPR :: `.+\B` + check_expression(t, EXPR, "abc", "ab") + } +} + +@test +test_empty_patterns :: proc(t: ^testing.T) { + { + EXPR :: "" + check_expression(t, EXPR, "abc", "") + } + { + EXPR :: "^$" + check_expression(t, EXPR, "", "") + check_expression(t, EXPR, "a") + } +} + +@test +test_unanchored :: proc(t: ^testing.T) { + EXPR :: "ab" + check_expression(t, EXPR, "cab", "ab") +} + +@test +test_affixes :: proc(t: ^testing.T) { + // This test is for the optimizer. + EXPR :: "^(?:samples|ample|sample)$" + check_expression(t, EXPR, "sample", "sample") + check_expression(t, EXPR, "samples", "samples") + check_expression(t, EXPR, "ample", "ample") + check_expression(t, EXPR, "amples") +} + +@test +test_anchored_capture_until_end :: proc(t: ^testing.T) { + // This test is for the optimizer. + { + EXPR :: `^hellope.*$` + check_expression(t, EXPR, "hellope world", "hellope world") + check_expression(t, EXPR, "hellope", "hellope") + check_expression(t, EXPR, "hellope !", "hellope !") + } + { + EXPR :: `^hellope.+$` + check_expression(t, EXPR, "hellope world", "hellope world") + check_expression(t, EXPR, "hellope") + check_expression(t, EXPR, "hellope !", "hellope !") + } + { + EXPR :: `^(aa|bb|cc.+$).*$` + check_expression(t, EXPR, "aa", "aa", "aa") + check_expression(t, EXPR, "bb", "bb", "bb") + check_expression(t, EXPR, "bbaa", "bbaa", "bb") + check_expression(t, EXPR, "cc") + check_expression(t, EXPR, "ccc", "ccc", "ccc") + check_expression(t, EXPR, "cccc", "cccc", "cccc") + } + // This makes sure that the `.*$` / `.*$` optimization doesn't cause + // any issues if someone does something strange like putting it in the + // middle of an expression. + { + EXPR :: `^(a(b.*$)c).*$` + check_expression(t, EXPR, "a") + check_expression(t, EXPR, "ab") + check_expression(t, EXPR, "abc") + } + { + EXPR :: `^(a(b.*$)?c).*$` + check_expression(t, EXPR, "a") + check_expression(t, EXPR, "ab") + check_expression(t, EXPR, "abc") + check_expression(t, EXPR, "ac", "ac", "ac") + check_expression(t, EXPR, "acc", "acc", "ac") + } +} + +@test +test_unicode_explicitly :: proc(t: ^testing.T) { + { + EXPR :: "^....!$" + check_expression_with_flags(t, EXPR, { .Unicode }, + "こにちは!", "こにちは!") + check_expression_with_flags(t, EXPR, { .Unicode, .No_Optimization }, + "こにちは!", "こにちは!") + } + { + EXPR :: "こにちは!" + check_expression_with_flags(t, EXPR, { .Global, .Unicode }, + "Hello こにちは!", "こにちは!") + check_expression_with_flags(t, EXPR, { .Global, .Unicode, .No_Optimization }, + "Hello こにちは!", "こにちは!") + } +} + +@test +test_no_capture_match :: proc(t: ^testing.T) { + EXPR :: "^abc$" + + rex, parse_err := regex.create(EXPR, { .No_Capture }) + if !testing.expect_value(t, parse_err, nil) { + return + } + defer regex.destroy(rex) + + _, matched := regex.match(rex, "abc") + testing.expect(t, matched) +} + +@test +test_comments :: proc(t: ^testing.T) { + EXPR :: `^[abc]# This is a comment. +[def]# This is another comment. +\#$# This is a comment following an escaped '#'.` + check_expression(t, EXPR, "ad#", "ad#") +} + +@test +test_ignore_whitespace :: proc(t: ^testing.T) { + EXPR :: "\f" + ` +\ H e l # Note that the first space on this line is escaped, thus it is not ignored. + l +o p e [ ] w o rld (?: [ ]) ! # Spaces in classes are fine, too. +` + "\r" + + check_expression(t, EXPR, " Hellope world !", " Hellope world !", extra_flags = { .Ignore_Whitespace }) +} + +@test +test_case_insensitive :: proc(t: ^testing.T) { + EXPR :: `hElLoPe [w!][o-P]+rLd!` + check_expression(t, EXPR, "HeLlOpE WoRlD!", "HeLlOpE WoRlD!", extra_flags = { .Case_Insensitive }) +} + +@test +test_multiline :: proc(t: ^testing.T) { + { + EXPR :: `^hellope$world$` + check_expression(t, EXPR, "\nhellope\nworld\n", "\nhellope\nworld\n", extra_flags = { .Multiline }) + check_expression(t, EXPR, "hellope\nworld", "hellope\nworld", extra_flags = { .Multiline }) + check_expression(t, EXPR, "hellope\rworld", "hellope\rworld", extra_flags = { .Multiline }) + check_expression(t, EXPR, "hellope\r\nworld", "hellope\r\nworld", extra_flags = { .Multiline }) + } + { + EXPR :: `^?.$` + check_expression(t, EXPR, "\nh", "\nh", extra_flags = { .Multiline }) + check_expression(t, EXPR, "h", "h", extra_flags = { .Multiline }) + } + { + EXPR :: `^$` + check_expression(t, EXPR, "\n", "\n", extra_flags = { .Multiline }) + check_expression(t, EXPR, "", "", extra_flags = { .Multiline }) + } + { + EXPR :: `$` + check_expression(t, EXPR, "\n", "\n", extra_flags = { .Multiline }) + check_expression(t, EXPR, "", "", extra_flags = { .Multiline }) + } +} + +@test +test_optional_inside_optional :: proc(t: ^testing.T) { + EXPR :: `(?:a?)?` + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "", "") +} + +@test +test_printing :: proc(t: ^testing.T) { + rex, parse_err := regex.create(`^/a$`, { + .Global, + .Multiline, + .Case_Insensitive, + .Unicode, + .Ignore_Whitespace, + .No_Optimization, + .No_Capture, + }) + if !testing.expect_value(t, parse_err, nil) { + return + } + defer regex.destroy(rex) + + str := fmt.tprint(rex) + str_hash := fmt.tprintf("%#v", rex) + testing.expect_value(t, str, `/^\/a$/gmixun-`) + testing.expect_value(t, str_hash, `/^\/a$/gmixun-`) +} + + + +@test +test_error_bad_repetitions :: proc(t: ^testing.T) { + check_repetition_error :: proc(t: ^testing.T, pattern: string, loc := #caller_location) { + rex, err := regex.create(pattern) + regex.destroy(rex) + parse_err, _ := err.(regex.Parser_Error) + _, ok := parse_err.(parser.Invalid_Repetition) + if !ok { + log.errorf("expected error Invalid_Repetition, got %v", parse_err, location = loc) + } + } + + check_repetition_error(t, "a{-1,2}") + check_repetition_error(t, "a{2,1}") + check_repetition_error(t, "a{bc}") + check_repetition_error(t, "a{,-3}") + check_repetition_error(t, "a{d,}") + check_repetition_error(t, "a{}") + check_repetition_error(t, "a{0,0}") + check_repetition_error(t, "a{,0}") + check_repetition_error(t, "a{,}") +} + +@test +test_error_invalid_unicode_in_pattern :: proc(t: ^testing.T) { + rex, err := regex.create("\xC0", { .Unicode }) + regex.destroy(rex) + parse_err := err.(regex.Parser_Error) + _, ok := parse_err.(parser.Invalid_Unicode) + if !ok { + log.errorf("expected error Invalid_Unicode, got %v", parse_err) + } +} + +@test +test_error_invalid_unicode_in_string :: proc(t: ^testing.T) { + EXPR :: "^...$" + // NOTE: Matching on invalid Unicode is currently safe. + // If `utf8.decode_rune` ever changes, this test may fail. + check_expression(t, EXPR, "\xC0\xFF\xFE", "\xC0\xFF\xFE") +} + +@test +test_error_too_many_capture_groups :: proc(t: ^testing.T) { + // NOTE: There are 1 + 9 + 1 capture groups in this pattern. + // Remember the implicit capture group 0. + rex, err := regex.create("(1)(2)(3)(4)(5)(6)(7)(8)(9) (A)") + regex.destroy(rex) + + parse_err, _ := err.(regex.Parser_Error) + _, ok := parse_err.(parser.Too_Many_Capture_Groups) + if !ok { + log.errorf("expected error Too_Many_Capture_Groups, got %v", parse_err) + } +} + +@test +test_error_unclosed_paren :: proc(t: ^testing.T) { + rex, err := regex.create("(Hellope") + regex.destroy(rex) + + parse_err, _ := err.(regex.Parser_Error) + _, ok := parse_err.(parser.Expected_Token) + if !ok { + log.errorf("expected error Expected_Token, got %v", parse_err) + } +} + +@test +test_error_unclosed_class :: proc(t: ^testing.T) { + rex, err := regex.create("[helope") + regex.destroy(rex) + + parse_err, _ := err.(regex.Parser_Error) + _, ok := parse_err.(parser.Unexpected_EOF) + if !ok { + log.errorf("expected error Unexpected_EOF, got %v", parse_err) + } +} + +@test +test_error_invalid_unicode_in_unclosed_class :: proc(t: ^testing.T) { + rex, err := regex.create("[\xC0", { .Unicode }) + regex.destroy(rex) + + parse_err, _ := err.(regex.Parser_Error) + _, ok := parse_err.(parser.Invalid_Unicode) + if !ok { + log.errorf("expected error Invalid_Unicode, got %v", parse_err) + } +} + +@test +test_program_too_big :: proc(t: ^testing.T) { + sb := strings.builder_make() + w := strings.to_writer(&sb) + defer strings.builder_destroy(&sb) + + // Each byte will turn into two bytes for the whole opcode and operand, + // then the compiler will insert 5 more bytes for the Save instructions + // and the Match. + N :: common.MAX_PROGRAM_SIZE/2 - 2 + for _ in 0.. Date: Sun, 21 Jul 2024 23:20:32 -0400 Subject: [PATCH 04/43] Add benchmarks for `core:text/regex` --- tests/benchmark/all.odin | 1 + .../benchmark/text/regex/benchmark_regex.odin | 258 ++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 tests/benchmark/text/regex/benchmark_regex.odin diff --git a/tests/benchmark/all.odin b/tests/benchmark/all.odin index d1b7662e2..36de14278 100644 --- a/tests/benchmark/all.odin +++ b/tests/benchmark/all.odin @@ -2,3 +2,4 @@ package benchmarks @(require) import "crypto" @(require) import "hash" +@(require) import "text/regex" diff --git a/tests/benchmark/text/regex/benchmark_regex.odin b/tests/benchmark/text/regex/benchmark_regex.odin new file mode 100644 index 000000000..cd9812b08 --- /dev/null +++ b/tests/benchmark/text/regex/benchmark_regex.odin @@ -0,0 +1,258 @@ +package benchmark_core_text_regex + +import "core:fmt" +import "core:log" +import "core:math/rand" +import "core:mem" +import "core:testing" +import "core:text/regex" +import "core:time" +import "core:unicode/utf8" + +randomize_ascii :: proc(data: []u8) { + for i in 0.. len(data) - i { + continue + } + + r_data, size := utf8.encode_rune(r) + for j in 0.. Date: Mon, 22 Jul 2024 13:27:06 -0400 Subject: [PATCH 05/43] Add `core:text/regex` to `examples/all` --- examples/all/all_main.odin | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/all/all_main.odin b/examples/all/all_main.odin index d92a6b8c4..71a1ad733 100644 --- a/examples/all/all_main.odin +++ b/examples/all/all_main.odin @@ -127,6 +127,7 @@ import testing "core:testing" import edit "core:text/edit" import i18n "core:text/i18n" import match "core:text/match" +import regex "core:text/regex" import scanner "core:text/scanner" import table "core:text/table" @@ -248,6 +249,7 @@ _ :: testing _ :: scanner _ :: i18n _ :: match +_ :: regex _ :: table _ :: edit _ :: thread From e642be8550a7adba2bfcfc47fb5589ba60d837d6 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Wed, 24 Jul 2024 15:17:37 -0400 Subject: [PATCH 06/43] Fix handling of unclosed `regex` classes and repetitions --- core/text/regex/tokenizer/tokenizer.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/text/regex/tokenizer/tokenizer.odin b/core/text/regex/tokenizer/tokenizer.odin index 2702c5434..5804439a8 100644 --- a/core/text/regex/tokenizer/tokenizer.odin +++ b/core/text/regex/tokenizer/tokenizer.odin @@ -267,7 +267,7 @@ scan :: proc(t: ^Tokenizer) -> (token: Token) { kind = .Rune_Class lit = text } else { - return { .EOF, "", pos } + kind = .EOF } case '{': @@ -275,7 +275,7 @@ scan :: proc(t: ^Tokenizer) -> (token: Token) { kind = .Repeat_N lit = text } else { - return { .EOF, "", pos } + kind = .EOF } case '(': From e8537a3134b0539c16907ed42b8832696d65112c Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Wed, 24 Jul 2024 15:21:41 -0400 Subject: [PATCH 07/43] Add test cases for unclosed classes and repetition Simplified error checking while I was at it, too. --- .../core/text/regex/test_core_text_regex.odin | 159 +++++++----------- 1 file changed, 64 insertions(+), 95 deletions(-) diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin index da44e6b2d..0bd1ff288 100644 --- a/tests/core/text/regex/test_core_text_regex.odin +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -3,6 +3,7 @@ package test_core_text_regex import "core:fmt" import "core:io" import "core:log" +import "core:reflect" import "core:strings" import "core:testing" import "core:text/regex" @@ -56,6 +57,17 @@ check_expression :: proc(t: ^testing.T, pattern, haystack: string, needles: ..st haystack, ..needles, loc = loc) } +expect_error :: proc(t: ^testing.T, pattern: string, expected_error: typeid, flags: regex.Flags = {}, loc := #caller_location) { + rex, err := regex.create(pattern, flags) + regex.destroy(rex) + + variant := reflect.get_union_variant(err) + variant_ti := reflect.union_variant_type_info(variant) + expected_ti := type_info_of(expected_error) + + testing.expect_value(t, variant_ti, expected_ti, loc = loc) +} + @test test_concatenation :: proc(t: ^testing.T) { @@ -109,12 +121,18 @@ test_rune_range_terminal_dash :: proc(t: ^testing.T) { @test test_rune_range_escaping_class :: proc(t: ^testing.T) { - EXPR :: `[\]a\[\.]` - check_expression(t, EXPR, "a", "a") - check_expression(t, EXPR, "[", "[") - check_expression(t, EXPR, "]", "]") - check_expression(t, EXPR, ".", ".") - check_expression(t, EXPR, "b") + { + EXPR :: `[\]a\[\.]` + check_expression(t, EXPR, "a", "a") + check_expression(t, EXPR, "[", "[") + check_expression(t, EXPR, "]", "]") + check_expression(t, EXPR, ".", ".") + check_expression(t, EXPR, "b") + } + { + EXPR :: `a[\\]b` + check_expression(t, EXPR, `a\b`, `a\b`) + } } @test @@ -546,8 +564,8 @@ test_unicode_explicitly :: proc(t: ^testing.T) { test_no_capture_match :: proc(t: ^testing.T) { EXPR :: "^abc$" - rex, parse_err := regex.create(EXPR, { .No_Capture }) - if !testing.expect_value(t, parse_err, nil) { + rex, err := regex.create(EXPR, { .No_Capture }) + if !testing.expect_value(t, err, nil) { return } defer regex.destroy(rex) @@ -616,7 +634,7 @@ test_optional_inside_optional :: proc(t: ^testing.T) { @test test_printing :: proc(t: ^testing.T) { - rex, parse_err := regex.create(`^/a$`, { + rex, err := regex.create(`^/a$`, { .Global, .Multiline, .Case_Insensitive, @@ -625,7 +643,7 @@ test_printing :: proc(t: ^testing.T) { .No_Optimization, .No_Capture, }) - if !testing.expect_value(t, parse_err, nil) { + if !testing.expect_value(t, err, nil) { return } defer regex.destroy(rex) @@ -640,36 +658,28 @@ test_printing :: proc(t: ^testing.T) { @test test_error_bad_repetitions :: proc(t: ^testing.T) { - check_repetition_error :: proc(t: ^testing.T, pattern: string, loc := #caller_location) { - rex, err := regex.create(pattern) - regex.destroy(rex) - parse_err, _ := err.(regex.Parser_Error) - _, ok := parse_err.(parser.Invalid_Repetition) - if !ok { - log.errorf("expected error Invalid_Repetition, got %v", parse_err, location = loc) - } - } + expect_error(t, "a{-1,2}", parser.Invalid_Repetition) + expect_error(t, "a{2,1}", parser.Invalid_Repetition) + expect_error(t, "a{bc}", parser.Invalid_Repetition) + expect_error(t, "a{,-3}", parser.Invalid_Repetition) + expect_error(t, "a{d,}", parser.Invalid_Repetition) + expect_error(t, "a{}", parser.Invalid_Repetition) + expect_error(t, "a{0,0}", parser.Invalid_Repetition) + expect_error(t, "a{,0}", parser.Invalid_Repetition) + expect_error(t, "a{,}", parser.Invalid_Repetition) - check_repetition_error(t, "a{-1,2}") - check_repetition_error(t, "a{2,1}") - check_repetition_error(t, "a{bc}") - check_repetition_error(t, "a{,-3}") - check_repetition_error(t, "a{d,}") - check_repetition_error(t, "a{}") - check_repetition_error(t, "a{0,0}") - check_repetition_error(t, "a{,0}") - check_repetition_error(t, "a{,}") + // Unclosed braces + expect_error(t, "a{", parser.Unexpected_EOF) + expect_error(t, "a{", parser.Unexpected_EOF) + expect_error(t, "a{1,2", parser.Unexpected_EOF) + expect_error(t, "a{0,", parser.Unexpected_EOF) + expect_error(t, "a{,3", parser.Unexpected_EOF) + expect_error(t, "a{,", parser.Unexpected_EOF) } @test test_error_invalid_unicode_in_pattern :: proc(t: ^testing.T) { - rex, err := regex.create("\xC0", { .Unicode }) - regex.destroy(rex) - parse_err := err.(regex.Parser_Error) - _, ok := parse_err.(parser.Invalid_Unicode) - if !ok { - log.errorf("expected error Invalid_Unicode, got %v", parse_err) - } + expect_error(t, "\xC0", parser.Invalid_Unicode) } @test @@ -684,50 +694,26 @@ test_error_invalid_unicode_in_string :: proc(t: ^testing.T) { test_error_too_many_capture_groups :: proc(t: ^testing.T) { // NOTE: There are 1 + 9 + 1 capture groups in this pattern. // Remember the implicit capture group 0. - rex, err := regex.create("(1)(2)(3)(4)(5)(6)(7)(8)(9) (A)") - regex.destroy(rex) - - parse_err, _ := err.(regex.Parser_Error) - _, ok := parse_err.(parser.Too_Many_Capture_Groups) - if !ok { - log.errorf("expected error Too_Many_Capture_Groups, got %v", parse_err) - } + expect_error(t, "(1)(2)(3)(4)(5)(6)(7)(8)(9) (A)", parser.Too_Many_Capture_Groups) } @test test_error_unclosed_paren :: proc(t: ^testing.T) { - rex, err := regex.create("(Hellope") - regex.destroy(rex) - - parse_err, _ := err.(regex.Parser_Error) - _, ok := parse_err.(parser.Expected_Token) - if !ok { - log.errorf("expected error Expected_Token, got %v", parse_err) - } + expect_error(t, "(Hellope", parser.Expected_Token) } @test test_error_unclosed_class :: proc(t: ^testing.T) { - rex, err := regex.create("[helope") - regex.destroy(rex) - - parse_err, _ := err.(regex.Parser_Error) - _, ok := parse_err.(parser.Unexpected_EOF) - if !ok { - log.errorf("expected error Unexpected_EOF, got %v", parse_err) - } + expect_error(t, "[helope", parser.Unexpected_EOF) + expect_error(t, `a[\]b`, parser.Unexpected_EOF) + expect_error(t, `a[\b`, parser.Unexpected_EOF) + expect_error(t, `a[\`, parser.Unexpected_EOF) + expect_error(t, `a[`, parser.Unexpected_EOF) } @test test_error_invalid_unicode_in_unclosed_class :: proc(t: ^testing.T) { - rex, err := regex.create("[\xC0", { .Unicode }) - regex.destroy(rex) - - parse_err, _ := err.(regex.Parser_Error) - _, ok := parse_err.(parser.Invalid_Unicode) - if !ok { - log.errorf("expected error Invalid_Unicode, got %v", parse_err) - } + expect_error(t, "[\xC0", parser.Invalid_Unicode, { .Unicode }) } @test @@ -794,35 +780,18 @@ test_lone_enders :: proc(t: ^testing.T) { @test test_invalid_unary_tokens :: proc(t: ^testing.T) { - check_token_error :: proc(t: ^testing.T, pattern: string, loc := #caller_location) { - rex, err := regex.create(pattern) - regex.destroy(rex) - parse_err, _ := err.(regex.Parser_Error) - _, ok := parse_err.(parser.Invalid_Token) - if !ok { - log.errorf("expected error Invalid_Token, got %v", parse_err, location = loc) - } - } - check_token_error(t, `*`) - check_token_error(t, `*?`) - check_token_error(t, `+`) - check_token_error(t, `+?`) - check_token_error(t, `?`) - check_token_error(t, `??`) - check_token_error(t, `{}`) - check_token_error(t, `{1,}`) - check_token_error(t, `{1,2}`) - check_token_error(t, `{,2}`) + expect_error(t, `*`, parser.Invalid_Token) + expect_error(t, `*?`, parser.Invalid_Token) + expect_error(t, `+`, parser.Invalid_Token) + expect_error(t, `+?`, parser.Invalid_Token) + expect_error(t, `?`, parser.Invalid_Token) + expect_error(t, `??`, parser.Invalid_Token) + expect_error(t, `{}`, parser.Invalid_Token) + expect_error(t, `{1,}`, parser.Invalid_Token) + expect_error(t, `{1,2}`, parser.Invalid_Token) + expect_error(t, `{,2}`, parser.Invalid_Token) - { - rex, err := regex.create(`\`) - regex.destroy(rex) - parse_err, _ := err.(regex.Parser_Error) - _, ok := parse_err.(parser.Unexpected_EOF) - if !ok { - log.errorf("expected error Unexpected_EOF, got %v", parse_err) - } - } + expect_error(t, `\`, parser.Unexpected_EOF) } @test From 16b644ad79ca80227c67c4a7a2234dcd47800161 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Wed, 24 Jul 2024 15:23:20 -0400 Subject: [PATCH 08/43] Use `slice.zero` instead --- core/text/regex/virtual_machine/virtual_machine.odin | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/text/regex/virtual_machine/virtual_machine.odin b/core/text/regex/virtual_machine/virtual_machine.odin index f92b84ace..f102fb78c 100644 --- a/core/text/regex/virtual_machine/virtual_machine.odin +++ b/core/text/regex/virtual_machine/virtual_machine.odin @@ -1,6 +1,7 @@ package regex_vm @require import "core:io" +import "core:slice" import "core:text/regex/common" import "core:text/regex/parser" import "core:unicode/utf8" @@ -348,9 +349,7 @@ run :: proc(vm: ^Machine, $UNICODE_MODE: bool) -> (saved: ^[2 * common.MAX_CAPTU } for { - for i := 0; i < len(vm.busy_map); i += 1 { - vm.busy_map[i] = 0 - } + slice.zero(vm.busy_map[:]) assert(vm.string_pointer <= len(vm.memory), "VM string pointer went out of bounds.") From c52a8a5f86707eeb71bcb44e2f691c67c9383500 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Wed, 24 Jul 2024 15:59:59 -0400 Subject: [PATCH 09/43] Allow configuring of `MAX_CAPTURE_GROUPS` for `n` > 10 --- core/text/regex/common/common.odin | 2 +- .../core/text/regex/test_core_text_regex.odin | 70 +++++++++++++++++-- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/core/text/regex/common/common.odin b/core/text/regex/common/common.odin index f53f043a1..f401658cb 100644 --- a/core/text/regex/common/common.odin +++ b/core/text/regex/common/common.odin @@ -2,7 +2,7 @@ package regex_common // VM limitations -MAX_CAPTURE_GROUPS :: 10 +MAX_CAPTURE_GROUPS :: max(#config(ODIN_REGEX_MAX_CAPTURE_GROUPS, 10), 10) MAX_PROGRAM_SIZE :: int(max(i16)) MAX_CLASSES :: int(max(u8)) diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin index 0bd1ff288..74a0b8cf7 100644 --- a/tests/core/text/regex/test_core_text_regex.odin +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -276,9 +276,58 @@ test_optional_capture_group :: proc(t: ^testing.T) { @test test_max_capture_groups :: proc(t: ^testing.T) { - EXPR :: "(1)(2)(3)(4)(5)(6)(7)(8)(9)" - check_expression(t, EXPR, "123456789", "123456789", - "1", "2", "3", "4", "5", "6", "7", "8", "9") + sb_pattern := strings.builder_make() + sb_haystack := strings.builder_make() + expected_captures: [dynamic]string + defer { + strings.builder_destroy(&sb_pattern) + strings.builder_destroy(&sb_haystack) + delete(expected_captures) + } + + w_pattern := strings.to_writer(&sb_pattern) + w_haystack := strings.to_writer(&sb_haystack) + + // The full expression capture, capture 0: + for i in 1.. Date: Wed, 24 Jul 2024 16:05:48 -0400 Subject: [PATCH 10/43] Remove printing facilities for `Regular_Expression` The `original_pattern` introduced a tenuous dependency to the expression value as a whole, and after some consideration, I decided that it would be better for the developer to manage their own pattern strings. In the event you need to print the text representation of a pattern, it's usually better that you manage the memory of it as well. --- core/fmt/fmt.odin | 16 -------------- core/text/regex/regex.odin | 2 -- .../benchmark/text/regex/benchmark_regex.odin | 8 +++---- .../core/text/regex/test_core_text_regex.odin | 22 ------------------- 4 files changed, 4 insertions(+), 44 deletions(-) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 22ac1cc36..9aa9c99dc 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -9,7 +9,6 @@ import "core:io" import "core:reflect" import "core:strconv" import "core:strings" -import "core:text/regex" import "core:time" import "core:unicode/utf8" @@ -2406,21 +2405,6 @@ fmt_named :: proc(fi: ^Info, v: any, verb: rune, info: runtime.Type_Info_Named) write_padded_number(fi, (ns), 9) io.write_string(fi.writer, " +0000 UTC", &fi.n) return - - case regex.Regular_Expression: - io.write_byte(fi.writer, '/') - for r in a.original_pattern { - if r == '/' { - io.write_string(fi.writer, `\/`) - } else { - io.write_rune(fi.writer, r) - } - } - io.write_byte(fi.writer, '/') - for flag in a.flags { - io.write_byte(fi.writer, regex.Flag_To_Letter[flag]) - } - return } } diff --git a/core/text/regex/regex.odin b/core/text/regex/regex.odin index 1736f2305..0bb0b7824 100644 --- a/core/text/regex/regex.odin +++ b/core/text/regex/regex.odin @@ -30,7 +30,6 @@ Capture :: struct { } Regular_Expression :: struct { - original_pattern: string, flags: Flags, class_data: []virtual_machine.Rune_Class_Data, program: []virtual_machine.Opcode `fmt:"-"`, @@ -92,7 +91,6 @@ create :: proc( // allocator so everything can be tightly packed. context.allocator = permanent_allocator - result.original_pattern = pattern result.flags = flags if len(class_data) > 0 { diff --git a/tests/benchmark/text/regex/benchmark_regex.odin b/tests/benchmark/text/regex/benchmark_regex.odin index cd9812b08..8d29888a3 100644 --- a/tests/benchmark/text/regex/benchmark_regex.odin +++ b/tests/benchmark/text/regex/benchmark_regex.odin @@ -111,7 +111,7 @@ global_capture_end_word :: proc(t: ^testing.T) { } defer regex.destroy(rex) - report := fmt.tprintf("Matching %v over a block of random ASCII text.", rex) + report := fmt.tprintf("Matching %q over a block of random ASCII text.", EXPR) for size in sizes { data := make([]u8, size) @@ -151,7 +151,7 @@ global_capture_end_word_unicode :: proc(t: ^testing.T) { } defer regex.destroy(rex) - report := fmt.tprintf("Matching %v over a block of random Unicode text.", rex) + report := fmt.tprintf("Matching %q over a block of random Unicode text.", EXPR) for size in sizes { data := make([]u8, size) @@ -191,7 +191,7 @@ alternations :: proc(t: ^testing.T) { } defer regex.destroy(rex) - report := fmt.tprintf("Matching %v over a text block of only `a`s.", rex) + report := fmt.tprintf("Matching %q over a text block of only `a`s.", EXPR) for size in sizes { data := make([]u8, size) @@ -225,7 +225,7 @@ classes :: proc(t: ^testing.T) { } defer regex.destroy(rex) - report := fmt.tprintf("Matching %v over a string of spaces with %q at the end.", rex, NEEDLE) + report := fmt.tprintf("Matching %q over a string of spaces with %q at the end.", EXPR, NEEDLE) for size in sizes { data := make([]u8, size) diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin index 74a0b8cf7..8ecf6cef2 100644 --- a/tests/core/text/regex/test_core_text_regex.odin +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -681,28 +681,6 @@ test_optional_inside_optional :: proc(t: ^testing.T) { check_expression(t, EXPR, "", "") } -@test -test_printing :: proc(t: ^testing.T) { - rex, err := regex.create(`^/a$`, { - .Global, - .Multiline, - .Case_Insensitive, - .Unicode, - .Ignore_Whitespace, - .No_Optimization, - .No_Capture, - }) - if !testing.expect_value(t, err, nil) { - return - } - defer regex.destroy(rex) - - str := fmt.tprint(rex) - str_hash := fmt.tprintf("%#v", rex) - testing.expect_value(t, str, `/^\/a$/gmixun-`) - testing.expect_value(t, str_hash, `/^\/a$/gmixun-`) -} - @test From ff492e615cc5523903b9b4d38214eefc531b4d0c Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Wed, 24 Jul 2024 16:22:04 -0400 Subject: [PATCH 11/43] Use `unaligned_load` for `regex` virtual machine This should hopefully avoid any issues with loading operands greater than 8 bits on alignment-sensitive platforms. --- core/text/regex/compiler/compiler.odin | 9 +++++---- core/text/regex/compiler/debugging.odin | 15 ++++++++------- .../regex/virtual_machine/virtual_machine.odin | 15 ++++++++------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/core/text/regex/compiler/compiler.odin b/core/text/regex/compiler/compiler.odin index 7617a7bcd..4404947f1 100644 --- a/core/text/regex/compiler/compiler.odin +++ b/core/text/regex/compiler/compiler.odin @@ -1,5 +1,6 @@ package regex_compiler +import "base:intrinsics" import "core:text/regex/common" import "core:text/regex/parser" import "core:text/regex/tokenizer" @@ -408,7 +409,7 @@ compile :: proc(tree: Node, flags: common.Flags) -> (code: Program, class_data: break add_global case .Rune: - operand := (cast(^rune)&code[pc+1])^ + operand := intrinsics.unaligned_load(cast(^rune)&code[pc+1]) inject_at(&code, pc_open, Opcode.Wait_For_Rune) pc_open += size_of(Opcode) inject_raw(&code, pc_open, operand) @@ -490,20 +491,20 @@ compile :: proc(tree: Node, flags: common.Flags) -> (code: Program, class_data: case .Jump: jmp := cast(^i16)&code[pc+size_of(Opcode)] if code[cast(i16)pc+jmp^] == .Jump { - next_jmp := (cast(^i16)&code[cast(i16)pc+jmp^+size_of(Opcode)])^ + next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp^+size_of(Opcode)]) jmp^ = jmp^ + next_jmp do_another_pass = true } case .Split: jmp_x := cast(^i16)&code[pc+size_of(Opcode)] if code[cast(i16)pc+jmp_x^] == .Jump { - next_jmp := (cast(^i16)&code[cast(i16)pc+jmp_x^+size_of(Opcode)])^ + next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp_x^+size_of(Opcode)]) jmp_x^ = jmp_x^ + next_jmp do_another_pass = true } jmp_y := cast(^i16)&code[pc+size_of(Opcode)+size_of(i16)] if code[cast(i16)pc+jmp_y^] == .Jump { - next_jmp := (cast(^i16)&code[cast(i16)pc+jmp_y^+size_of(Opcode)])^ + next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp_y^+size_of(Opcode)]) jmp_y^ = jmp_y^ + next_jmp do_another_pass = true } diff --git a/core/text/regex/compiler/debugging.odin b/core/text/regex/compiler/debugging.odin index 1ef3e6d78..114b88fa2 100644 --- a/core/text/regex/compiler/debugging.odin +++ b/core/text/regex/compiler/debugging.odin @@ -1,5 +1,6 @@ package regex_compiler +import "base:intrinsics" import "core:io" import "core:text/regex/common" import "core:text/regex/virtual_machine" @@ -9,11 +10,11 @@ get_jump_targets :: proc(code: []Opcode) -> (jump_targets: map[int]int) { for opcode, pc in virtual_machine.iterate_opcodes(&iter) { #partial switch opcode { case .Jump: - jmp := cast(int)(cast(^u16)&code[pc+1])^ + jmp := cast(int)intrinsics.unaligned_load(cast(^u16)&code[pc+1]) jump_targets[jmp] = pc case .Split: - jmp_x := cast(int)(cast(^u16)&code[pc+1])^ - jmp_y := cast(int)(cast(^u16)&code[pc+3])^ + jmp_x := cast(int)intrinsics.unaligned_load(cast(^u16)&code[pc+1]) + jmp_y := cast(int)intrinsics.unaligned_load(cast(^u16)&code[pc+3]) jump_targets[jmp_x] = pc jump_targets[jmp_y] = pc } @@ -46,18 +47,18 @@ trace :: proc(w: io.Writer, code: []Opcode) { operand := cast(rune)code[pc+1] io.write_encoded_rune(w, operand) case .Rune: - operand := (cast(^rune)&code[pc+1])^ + operand := intrinsics.unaligned_load(cast(^rune)&code[pc+1]) io.write_encoded_rune(w, operand) case .Rune_Class, .Rune_Class_Negated: operand := cast(u8)code[pc+1] common.write_padded_hex(w, operand, 2) case .Jump: - jmp := (cast(^u16)&code[pc+1])^ + jmp := intrinsics.unaligned_load(cast(^u16)&code[pc+1]) io.write_string(w, "-> $") common.write_padded_hex(w, jmp, 4) case .Split: - jmp_x := (cast(^u16)&code[pc+1])^ - jmp_y := (cast(^u16)&code[pc+3])^ + jmp_x := intrinsics.unaligned_load(cast(^u16)&code[pc+1]) + jmp_y := intrinsics.unaligned_load(cast(^u16)&code[pc+3]) io.write_string(w, "=> $") common.write_padded_hex(w, jmp_x, 4) io.write_string(w, ", $") diff --git a/core/text/regex/virtual_machine/virtual_machine.odin b/core/text/regex/virtual_machine/virtual_machine.odin index f102fb78c..7eb6b1f9b 100644 --- a/core/text/regex/virtual_machine/virtual_machine.odin +++ b/core/text/regex/virtual_machine/virtual_machine.odin @@ -1,5 +1,6 @@ package regex_vm +import "base:intrinsics" @require import "core:io" import "core:slice" import "core:text/regex/common" @@ -121,12 +122,12 @@ add_thread :: proc(vm: ^Machine, saved: ^[2 * common.MAX_CAPTURE_GROUPS]int, pc: #partial switch vm.code[pc] { case .Jump: - pc = cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode)])^ + pc = cast(int)intrinsics.unaligned_load(cast(^u16)&vm.code[pc + size_of(Opcode)]) continue case .Split: - jmp_x := cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode)])^ - jmp_y := cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode) + size_of(u16)])^ + jmp_x := cast(int)intrinsics.unaligned_load(cast(^u16)&vm.code[pc + size_of(Opcode)]) + jmp_y := cast(int)intrinsics.unaligned_load(cast(^u16)&vm.code[pc + size_of(Opcode) + size_of(u16)]) add_thread(vm, saved, jmp_x) pc = jmp_y @@ -236,7 +237,7 @@ add_thread :: proc(vm: ^Machine, saved: ^[2 * common.MAX_CAPTURE_GROUPS]int, pc: vm.top_thread += 1 case .Wait_For_Rune: - operand := (cast(^rune)&vm.code[pc + size_of(Opcode)])^ + operand := intrinsics.unaligned_load(cast(^rune)&vm.code[pc + size_of(Opcode)]) if vm.next_rune == operand { add_thread(vm, saved, pc + size_of(Opcode) + size_of(rune)) } @@ -409,7 +410,7 @@ run :: proc(vm: ^Machine, $UNICODE_MODE: bool) -> (saved: ^[2 * common.MAX_CAPTU } case .Rune: - operand := (cast(^rune)&vm.code[t.pc + size_of(Opcode)])^ + operand := intrinsics.unaligned_load(cast(^rune)&vm.code[t.pc + size_of(Opcode)]) if current_rune == operand { add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(rune)) } @@ -482,7 +483,7 @@ run :: proc(vm: ^Machine, $UNICODE_MODE: bool) -> (saved: ^[2 * common.MAX_CAPTU vm.top_thread += 1 case .Wait_For_Rune: - operand := (cast(^rune)&vm.code[t.pc + size_of(Opcode)])^ + operand := intrinsics.unaligned_load(cast(^rune)&vm.code[t.pc + size_of(Opcode)]) if vm.next_rune == operand { add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(rune)) } @@ -558,7 +559,7 @@ run :: proc(vm: ^Machine, $UNICODE_MODE: bool) -> (saved: ^[2 * common.MAX_CAPTU break escape_loop case .Jump: - t.pc = cast(int)(cast(^u16)&vm.code[t.pc + size_of(Opcode)])^ + t.pc = cast(int)intrinsics.unaligned_load(cast(^u16)&vm.code[t.pc + size_of(Opcode)]) case .Save: index := vm.code[t.pc + size_of(Opcode)] From 90f1f7fbdfc283b03216e15e2700331395539161 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Wed, 24 Jul 2024 16:48:49 -0400 Subject: [PATCH 12/43] Use `unaligned_store` in `regex` too --- core/text/regex/compiler/compiler.odin | 27 ++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/core/text/regex/compiler/compiler.odin b/core/text/regex/compiler/compiler.odin index 4404947f1..f5d6d2f6a 100644 --- a/core/text/regex/compiler/compiler.odin +++ b/core/text/regex/compiler/compiler.odin @@ -490,22 +490,25 @@ compile :: proc(tree: Node, flags: common.Flags) -> (code: Program, class_data: #partial switch opcode { case .Jump: jmp := cast(^i16)&code[pc+size_of(Opcode)] - if code[cast(i16)pc+jmp^] == .Jump { - next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp^+size_of(Opcode)]) - jmp^ = jmp^ + next_jmp + jmp_value := intrinsics.unaligned_load(jmp) + if code[cast(i16)pc+jmp_value] == .Jump { + next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp_value+size_of(Opcode)]) + intrinsics.unaligned_store(jmp, jmp_value + next_jmp) do_another_pass = true } case .Split: jmp_x := cast(^i16)&code[pc+size_of(Opcode)] - if code[cast(i16)pc+jmp_x^] == .Jump { - next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp_x^+size_of(Opcode)]) - jmp_x^ = jmp_x^ + next_jmp + jmp_x_value := intrinsics.unaligned_load(jmp_x) + if code[cast(i16)pc+jmp_x_value] == .Jump { + next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp_x_value+size_of(Opcode)]) + intrinsics.unaligned_store(jmp_x, jmp_x_value + next_jmp) do_another_pass = true } jmp_y := cast(^i16)&code[pc+size_of(Opcode)+size_of(i16)] - if code[cast(i16)pc+jmp_y^] == .Jump { - next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp_y^+size_of(Opcode)]) - jmp_y^ = jmp_y^ + next_jmp + jmp_y_value := intrinsics.unaligned_load(jmp_y) + if code[cast(i16)pc+jmp_y_value] == .Jump { + next_jmp := intrinsics.unaligned_load(cast(^i16)&code[cast(i16)pc+jmp_y_value+size_of(Opcode)]) + intrinsics.unaligned_store(jmp_y, jmp_y_value + next_jmp) do_another_pass = true } } @@ -526,12 +529,12 @@ compile :: proc(tree: Node, flags: common.Flags) -> (code: Program, class_data: #partial switch opcode { case .Jump: jmp := cast(^u16)&code[pc+size_of(Opcode)] - jmp^ = jmp^ + cast(u16)pc + intrinsics.unaligned_store(jmp, intrinsics.unaligned_load(jmp) + cast(u16)pc) case .Split: jmp_x := cast(^u16)&code[pc+size_of(Opcode)] - jmp_x^ = jmp_x^ + cast(u16)pc + intrinsics.unaligned_store(jmp_x, intrinsics.unaligned_load(jmp_x) + cast(u16)pc) jmp_y := cast(^u16)&code[pc+size_of(Opcode)+size_of(i16)] - jmp_y^ = jmp_y^ + cast(u16)pc + intrinsics.unaligned_store(jmp_y, intrinsics.unaligned_load(jmp_y) + cast(u16)pc) } } From 62527123638ba5ac9e707e0fbcdeb9e9e56b9c31 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 13:21:13 -0400 Subject: [PATCH 13/43] Add missing features to `regex` package documentation --- core/text/regex/doc.odin | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/text/regex/doc.odin b/core/text/regex/doc.odin index 8899e1af6..7b28bbc3d 100644 --- a/core/text/regex/doc.odin +++ b/core/text/regex/doc.odin @@ -9,12 +9,16 @@ Odin's regex library implements the following features: Alternation: `apple|cherry` Classes: `[0-9_]` + Classes, negated: `[^0-9_]` + Shorthands: `\d\s\w` + Shorthands, negated: `\D\S\W` Wildcards: `.` Repeat, optional: `a*` Repeat, at least once: `a+` + Repetition: `a{1,2}` Optional: `a?` - Group Capture: `([0-9])` - Group Non-Capture: `(?:[0-9])` + Group, capture: `([0-9])` + Group, non-capture: `(?:[0-9])` Start & End Anchors: `^hello$` Word Boundaries: `\bhello\b` Non-Word Boundaries: `hello\B` From cd8272557feae951543c0d661b3c8d82e1a67c44 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:33:36 -0400 Subject: [PATCH 14/43] Test that a RegEx Capture `pos` corresponds to its `groups` --- tests/core/text/regex/test_core_text_regex.odin | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin index 8ecf6cef2..d9d4f8cbc 100644 --- a/tests/core/text/regex/test_core_text_regex.odin +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -44,6 +44,13 @@ check_expression_with_flags :: proc(t: ^testing.T, pattern: string, flags: regex } else { log.infof("match groups were: %v", capture.groups, location = loc) } + + for pos, g in capture.pos { + pos_str := haystack[pos[0]:pos[1]] + if !testing.expectf(t, pos_str == capture.groups[g], "position string %v %q does not correspond to group string %q", pos, pos_str, capture.groups[g]) { + break + } + } } check_expression :: proc(t: ^testing.T, pattern, haystack: string, needles: ..string, extra_flags: regex.Flags = {}, loc := #caller_location) { From d3a51e208d01d2ab59067018a219340d42a228fa Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:40:27 -0400 Subject: [PATCH 15/43] Hide `Regular_Expression` values We don't directly support printing these. To prevent future issues being raised about the pattern being missing if someone tries to print one, hide everything. --- core/text/regex/regex.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/text/regex/regex.odin b/core/text/regex/regex.odin index 0bb0b7824..9da7e9246 100644 --- a/core/text/regex/regex.odin +++ b/core/text/regex/regex.odin @@ -30,8 +30,8 @@ Capture :: struct { } Regular_Expression :: struct { - flags: Flags, - class_data: []virtual_machine.Rune_Class_Data, + flags: Flags `fmt:"-"`, + class_data: []virtual_machine.Rune_Class_Data `fmt:"-"`, program: []virtual_machine.Opcode `fmt:"-"`, } From babdc432e963f2c6456b0d85102419c526600718 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:42:46 -0400 Subject: [PATCH 16/43] Move `Flag_To_Letter` to `core:text/regex/common` --- core/text/regex/common/common.odin | 11 +++++++++++ core/text/regex/regex.odin | 11 ----------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/core/text/regex/common/common.odin b/core/text/regex/common/common.odin index f401658cb..1169bb3d4 100644 --- a/core/text/regex/common/common.odin +++ b/core/text/regex/common/common.odin @@ -25,3 +25,14 @@ Flag :: enum u8 { } Flags :: bit_set[Flag; u8] + +@(rodata) +Flag_To_Letter := #sparse[Flag]u8 { + .Global = 'g', + .Multiline = 'm', + .Case_Insensitive = 'i', + .Ignore_Whitespace = 'x', + .Unicode = 'u', + .No_Capture = 'n', + .No_Optimization = '-', +} diff --git a/core/text/regex/regex.odin b/core/text/regex/regex.odin index 9da7e9246..a4bd4292e 100644 --- a/core/text/regex/regex.odin +++ b/core/text/regex/regex.odin @@ -36,17 +36,6 @@ Regular_Expression :: struct { } -@(rodata) -Flag_To_Letter := #sparse[Flag]u8 { - .Global = 'g', - .Multiline = 'm', - .Case_Insensitive = 'i', - .Ignore_Whitespace = 'x', - .Unicode = 'u', - .No_Capture = 'n', - .No_Optimization = '-', -} - /* Create a regular expression from a string pattern and a set of flags. From 1ccb0b25583cc2e5b1b58ba6d60571d637a28370 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:45:20 -0400 Subject: [PATCH 17/43] Remove unused code --- core/text/regex/compiler/compiler.odin | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/text/regex/compiler/compiler.odin b/core/text/regex/compiler/compiler.odin index f5d6d2f6a..1ce881894 100644 --- a/core/text/regex/compiler/compiler.odin +++ b/core/text/regex/compiler/compiler.odin @@ -41,7 +41,6 @@ SPLIT_SIZE :: size_of(Opcode) + 2 * size_of(u16) Compiler :: struct { flags: common.Flags, - anchor_start_seen: bool, class_data: [dynamic]Rune_Class_Data, } @@ -192,7 +191,6 @@ generate_code :: proc(c: ^Compiler, node: Node) -> (code: Program) { append(&code, Opcode.Multiline_Close) } else { if specific.start { - c.anchor_start_seen = true append(&code, Opcode.Assert_Start) } else { append(&code, Opcode.Assert_End) From 743480b1a489ecd889a0ea9d29842555b136606d Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:51:40 -0400 Subject: [PATCH 18/43] Use `regex.destroy` for test captures --- tests/core/text/regex/test_core_text_regex.odin | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin index d9d4f8cbc..96a305856 100644 --- a/tests/core/text/regex/test_core_text_regex.odin +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -21,10 +21,7 @@ check_expression_with_flags :: proc(t: ^testing.T, pattern: string, flags: regex defer regex.destroy(rex) capture, success := regex.match(rex, haystack) - defer { - delete(capture.groups) - delete(capture.pos) - } + defer regex.destroy(capture) if len(needles) > 0 { testing.expect(t, success, "match failed", loc = loc) From ca7e46d56f54b33bb3ac630ead662c327733774d Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:51:51 -0400 Subject: [PATCH 19/43] Add explicit test case for Capture `pos` --- .../core/text/regex/test_core_text_regex.odin | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin index 96a305856..d272dcc0b 100644 --- a/tests/core/text/regex/test_core_text_regex.odin +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -499,6 +499,31 @@ test_word_boundaries :: proc(t: ^testing.T) { } } +@test +test_pos_index_explicitly :: proc(t: ^testing.T) { + STR :: "This is an island." + EXPR :: `\bis\b` + + rex, err := regex.create(EXPR, { .Global }) + if !testing.expect_value(t, err, nil) { + return + } + defer regex.destroy(rex) + + capture, success := regex.match(rex, STR) + log.info(capture, success) + if !testing.expect(t, success) { + return + } + defer regex.destroy(capture) + + if !testing.expect_value(t, len(capture.pos), 1) { + return + } + testing.expect_value(t, capture.pos[0][0], 5) + testing.expect_value(t, capture.pos[0][1], 7) +} + @test test_non_word_boundaries :: proc(t: ^testing.T) { { From dde42f0ebcef9dd7741761e6a7cc5ba738b63320 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:55:54 -0400 Subject: [PATCH 20/43] Add more documentation for `core:text/regex` API --- core/text/regex/regex.odin | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/core/text/regex/regex.odin b/core/text/regex/regex.odin index a4bd4292e..0d8a1d9c0 100644 --- a/core/text/regex/regex.odin +++ b/core/text/regex/regex.odin @@ -13,22 +13,43 @@ Compiler_Error :: compiler.Error Creation_Error :: enum { None, + // A `\` was supplied as the delimiter to `create_by_user`. Bad_Delimiter, + // A pair of delimiters for `create_by_user` was not found. Expected_Delimiter, + // An unknown letter was supplied to `create_by_user` after the last delimiter. Unknown_Flag, } Error :: union #shared_nil { + // An error that can occur in the pattern parsing phase. + // + // Most of these are regular expression syntax errors and are either + // context-dependent as to what they mean or have self-explanatory names. Parser_Error, + // An error that can occur in the pattern compiling phase. + // + // Of the two that can be returned, they have to do with exceeding the + // limitations of the Virtual Machine. Compiler_Error, + // An error that occurs only for `create_by_user`. Creation_Error, } +/* +This struct corresponds to a set of string captures from a RegEx match. + +`pos` will contain the start and end positions for each string in `groups`, +such that `str[pos[0][0]:pos[0][1]] == groups[0]`. +*/ Capture :: struct { pos: [][2]int, groups: []string, } +/* +A compiled Regular Expression value, to be used with the `match_*` procedures. +*/ Regular_Expression :: struct { flags: Flags `fmt:"-"`, class_data: []virtual_machine.Rune_Class_Data `fmt:"-"`, From e17fc8272b08d1e2f59c13ff23df9a3d84a0c8a0 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 19:12:46 -0400 Subject: [PATCH 21/43] Document rationale behind RegEx shorthand classes --- core/text/regex/doc.odin | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/text/regex/doc.odin b/core/text/regex/doc.odin index 7b28bbc3d..61ab8b80e 100644 --- a/core/text/regex/doc.odin +++ b/core/text/regex/doc.odin @@ -29,6 +29,24 @@ These specifiers can be composed together, such as an optional group: This package also supports the non-greedy variants of the repeating and optional specifiers by appending a `?` to them. +Of the shorthand classes that are supported, they are all ASCII-based, even +when compiling in Unicode mode. This is for the sake of general performance and +simplicity, as there are thousands of Unicode codepoints which would qualify as +either a digit, space, or word character which could be irrelevant depending on +what is being matched. + +Here are the shorthand class equivalencies: + \d: [0-9] + \s: [\t\n\f\r ] + \w: [0-9A-Z_a-z] + +If you need your own shorthands, you can compose strings together like so: + MY_HEX :: "[0-9A-Fa-f]" + PATTERN :: MY_HEX + "-" + MY_HEX + +The compiler will handle turning multiple identical classes into references to +the same set of matching runes, so there's no penalty for doing it like this. + ``Some people, when confronted with a problem, think From 14858309f082e7fccdeb9859422f7ba1f0ee98c8 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Sun, 4 Aug 2024 19:18:16 -0400 Subject: [PATCH 22/43] Add explicit license info to `core:text/regex` --- core/text/regex/common/common.odin | 8 ++++++++ core/text/regex/common/debugging.odin | 8 ++++++++ core/text/regex/compiler/compiler.odin | 8 ++++++++ core/text/regex/compiler/debugging.odin | 8 ++++++++ core/text/regex/optimizer/optimizer.odin | 8 ++++++++ core/text/regex/parser/debugging.odin | 8 ++++++++ core/text/regex/parser/parser.odin | 8 ++++++++ core/text/regex/regex.odin | 8 ++++++++ core/text/regex/tokenizer/tokenizer.odin | 8 ++++++++ core/text/regex/virtual_machine/util.odin | 8 ++++++++ core/text/regex/virtual_machine/virtual_machine.odin | 8 ++++++++ 11 files changed, 88 insertions(+) diff --git a/core/text/regex/common/common.odin b/core/text/regex/common/common.odin index 1169bb3d4..4a303e0a3 100644 --- a/core/text/regex/common/common.odin +++ b/core/text/regex/common/common.odin @@ -1,6 +1,14 @@ // This package helps break dependency cycles. package regex_common +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + // VM limitations MAX_CAPTURE_GROUPS :: max(#config(ODIN_REGEX_MAX_CAPTURE_GROUPS, 10), 10) MAX_PROGRAM_SIZE :: int(max(i16)) diff --git a/core/text/regex/common/debugging.odin b/core/text/regex/common/debugging.odin index 062c314cc..0e4161a92 100644 --- a/core/text/regex/common/debugging.odin +++ b/core/text/regex/common/debugging.odin @@ -1,5 +1,13 @@ package regex_common +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + @require import "core:os" import "core:io" import "core:strings" diff --git a/core/text/regex/compiler/compiler.odin b/core/text/regex/compiler/compiler.odin index 1ce881894..b3ded0104 100644 --- a/core/text/regex/compiler/compiler.odin +++ b/core/text/regex/compiler/compiler.odin @@ -1,5 +1,13 @@ package regex_compiler +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + import "base:intrinsics" import "core:text/regex/common" import "core:text/regex/parser" diff --git a/core/text/regex/compiler/debugging.odin b/core/text/regex/compiler/debugging.odin index 114b88fa2..142cb8839 100644 --- a/core/text/regex/compiler/debugging.odin +++ b/core/text/regex/compiler/debugging.odin @@ -1,5 +1,13 @@ package regex_compiler +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + import "base:intrinsics" import "core:io" import "core:text/regex/common" diff --git a/core/text/regex/optimizer/optimizer.odin b/core/text/regex/optimizer/optimizer.odin index fbb65cf79..835e5022c 100644 --- a/core/text/regex/optimizer/optimizer.odin +++ b/core/text/regex/optimizer/optimizer.odin @@ -1,5 +1,13 @@ package regex_optimizer +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + import "base:intrinsics" @require import "core:io" import "core:slice" diff --git a/core/text/regex/parser/debugging.odin b/core/text/regex/parser/debugging.odin index 4d531965c..e060f58c2 100644 --- a/core/text/regex/parser/debugging.odin +++ b/core/text/regex/parser/debugging.odin @@ -1,5 +1,13 @@ package regex_parser +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + import "core:io" write_node :: proc(w: io.Writer, node: Node) { diff --git a/core/text/regex/parser/parser.odin b/core/text/regex/parser/parser.odin index 1958ee399..720992cb9 100644 --- a/core/text/regex/parser/parser.odin +++ b/core/text/regex/parser/parser.odin @@ -1,5 +1,13 @@ package regex_parser +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + import "base:intrinsics" import "core:strconv" import "core:strings" diff --git a/core/text/regex/regex.odin b/core/text/regex/regex.odin index 0d8a1d9c0..9ff924192 100644 --- a/core/text/regex/regex.odin +++ b/core/text/regex/regex.odin @@ -1,5 +1,13 @@ package regex +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + import "core:text/regex/common" import "core:text/regex/compiler" import "core:text/regex/optimizer" diff --git a/core/text/regex/tokenizer/tokenizer.odin b/core/text/regex/tokenizer/tokenizer.odin index 5804439a8..447fe4329 100644 --- a/core/text/regex/tokenizer/tokenizer.odin +++ b/core/text/regex/tokenizer/tokenizer.odin @@ -1,5 +1,13 @@ package regex_tokenizer +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + import "core:text/regex/common" import "core:unicode/utf8" diff --git a/core/text/regex/virtual_machine/util.odin b/core/text/regex/virtual_machine/util.odin index edf055bc7..fa94a139f 100644 --- a/core/text/regex/virtual_machine/util.odin +++ b/core/text/regex/virtual_machine/util.odin @@ -1,5 +1,13 @@ package regex_vm +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + Opcode_Iterator :: struct { code: Program, pc: int, diff --git a/core/text/regex/virtual_machine/virtual_machine.odin b/core/text/regex/virtual_machine/virtual_machine.odin index 7eb6b1f9b..a4fca6c4d 100644 --- a/core/text/regex/virtual_machine/virtual_machine.odin +++ b/core/text/regex/virtual_machine/virtual_machine.odin @@ -1,5 +1,13 @@ package regex_vm +/* + (c) Copyright 2024 Feoramund . + Made available under Odin's BSD-3 license. + + List of contributors: + Feoramund: Initial implementation. +*/ + import "base:intrinsics" @require import "core:io" import "core:slice" From 8f5b838a071d010959a3f38a2c73cf49f31a16cf Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Mon, 5 Aug 2024 03:49:29 -0400 Subject: [PATCH 23/43] Review manual `for` loops in `core:text/regex` --- core/text/regex/optimizer/optimizer.odin | 8 ++++---- core/text/regex/parser/parser.odin | 4 +++- core/text/regex/regex.odin | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/text/regex/optimizer/optimizer.odin b/core/text/regex/optimizer/optimizer.odin index 835e5022c..e23cc1bc5 100644 --- a/core/text/regex/optimizer/optimizer.odin +++ b/core/text/regex/optimizer/optimizer.odin @@ -86,7 +86,7 @@ optimize_subtree :: proc(tree: Node, flags: common.Flags) -> (result: Node, chan } // Only recursive optimizations: - for i := 0; i < len(specific.nodes); i += 1 { + #no_bounds_check for i := 0; i < len(specific.nodes); i += 1 { subnode, subnode_changes := optimize_subtree(specific.nodes[i], flags) changes += subnode_changes if subnode == nil { @@ -194,7 +194,7 @@ optimize_subtree :: proc(tree: Node, flags: common.Flags) -> (result: Node, chan new_range.lower = specific.runes[0] new_range.upper = specific.runes[0] - for i := 1; i < len(specific.runes); i += 1 { + #no_bounds_check for i := 1; i < len(specific.runes); i += 1 { r := specific.runes[i] if new_range.lower == -1 { new_range = { r, r } @@ -228,7 +228,7 @@ optimize_subtree :: proc(tree: Node, flags: common.Flags) -> (result: Node, chan // // DO: `[aa-c]` => `[a-c]` for range in specific.ranges { - for i := 0; i < len(specific.runes); i += 1 { + #no_bounds_check for i := 0; i < len(specific.runes); i += 1 { r := specific.runes[i] if range.lower <= r && r <= range.upper { ordered_remove(&specific.runes, i) @@ -244,7 +244,7 @@ optimize_subtree :: proc(tree: Node, flags: common.Flags) -> (result: Node, chan // DO: `[a-cd-e]` => `[a-e]` // DO: `[a-cb-e]` => `[a-e]` slice.sort_by(specific.ranges[:], class_range_sorter) - for i := 0; i < len(specific.ranges) - 1; i += 1 { + #no_bounds_check for i := 0; i < len(specific.ranges) - 1; i += 1 { for j := i + 1; j < len(specific.ranges); j += 1 { left_range := &specific.ranges[i] right_range := specific.ranges[j] diff --git a/core/text/regex/parser/parser.odin b/core/text/regex/parser/parser.odin index 720992cb9..038d4cb85 100644 --- a/core/text/regex/parser/parser.odin +++ b/core/text/regex/parser/parser.odin @@ -225,7 +225,7 @@ null_denotation :: proc(p: ^Parser, token: Token) -> (result: Node, err: Error) node := new(Node_Rune_Class) - for i := 0; i < len(token.text); /**/ { + #no_bounds_check for i := 0; i < len(token.text); /**/ { r, size := utf8.decode_rune(token.text[i:]) if i == 0 && r == '^' { node.negating = true @@ -298,6 +298,8 @@ null_denotation :: proc(p: ^Parser, token: Token) -> (result: Node, err: Error) } if .Case_Insensitive in p.flags { + // These two loops cannot be in the form of `for x in y` because + // they append to the data that they iterate over. length := len(node.runes) #no_bounds_check for i := 0; i < length; i += 1 { r := node.runes[i] diff --git a/core/text/regex/regex.odin b/core/text/regex/regex.odin index 9ff924192..3dc26b5c6 100644 --- a/core/text/regex/regex.odin +++ b/core/text/regex/regex.odin @@ -291,7 +291,7 @@ match_and_allocate_capture :: proc( context.allocator = permanent_allocator num_groups := 0 - for i := 0; i < len(saved); i += 2 { + #no_bounds_check for i := 0; i < len(saved); i += 2 { a, b := saved[i], saved[i + 1] if a == -1 || b == -1 { continue From d0d4f19097a8df19ebe5ef471047c656bd2318b3 Mon Sep 17 00:00:00 2001 From: Feoramund <161657516+Feoramund@users.noreply.github.com> Date: Mon, 5 Aug 2024 03:50:41 -0400 Subject: [PATCH 24/43] Remove debug line from test --- tests/core/text/regex/test_core_text_regex.odin | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin index d272dcc0b..d99130584 100644 --- a/tests/core/text/regex/test_core_text_regex.odin +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -511,7 +511,6 @@ test_pos_index_explicitly :: proc(t: ^testing.T) { defer regex.destroy(rex) capture, success := regex.match(rex, STR) - log.info(capture, success) if !testing.expect(t, success) { return } From 142bda2804d45304ece9916dae845a5da8e43a10 Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Sat, 3 Aug 2024 23:55:48 +0200 Subject: [PATCH 25/43] posix: start on process API --- core/os/os2/process_posix.odin | 162 ++++++++++++++++++++++++-- core/os/os2/process_posix_darwin.odin | 32 +++++ 2 files changed, 183 insertions(+), 11 deletions(-) diff --git a/core/os/os2/process_posix.odin b/core/os/os2/process_posix.odin index 8a27efce4..fc6e135d8 100644 --- a/core/os/os2/process_posix.odin +++ b/core/os/os2/process_posix.odin @@ -43,27 +43,167 @@ _current_process_info :: proc(selection: Process_Info_Fields, allocator: runtime return _process_info_by_pid(_get_pid(), selection, allocator) } -_process_open :: proc(pid: int, flags: Process_Open_Flags) -> (process: Process, err: Error) { - err = .Unsupported - return -} - _Sys_Process_Attributes :: struct {} _process_start :: proc(desc: Process_Desc) -> (process: Process, err: Error) { - err = .Unsupported - return + if len(desc.command) == 0 { + err = .Invalid_Path + return + } + + cwd: cstring; if desc.working_dir != "" { + cwd = temp_cstring(desc.working_dir) + } + + cmd := make([]cstring, len(desc.command)+1, temp_allocator()) + for part, i in desc.command { + cmd[i] = temp_cstring(part) + } + + switch pid := posix.fork(); pid { + case -1: + err = _get_platform_error() + return + + case 0: + // NOTE(laytan): would need to use execvp and look up the command in the PATH. + assert(len(desc.env) == 0, "unimplemented: process_start with env") + + null := posix.open("/dev/null", { .RDWR, .CLOEXEC }) + assert(null != -1) // TODO: Does this happen/need to be handled? + + stderr := (^File_Impl)(desc.stderr.impl).fd if desc.stderr != nil else null + stdout := (^File_Impl)(desc.stdout.impl).fd if desc.stdout != nil else null + stdin := (^File_Impl)(desc.stdin.impl).fd if desc.stdin != nil else null + + posix.dup2(stderr, posix.STDERR_FILENO) + posix.dup2(stdout, posix.STDOUT_FILENO) + posix.dup2(stdin, posix.STDIN_FILENO ) + + // NOTE(laytan): is this how we should handle these? + // Maybe we can try to `stat` the cwd in the parent before forking? + // Does that mean no other errors could happen in chdir? + // How about execvp? + + if cwd != nil { + if posix.chdir(cwd) != .OK { + posix.exit(i32(posix.errno())) // TODO: handle, or is it fine this way? + } + } + + posix.execvp(cmd[0], raw_data(cmd)) + posix.exit(i32(posix.errno())) // TODO: handle, or is it fine this way? + + case: + fmt.println("returning") + process, _ = _process_open(int(pid), {}) + process.pid = int(pid) + return + } } +import "core:fmt" +import "core:nbio/kqueue" + _process_wait :: proc(process: Process, timeout: time.Duration) -> (process_state: Process_State, err: Error) { - err = .Unsupported + process_state.pid = process.pid + + if !process_posix_handle_still_valid(process) { + err = Platform_Error(posix.Errno.ESRCH) + return + } + + // prev := posix.signal(.SIGALRM, proc "c" (_: posix.Signal) { + // context = runtime.default_context() + // fmt.println("alarm") + // }) + // defer posix.signal(.SIGALRM, prev) + // + // posix.alarm(u32(time.duration_seconds(timeout))) + // defer posix.alarm(0) + + // TODO: if there's no timeout, don't set up a kqueue. + + // TODO: if timeout is 0, don't set up a kqueue and use NO_HANG. + + kq, qerr := kqueue.kqueue() + if qerr != nil { + err = Platform_Error(qerr) + return + } + + changelist, eventlist: [1]kqueue.KEvent + + changelist[0] = { + ident = uintptr(process.pid), + filter = .Proc, + flags = { .Add }, + fflags = { + fproc = 0x80000000, + }, + } + + // NOTE: could this be interrupted which means it should be looped and subtracting the timeout on EINTR. + + n, eerr := kqueue.kevent(kq, changelist[:], eventlist[:], &{ + seconds = i64(timeout / time.Second), + nanoseconds = i64(timeout % time.Second), + }) + if eerr != nil { + err = Platform_Error(eerr) + return + } + + if n == 0 { + err = .Timeout + + // TODO: populate the time fields. + + return + } + + // NOTE(laytan): should this be looped untill WIFEXITED/WIFSIGNALED? + + status: i32 + wpid := posix.waitpid(posix.pid_t(process.pid), &status, {}) + if wpid == -1 { + err = _get_platform_error() + return + } + + process_state.exited = true + + // TODO: populate times + + switch { + case posix.WIFEXITED(status): + fmt.printfln("child exited, status=%v", posix.WEXITSTATUS(status)) + process_state.exit_code = int(posix.WEXITSTATUS(status)) + process_state.success = true + case posix.WIFSIGNALED(status): + fmt.printfln("child killed (signal %v)", posix.WTERMSIG(status)) + process_state.exit_code = int(posix.WTERMSIG(status)) + process_state.success = false + case: + fmt.panicf("unexpected status (%x)", status) + } + return } _process_close :: proc(process: Process) -> Error { - return .Unsupported + return nil } -_process_kill :: proc(process: Process) -> Error { - return .Unsupported +_process_kill :: proc(process: Process) -> (err: Error) { + if !process_posix_handle_still_valid(process) { + err = Platform_Error(posix.Errno.ESRCH) + return + } + + if posix.kill(posix.pid_t(process.pid), .SIGKILL) != .OK { + err = _get_platform_error() + } + + return } diff --git a/core/os/os2/process_posix_darwin.odin b/core/os/os2/process_posix_darwin.odin index 4fbb84886..5558ef00a 100644 --- a/core/os/os2/process_posix_darwin.odin +++ b/core/os/os2/process_posix_darwin.odin @@ -19,6 +19,8 @@ foreign lib { ) -> posix.result --- } +import "core:fmt" + _process_info_by_pid :: proc(pid: int, selection: Process_Info_Fields, allocator: runtime.Allocator) -> (info: Process_Info, err: Error) { get_pidinfo :: proc(pid: int, selection: Process_Info_Fields) -> (ppid: u32, prio: Maybe(i32), uid: posix.uid_t, ok: bool) { // Short info is enough and requires less permissions if the priority isn't requested. @@ -254,3 +256,33 @@ _process_list :: proc(allocator: runtime.Allocator) -> (list: []int, err: Error) return } + +_process_open :: proc(pid: int, flags: Process_Open_Flags) -> (process: Process, err: Error) { + + // NOTE(laytan): pids can get reused, and afaik posix/macos doesn't have a unique identifier + // for a specific process execution, next best thing to me is checking the time the process + // started as some extra "uniqueness". We could also hash a bunch of the fields in this info. + + // This incidentally also checks if the pid is actually valid so that's nice. + + pinfo: darwin.proc_bsdinfo + ret := darwin.proc_pidinfo(posix.pid_t(pid), .BSDINFO, 0, &pinfo, size_of(pinfo)) + if ret <= 0 { + err = _get_platform_error() + return + } + + assert(ret == size_of(pinfo)) + process = { int(pid), uintptr(pinfo.pbi_start_tvusec) } + return +} + +process_posix_handle_still_valid :: proc(p: Process) -> bool { + pinfo: darwin.proc_bsdinfo + ret := darwin.proc_pidinfo(posix.pid_t(p.pid), .BSDINFO, 0, &pinfo, size_of(pinfo)) + if ret <= 0 { + return false + } + + return uintptr(pinfo.pbi_start_tvusec) == p.handle +} From f96991364a8e0fe29c8c1e6c1915ed39776b5fea Mon Sep 17 00:00:00 2001 From: Yeongju Kang <979156@gmail.com> Date: Mon, 19 Aug 2024 08:51:52 +0900 Subject: [PATCH 26/43] implement clock_settime, clock_getres and clock_nanosleep --- core/sys/linux/sys.odin | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/core/sys/linux/sys.odin b/core/sys/linux/sys.odin index f4f609ab9..8aa2f09cc 100644 --- a/core/sys/linux/sys.odin +++ b/core/sys/linux/sys.odin @@ -2404,17 +2404,43 @@ timer_delete :: proc "contextless" (timer: Timer) -> (Errno) { return Errno(-ret) } -// TODO(flysand): clock_settime +/* + Set the time of the specified clock. + Available since Linux 2.6. +*/ +clock_settime :: proc "contextless" (clock: Clock_Id, ts: ^Time_Spec) -> (Errno) { + ret := syscall(SYS_clock_settime, clock, ts) + return Errno(-ret) +} +/* + Retrieve the time of the specified clock. + Available since Linux 2.6. +*/ clock_gettime :: proc "contextless" (clock: Clock_Id) -> (ts: Time_Spec, err: Errno) { ret := syscall(SYS_clock_gettime, clock, &ts) err = Errno(-ret) return } -// TODO(flysand): clock_getres -// TODO(flysand): clock_nanosleep +/* + Finds the resolution of the specified clock. + Available since Linux 2.6. +*/ +clock_getres :: proc "contextless" (clock: Clock_Id, res: ^Time_Spec) -> (Errno) { + ret := syscall(SYS_clock_getres, clock, res) + return Errno(-ret) +} + +/* + Sleep for an interval specified with nanosecond precision. + Available since Linux 2.6. +*/ +clock_nanosleep :: proc "contextless" (clock: Clock_Id, flags: ITimer_Flags, request: ^Time_Spec, remain: ^Time_Spec) -> (Errno) { + ret := syscall(SYS_clock_nanosleep, clock, transmute(u32) flags, request, remain) + return Errno(-ret) +} /* Exit the thread group. From ca6ef95b038f3eb443971240de73924a721485cc Mon Sep 17 00:00:00 2001 From: Laytan Date: Thu, 15 Aug 2024 20:39:35 +0200 Subject: [PATCH 27/43] add support for linux_riscv64 and freestanding_riscv64 --- .github/workflows/ci.yml | 50 +++ base/runtime/entry_unix.odin | 3 + base/runtime/entry_unix_no_crt_riscv64.asm | 10 + base/runtime/os_specific_linux.odin | 2 + .../_chacha20/simd128/chacha20_simd128.odin | 2 +- core/net/socket_linux.odin | 4 +- core/os/os_linux.odin | 19 + core/sys/info/cpu_linux_riscv64.odin | 46 +++ core/sys/info/cpu_riscv64.odin | 16 + core/sys/info/sysinfo.odin | 2 +- core/sys/linux/bits.odin | 14 +- core/sys/linux/sys.odin | 54 +-- core/sys/linux/syscall_riscv64.odin | 334 +++++++++++++++ core/sys/linux/types.odin | 48 ++- core/sys/unix/syscalls_linux.odin | 380 ++++++++++++++++-- misc/featuregen/README.md | 4 +- misc/featuregen/build_featuregen.sh | 5 + misc/featuregen/featuregen.py | 19 +- src/build_settings.cpp | 20 +- src/build_settings_microarch.cpp | 102 +++-- src/check_builtin.cpp | 5 + src/checker.cpp | 1 + src/linker.cpp | 29 +- src/llvm_abi.cpp | 269 ++++++++++++- src/llvm_backend.cpp | 31 +- src/llvm_backend_general.cpp | 8 +- src/llvm_backend_proc.cpp | 25 ++ src/main.cpp | 9 + 28 files changed, 1395 insertions(+), 116 deletions(-) create mode 100644 base/runtime/entry_unix_no_crt_riscv64.asm create mode 100644 core/sys/info/cpu_linux_riscv64.odin create mode 100644 core/sys/info/cpu_riscv64.odin create mode 100644 core/sys/linux/syscall_riscv64.odin create mode 100755 misc/featuregen/build_featuregen.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb5ad0d27..455a451e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -220,3 +220,53 @@ jobs: run: | call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat odin check examples/all -strict-style -target:windows_i386 + + build_linux_riscv64: + runs-on: ubuntu-latest + name: Linux riscv64 (emulated) Build, Check and Test + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - name: Download LLVM (Linux) + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + echo "/usr/lib/llvm-18/bin" >> $GITHUB_PATH + + - name: Build Odin + run: ./build_odin.sh release + + - name: Odin version + run: ./odin version + + - name: Odin report + run: ./odin report + + - name: Compile needed Vendor + run: | + make -C vendor/stb/src + make -C vendor/cgltf/src + make -C vendor/miniaudio/src + + - name: Odin check + run: ./odin check examples/all -target:linux_riscv64 -vet -strict-style -disallow-do + + - name: Install riscv64 toolchain and qemu + run: sudo apt-get install -y qemu-user qemu-user-static gcc-12-riscv64-linux-gnu libc6-riscv64-cross + + - name: Odin run + run: ./odin run examples/demo -target:linux_riscv64 -extra-linker-flags:"-fuse-ld=/usr/bin/riscv64-linux-gnu-gcc-12 -static -Wl,-static" + + - name: Odin run -debug + run: ./odin run examples/demo -debug -target:linux_riscv64 -extra-linker-flags:"-fuse-ld=/usr/bin/riscv64-linux-gnu-gcc-12 -static -Wl,-static" + + - name: Normal Core library tests + run: ./odin test tests/core/normal.odin -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true -target:linux_riscv64 -extra-linker-flags:"-fuse-ld=/usr/bin/riscv64-linux-gnu-gcc-12 -static -Wl,-static" + + - name: Optimized Core library tests + run: ./odin test tests/core/speed.odin -o:speed -file -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true -target:linux_riscv64 -extra-linker-flags:"-fuse-ld=/usr/bin/riscv64-linux-gnu-gcc-12 -static -Wl,-static" + + - name: Internals tests + run: ./odin test tests/internal -all-packages -define:ODIN_TEST_FANCY=false -define:ODIN_TEST_FAIL_ON_BAD_MEMORY=true -target:linux_riscv64 -extra-linker-flags:"-fuse-ld=/usr/bin/riscv64-linux-gnu-gcc-12 -static -Wl,-static" diff --git a/base/runtime/entry_unix.odin b/base/runtime/entry_unix.odin index 7d7252625..5dfd37f99 100644 --- a/base/runtime/entry_unix.odin +++ b/base/runtime/entry_unix.odin @@ -34,6 +34,9 @@ when ODIN_BUILD_MODE == .Dynamic { } else when ODIN_OS == .Darwin && ODIN_ARCH == .arm64 { @require foreign import entry "entry_unix_no_crt_darwin_arm64.asm" SYS_exit :: 1 + } else when ODIN_ARCH == .riscv64 { + @require foreign import entry "entry_unix_no_crt_riscv64.asm" + SYS_exit :: 93 } @(link_name="_start_odin", linkage="strong", require) _start_odin :: proc "c" (argc: i32, argv: [^]cstring) -> ! { diff --git a/base/runtime/entry_unix_no_crt_riscv64.asm b/base/runtime/entry_unix_no_crt_riscv64.asm new file mode 100644 index 000000000..756515b72 --- /dev/null +++ b/base/runtime/entry_unix_no_crt_riscv64.asm @@ -0,0 +1,10 @@ +.text + +.globl _start + +_start: + ld a0, 0(sp) + addi a1, sp, 8 + addi sp, sp, ~15 + call _start_odin + ebreak diff --git a/base/runtime/os_specific_linux.odin b/base/runtime/os_specific_linux.odin index a944ba309..146e647fb 100644 --- a/base/runtime/os_specific_linux.odin +++ b/base/runtime/os_specific_linux.odin @@ -12,6 +12,8 @@ _stderr_write :: proc "contextless" (data: []byte) -> (int, _OS_Errno) { SYS_write :: uintptr(4) } else when ODIN_ARCH == .arm32 { SYS_write :: uintptr(4) + } else when ODIN_ARCH == .riscv64 { + SYS_write :: uintptr(64) } stderr :: 2 diff --git a/core/crypto/_chacha20/simd128/chacha20_simd128.odin b/core/crypto/_chacha20/simd128/chacha20_simd128.odin index 4cab3c5e8..2f91ac52a 100644 --- a/core/crypto/_chacha20/simd128/chacha20_simd128.odin +++ b/core/crypto/_chacha20/simd128/chacha20_simd128.odin @@ -3,7 +3,7 @@ package chacha20_simd128 import "base:intrinsics" import "core:crypto/_chacha20" import "core:simd" -import "core:sys/info" +@(require) import "core:sys/info" // Portable 128-bit `core:simd` implementation. // diff --git a/core/net/socket_linux.odin b/core/net/socket_linux.odin index dce428685..52f328814 100644 --- a/core/net/socket_linux.odin +++ b/core/net/socket_linux.odin @@ -33,8 +33,8 @@ Socket_Option :: enum c.int { Linger = c.int(linux.Socket_Option.LINGER), Receive_Buffer_Size = c.int(linux.Socket_Option.RCVBUF), Send_Buffer_Size = c.int(linux.Socket_Option.SNDBUF), - Receive_Timeout = c.int(linux.Socket_Option.RCVTIMEO_NEW), - Send_Timeout = c.int(linux.Socket_Option.SNDTIMEO_NEW), + Receive_Timeout = c.int(linux.Socket_Option.RCVTIMEO), + Send_Timeout = c.int(linux.Socket_Option.SNDTIMEO), } // Wrappers and unwrappers for system-native types diff --git a/core/os/os_linux.odin b/core/os/os_linux.odin index 0fcd1a21a..f1b3720c6 100644 --- a/core/os/os_linux.odin +++ b/core/os/os_linux.odin @@ -284,6 +284,25 @@ when ODIN_ARCH == .arm64 { _reserved: [2]i32, } #assert(size_of(OS_Stat) == 128) +} else when ODIN_ARCH == .riscv64 { + OS_Stat :: struct { + device_id: u64, + serial: u64, + mode: u32, + nlink: u32, + uid: u32, + gid: u32, + rdev: u64, + _: u64, + size: i64, + block_size: i32, + _: i32, + blocks: i64, + last_access: Unix_File_Time, + modified: Unix_File_Time, + status_change: Unix_File_Time, + _: [3]uint, + } } else { OS_Stat :: struct { device_id: u64, // ID of device containing file diff --git a/core/sys/info/cpu_linux_riscv64.odin b/core/sys/info/cpu_linux_riscv64.odin new file mode 100644 index 000000000..0f109e7ba --- /dev/null +++ b/core/sys/info/cpu_linux_riscv64.odin @@ -0,0 +1,46 @@ +//+build riscv64 +//+build linux +package sysinfo + +import "base:intrinsics" + +import "core:sys/linux" + +@(init, private) +init_cpu_features :: proc() { + fd, err := linux.open("/proc/self/auxv", {}) + if err != .NONE { return } + defer linux.close(fd) + + // This is probably enough right? + buf: [4096]byte + n, rerr := linux.read(fd, buf[:]) + if rerr != .NONE || n == 0 { return } + + ulong :: u64 + AT_HWCAP :: 16 + + // TODO: using these we could get more information than just the basics. + // AT_HWCAP2 :: 26 + // AT_HWCAP3 :: 29 + // AT_HWCAP4 :: 30 + + auxv := buf[:n] + for len(auxv) >= size_of(ulong)*2 { + key := intrinsics.unaligned_load((^ulong)(&auxv[0])) + val := intrinsics.unaligned_load((^ulong)(&auxv[size_of(ulong)])) + auxv = auxv[2*size_of(ulong):] + + if key != AT_HWCAP { + continue + } + + cpu_features = transmute(CPU_Features)(val) + break + } +} + +@(init, private) +init_cpu_name :: proc() { + cpu_name = "RISCV64" +} diff --git a/core/sys/info/cpu_riscv64.odin b/core/sys/info/cpu_riscv64.odin new file mode 100644 index 000000000..754110911 --- /dev/null +++ b/core/sys/info/cpu_riscv64.odin @@ -0,0 +1,16 @@ +package sysinfo + +CPU_Feature :: enum u64 { + I = 'I' - 'A', // Base features, don't think this is ever not here. + M = 'M' - 'A', // Integer multiplication and division, currently required by Odin. + A = 'A' - 'A', // Atomics. + F = 'F' - 'A', // Single precision floating point, currently required by Odin. + D = 'D' - 'A', // Double precision floating point, currently required by Odin. + C = 'C' - 'A', // Compressed instructions. + V = 'V' - 'A', // Vector operations. +} + +CPU_Features :: distinct bit_set[CPU_Feature; u64] + +cpu_features: Maybe(CPU_Features) +cpu_name: Maybe(string) diff --git a/core/sys/info/sysinfo.odin b/core/sys/info/sysinfo.odin index f0262f317..f624a1718 100644 --- a/core/sys/info/sysinfo.odin +++ b/core/sys/info/sysinfo.odin @@ -1,6 +1,6 @@ package sysinfo -when !(ODIN_ARCH == .amd64 || ODIN_ARCH == .i386 || ODIN_ARCH == .arm32 || ODIN_ARCH == .arm64) { +when !(ODIN_ARCH == .amd64 || ODIN_ARCH == .i386 || ODIN_ARCH == .arm32 || ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64) { #assert(false, "This package is unsupported on this architecture.") } diff --git a/core/sys/linux/bits.odin b/core/sys/linux/bits.odin index b8ec3c133..f78891bc8 100644 --- a/core/sys/linux/bits.odin +++ b/core/sys/linux/bits.odin @@ -1343,14 +1343,16 @@ Socket_Option :: enum { RESERVE_MEM = 73, TXREHASH = 74, RCVMARK = 75, - // Hardcoded 64-bit Time. It's time to move on. - TIMESTAMP = TIMESTAMP_NEW, - TIMESTAMPNS = TIMESTAMPNS_NEW, - TIMESTAMPING = TIMESTAMPING_NEW, - RCVTIMEO = RCVTIMEO_NEW, - SNDTIMEO = SNDTIMEO_NEW, + TIMESTAMP = TIMESTAMP_OLD when _SOCKET_OPTION_OLD else TIMESTAMP_NEW, + TIMESTAMPNS = TIMESTAMPNS_OLD when _SOCKET_OPTION_OLD else TIMESTAMPNS_NEW, + TIMESTAMPING = TIMESTAMPING_OLD when _SOCKET_OPTION_OLD else TIMESTAMPING_NEW, + RCVTIMEO = RCVTIMEO_OLD when _SOCKET_OPTION_OLD else RCVTIMEO_NEW, + SNDTIMEO = SNDTIMEO_OLD when _SOCKET_OPTION_OLD else SNDTIMEO_NEW, } +@(private) +_SOCKET_OPTION_OLD :: size_of(rawptr) == 8 /* || size_of(time_t) == size_of(__kernel_long_t) */ + Socket_UDP_Option :: enum { CORK = 1, ENCAP = 100, diff --git a/core/sys/linux/sys.odin b/core/sys/linux/sys.odin index f4f609ab9..a6d4f723d 100644 --- a/core/sys/linux/sys.odin +++ b/core/sys/linux/sys.odin @@ -39,7 +39,7 @@ write :: proc "contextless" (fd: Fd, buf: []u8) -> (int, Errno) { On ARM64 available since Linux 2.6.16. */ open :: proc "contextless" (name: cstring, flags: Open_Flags, mode: Mode = {}) -> (Fd, Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_openat, AT_FDCWD, transmute(uintptr) name, transmute(u32) flags, transmute(u32) mode) return errno_unwrap(ret, Fd) } else { @@ -68,7 +68,7 @@ close :: proc "contextless" (fd: Fd) -> (Errno) { */ stat :: proc "contextless" (filename: cstring, stat: ^Stat) -> (Errno) { when size_of(int) == 8 { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_fstatat, AT_FDCWD, cast(rawptr) filename, stat, 0) return Errno(-ret) } else { @@ -111,7 +111,7 @@ fstat :: proc "contextless" (fd: Fd, stat: ^Stat) -> (Errno) { */ lstat :: proc "contextless" (filename: cstring, stat: ^Stat) -> (Errno) { when size_of(int) == 8 { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { return fstatat(AT_FDCWD, filename, stat, {.SYMLINK_NOFOLLOW}) } else { ret := syscall(SYS_lstat, cast(rawptr) filename, stat) @@ -128,7 +128,7 @@ lstat :: proc "contextless" (filename: cstring, stat: ^Stat) -> (Errno) { Available since Linux 2.2. */ poll :: proc "contextless" (fds: []Poll_Fd, timeout: i32) -> (i32, Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { seconds := cast(uint) timeout / 1000 nanoseconds := cast(uint) (timeout % 1000) * 1_000_000 timeout_spec := Time_Spec{seconds, nanoseconds} @@ -291,7 +291,7 @@ writev :: proc "contextless" (fd: Fd, iov: []IO_Vec) -> (int, Errno) { For ARM64 available since Linux 2.6.16. */ access :: proc "contextless" (name: cstring, mode: Mode = F_OK) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_faccessat, AT_FDCWD, cast(rawptr) name, transmute(u32) mode) return Errno(-ret) } else { @@ -407,7 +407,7 @@ dup :: proc "contextless" (fd: Fd) -> (Fd, Errno) { On ARM64 available since Linux 2.6.27. */ dup2 :: proc "contextless" (old: Fd, new: Fd) -> (Fd, Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_dup3, old, new, 0) return errno_unwrap(ret, Fd) } else { @@ -422,7 +422,7 @@ dup2 :: proc "contextless" (old: Fd, new: Fd) -> (Fd, Errno) { On ARM64 available since Linux 2.6.16. */ pause :: proc "contextless" () { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { syscall(SYS_ppoll, 0, 0, 0, 0) } else { syscall(SYS_pause) @@ -452,7 +452,7 @@ getitimer :: proc "contextless" (which: ITimer_Which, cur: ^ITimer_Val) -> (Errn Available since Linux 1.0. */ alarm :: proc "contextless" (seconds: u32) -> u32 { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { new := ITimer_Val { value = { seconds = cast(int) seconds } } old := ITimer_Val {} syscall(SYS_setitimer, ITimer_Which.REAL, &new, &old) @@ -765,7 +765,7 @@ getsockopt :: proc { Available since Linux 1.0. */ fork :: proc "contextless" () -> (Pid, Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_clone, u64(Signal.SIGCHLD), cast(rawptr) nil, cast(rawptr) nil, cast(rawptr) nil, u64(0)) return errno_unwrap(ret, Pid) } else { @@ -779,7 +779,7 @@ fork :: proc "contextless" () -> (Pid, Errno) { Available since Linux 2.2. */ vfork :: proc "contextless" () -> Pid { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return Pid(syscall(SYS_vfork)) } else { return Pid(syscall(SYS_clone, Signal.SIGCHLD)) @@ -792,7 +792,7 @@ vfork :: proc "contextless" () -> Pid { On ARM64 available since Linux 3.19. */ execve :: proc "contextless" (name: cstring, argv: [^]cstring, envp: [^]cstring) -> (Errno) { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { ret := syscall(SYS_execve, cast(rawptr) name, cast(rawptr) argv, cast(rawptr) envp) return Errno(-ret) } else { @@ -1193,7 +1193,7 @@ fchdir :: proc "contextless" (fd: Fd) -> (Errno) { On ARM64 available since Linux 2.6.16. */ rename :: proc "contextless" (old: cstring, new: cstring) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_renameat, AT_FDCWD, cast(rawptr) old, AT_FDCWD, cast(rawptr) new) return Errno(-ret) } else { @@ -1208,7 +1208,7 @@ rename :: proc "contextless" (old: cstring, new: cstring) -> (Errno) { On ARM64 available since Linux 2.6.16. */ mkdir :: proc "contextless" (name: cstring, mode: Mode) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_mkdirat, AT_FDCWD, cast(rawptr) name, transmute(u32) mode) return Errno(-ret) } else { @@ -1223,7 +1223,7 @@ mkdir :: proc "contextless" (name: cstring, mode: Mode) -> (Errno) { On ARM64 available since Linux 2.6.16. */ rmdir :: proc "contextless" (name: cstring) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_unlinkat, AT_FDCWD, cast(rawptr) name, transmute(i32) FD_Flags{.REMOVEDIR}) return Errno(-ret) } else { @@ -1238,7 +1238,7 @@ rmdir :: proc "contextless" (name: cstring) -> (Errno) { On ARM64 available since Linux 2.6.16. */ creat :: proc "contextless" (name: cstring, mode: Mode) -> (Fd, Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { return openat(AT_FDCWD, name, {.CREAT, .WRONLY,.TRUNC}, mode) } else { ret := syscall(SYS_creat, cast(rawptr) name, transmute(u32) mode) @@ -1252,7 +1252,7 @@ creat :: proc "contextless" (name: cstring, mode: Mode) -> (Fd, Errno) { On ARM64 available since Linux 2.6.16. */ link :: proc "contextless" (target: cstring, linkpath: cstring) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_linkat, AT_FDCWD, cast(rawptr) target, AT_FDCWD, cast(rawptr) linkpath, 0) return Errno(-ret) } else { @@ -1267,7 +1267,7 @@ link :: proc "contextless" (target: cstring, linkpath: cstring) -> (Errno) { On ARM64 available since Linux 2.6.16. */ unlink :: proc "contextless" (name: cstring) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_unlinkat, AT_FDCWD, cast(rawptr) name, 0) return Errno(-ret) } else { @@ -1282,7 +1282,7 @@ unlink :: proc "contextless" (name: cstring) -> (Errno) { On arm64 available since Linux 2.6.16. */ symlink :: proc "contextless" (target: cstring, linkpath: cstring) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_symlinkat, cast(rawptr) target, AT_FDCWD, cast(rawptr) linkpath) return Errno(-ret) } else { @@ -1297,7 +1297,7 @@ symlink :: proc "contextless" (target: cstring, linkpath: cstring) -> (Errno) { On arm64 available since Linux 2.6.16. */ readlink :: proc "contextless" (name: cstring, buf: []u8) -> (int, Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_readlinkat, AT_FDCWD, cast(rawptr) name, raw_data(buf), len(buf)) return errno_unwrap(ret, int) } else { @@ -1312,7 +1312,7 @@ readlink :: proc "contextless" (name: cstring, buf: []u8) -> (int, Errno) { On ARM64 available since Linux 2.6.16. */ chmod :: proc "contextless" (name: cstring, mode: Mode) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_fchmodat, AT_FDCWD, cast(rawptr) name, transmute(u32) mode) return Errno(-ret) } else { @@ -1340,7 +1340,7 @@ chown :: proc "contextless" (name: cstring, uid: Uid, gid: Gid) -> (Errno) { when size_of(int) == 4 { ret := syscall(SYS_chown32, cast(rawptr) name, uid, gid) return Errno(-ret) - } else when ODIN_ARCH == .arm64 { + } else when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_fchownat, AT_FDCWD, cast(rawptr) name, uid, gid, 0) return Errno(-ret) } else { @@ -1374,7 +1374,7 @@ lchown :: proc "contextless" (name: cstring, uid: Uid, gid: Gid) -> (Errno) { when size_of(int) == 4 { ret := syscall(SYS_lchown32, cast(rawptr) name, uid, gid) return Errno(-ret) - } else when ODIN_ARCH == .arm64 { + } else when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_fchownat, AT_FDCWD, cast(rawptr) name, uid, gid, transmute(i32) FD_Flags{.SYMLINK_NOFOLLOW}) return Errno(-ret) } else { @@ -1727,7 +1727,7 @@ getppid :: proc "contextless" () -> Pid { Available since Linux 1.0. */ getpgrp :: proc "contextless" () -> (Pid, Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_getpgid, 0) return errno_unwrap(ret, Pid) } else { @@ -1950,7 +1950,7 @@ sigaltstack :: proc "contextless" (stack: ^Sig_Stack, old_stack: ^Sig_Stack) -> On ARM64 available since Linux 2.6.16. */ mknod :: proc "contextless" (name: cstring, mode: Mode, dev: Dev) -> (Errno) { - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { ret := syscall(SYS_mknodat, AT_FDCWD, cast(rawptr) name, transmute(u32) mode, dev) return Errno(-ret) } else { @@ -2207,7 +2207,7 @@ gettid :: proc "contextless" () -> Pid { Available since Linux 1.0. */ time :: proc "contextless" (tloc: ^uint) -> (Errno) { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { ret := syscall(SYS_time, tloc) return Errno(-ret) } else { @@ -2335,7 +2335,7 @@ futex :: proc{ Available since Linux 2.6. */ epoll_create :: proc(size: i32 = 1) -> (Fd, Errno) { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { ret := syscall(SYS_epoll_create, i32(1)) return errno_unwrap(ret, Fd) } else { @@ -2433,7 +2433,7 @@ exit_group :: proc "contextless" (code: i32) -> ! { Available since Linux 2.6. */ epoll_wait :: proc(epfd: Fd, events: [^]EPoll_Event, count: i32, timeout: i32) -> (i32, Errno) { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { ret := syscall(SYS_epoll_wait, epfd, events, count, timeout) return errno_unwrap(ret, i32) } else { diff --git a/core/sys/linux/syscall_riscv64.odin b/core/sys/linux/syscall_riscv64.odin new file mode 100644 index 000000000..ce374312e --- /dev/null +++ b/core/sys/linux/syscall_riscv64.odin @@ -0,0 +1,334 @@ +//+build riscv64 +package linux + +// https://github.com/riscv-collab/riscv-gnu-toolchain/blob/master/linux-headers/include/asm-generic/unistd.h + +SYS_io_setup :: uintptr(0) +SYS_io_destroy :: uintptr(1) +SYS_io_submit :: uintptr(2) +SYS_io_cancel :: uintptr(3) +SYS_io_getevents :: uintptr(4) +SYS_setxattr :: uintptr(5) +SYS_lsetxattr :: uintptr(6) +SYS_fsetxattr :: uintptr(7) +SYS_getxattr :: uintptr(8) +SYS_lgetxattr :: uintptr(9) +SYS_fgetxattr :: uintptr(10) +SYS_listxattr :: uintptr(11) +SYS_llistxattr :: uintptr(12) +SYS_flistxattr :: uintptr(13) +SYS_removexattr :: uintptr(14) +SYS_lremovexattr :: uintptr(15) +SYS_fremovexattr :: uintptr(16) +SYS_getcwd :: uintptr(17) +SYS_lookup_dcookie :: uintptr(18) +SYS_eventfd2 :: uintptr(19) +SYS_epoll_create1 :: uintptr(20) +SYS_epoll_ctl :: uintptr(21) +SYS_epoll_pwait :: uintptr(22) +SYS_dup :: uintptr(23) +SYS_dup3 :: uintptr(24) +SYS_fcntl :: uintptr(25) +SYS_inotify_init1 :: uintptr(26) +SYS_inotify_add_watch :: uintptr(27) +SYS_inotify_rm_watch :: uintptr(28) +SYS_ioctl :: uintptr(29) +SYS_ioprio_set :: uintptr(30) +SYS_ioprio_get :: uintptr(31) +SYS_flock :: uintptr(32) +SYS_mknodat :: uintptr(33) +SYS_mkdirat :: uintptr(34) +SYS_unlinkat :: uintptr(35) +SYS_symlinkat :: uintptr(36) +SYS_linkat :: uintptr(37) +SYS_renameat :: uintptr(38) +SYS_umount2 :: uintptr(39) +SYS_mount :: uintptr(40) +SYS_pivot_root :: uintptr(41) +SYS_nfsservctl :: uintptr(42) +SYS_statfs :: uintptr(43) +SYS_fstatfs :: uintptr(44) +SYS_truncate :: uintptr(45) +SYS_ftruncate :: uintptr(46) +SYS_fallocate :: uintptr(47) +SYS_faccessat :: uintptr(48) +SYS_chdir :: uintptr(49) +SYS_fchdir :: uintptr(50) +SYS_chroot :: uintptr(51) +SYS_fchmod :: uintptr(52) +SYS_fchmodat :: uintptr(53) +SYS_fchownat :: uintptr(54) +SYS_fchown :: uintptr(55) +SYS_openat :: uintptr(56) +SYS_close :: uintptr(57) +SYS_vhangup :: uintptr(58) +SYS_pipe2 :: uintptr(59) +SYS_quotactl :: uintptr(60) +SYS_getdents64 :: uintptr(61) +SYS_lseek :: uintptr(62) +SYS_read :: uintptr(63) +SYS_write :: uintptr(64) +SYS_readv :: uintptr(65) +SYS_writev :: uintptr(66) +SYS_pread64 :: uintptr(67) +SYS_pwrite64 :: uintptr(68) +SYS_preadv :: uintptr(69) +SYS_pwritev :: uintptr(70) +SYS_sendfile :: uintptr(71) +SYS_pselect6 :: uintptr(72) +SYS_ppoll :: uintptr(73) +SYS_signalfd4 :: uintptr(74) +SYS_vmsplice :: uintptr(75) +SYS_splice :: uintptr(76) +SYS_tee :: uintptr(77) +SYS_readlinkat :: uintptr(78) +SYS_fstatat :: uintptr(79) +SYS_fstat :: uintptr(80) +SYS_sync :: uintptr(81) +SYS_fsync :: uintptr(82) +SYS_fdatasync :: uintptr(83) +SYS_sync_file_range2 :: uintptr(84) +SYS_sync_file_range :: uintptr(84) +SYS_timerfd_create :: uintptr(85) +SYS_timerfd_settime :: uintptr(86) +SYS_timerfd_gettime :: uintptr(87) +SYS_utimensat :: uintptr(88) +SYS_acct :: uintptr(89) +SYS_capget :: uintptr(90) +SYS_capset :: uintptr(91) +SYS_personality :: uintptr(92) +SYS_exit :: uintptr(93) +SYS_exit_group :: uintptr(94) +SYS_waitid :: uintptr(95) +SYS_set_tid_address :: uintptr(96) +SYS_unshare :: uintptr(97) +SYS_futex :: uintptr(98) +SYS_set_robust_list :: uintptr(99) +SYS_get_robust_list :: uintptr(100) +SYS_nanosleep :: uintptr(101) +SYS_getitimer :: uintptr(102) +SYS_setitimer :: uintptr(103) +SYS_kexec_load :: uintptr(104) +SYS_init_module :: uintptr(105) +SYS_delete_module :: uintptr(106) +SYS_timer_create :: uintptr(107) +SYS_timer_gettime :: uintptr(108) +SYS_timer_getoverrun :: uintptr(109) +SYS_timer_settime :: uintptr(110) +SYS_timer_delete :: uintptr(111) +SYS_clock_settime :: uintptr(112) +SYS_clock_gettime :: uintptr(113) +SYS_clock_getres :: uintptr(114) +SYS_clock_nanosleep :: uintptr(115) +SYS_syslog :: uintptr(116) +SYS_ptrace :: uintptr(117) +SYS_sched_setparam :: uintptr(118) +SYS_sched_setscheduler :: uintptr(119) +SYS_sched_getscheduler :: uintptr(120) +SYS_sched_getparam :: uintptr(121) +SYS_sched_setaffinity :: uintptr(122) +SYS_sched_getaffinity :: uintptr(123) +SYS_sched_yield :: uintptr(124) +SYS_sched_get_priority_max :: uintptr(125) +SYS_sched_get_priority_min :: uintptr(126) +SYS_sched_rr_get_interval :: uintptr(127) +SYS_restart_syscall :: uintptr(128) +SYS_kill :: uintptr(129) +SYS_tkill :: uintptr(130) +SYS_tgkill :: uintptr(131) +SYS_sigaltstack :: uintptr(132) +SYS_rt_sigsuspend :: uintptr(133) +SYS_rt_sigaction :: uintptr(134) +SYS_rt_sigprocmask :: uintptr(135) +SYS_rt_sigpending :: uintptr(136) +SYS_rt_sigtimedwait :: uintptr(137) +SYS_rt_sigqueueinfo :: uintptr(138) +SYS_rt_sigreturn :: uintptr(139) +SYS_setpriority :: uintptr(140) +SYS_getpriority :: uintptr(141) +SYS_reboot :: uintptr(142) +SYS_setregid :: uintptr(143) +SYS_setgid :: uintptr(144) +SYS_setreuid :: uintptr(145) +SYS_setuid :: uintptr(146) +SYS_setresuid :: uintptr(147) +SYS_getresuid :: uintptr(148) +SYS_setresgid :: uintptr(149) +SYS_getresgid :: uintptr(150) +SYS_setfsuid :: uintptr(151) +SYS_setfsgid :: uintptr(152) +SYS_times :: uintptr(153) +SYS_setpgid :: uintptr(154) +SYS_getpgid :: uintptr(155) +SYS_getsid :: uintptr(156) +SYS_setsid :: uintptr(157) +SYS_getgroups :: uintptr(158) +SYS_setgroups :: uintptr(159) +SYS_uname :: uintptr(160) +SYS_sethostname :: uintptr(161) +SYS_setdomainname :: uintptr(162) +SYS_getrlimit :: uintptr(163) +SYS_setrlimit :: uintptr(164) +SYS_getrusage :: uintptr(165) +SYS_umask :: uintptr(166) +SYS_prctl :: uintptr(167) +SYS_getcpu :: uintptr(168) +SYS_gettimeofday :: uintptr(169) +SYS_settimeofday :: uintptr(170) +SYS_adjtimex :: uintptr(171) +SYS_getpid :: uintptr(172) +SYS_getppid :: uintptr(173) +SYS_getuid :: uintptr(174) +SYS_geteuid :: uintptr(175) +SYS_getgid :: uintptr(176) +SYS_getegid :: uintptr(177) +SYS_gettid :: uintptr(178) +SYS_sysinfo :: uintptr(179) +SYS_mq_open :: uintptr(180) +SYS_mq_unlink :: uintptr(181) +SYS_mq_timedsend :: uintptr(182) +SYS_mq_timedreceive :: uintptr(183) +SYS_mq_notify :: uintptr(184) +SYS_mq_getsetattr :: uintptr(185) +SYS_msgget :: uintptr(186) +SYS_msgctl :: uintptr(187) +SYS_msgrcv :: uintptr(188) +SYS_msgsnd :: uintptr(189) +SYS_semget :: uintptr(190) +SYS_semctl :: uintptr(191) +SYS_semtimedop :: uintptr(192) +SYS_semop :: uintptr(193) +SYS_shmget :: uintptr(194) +SYS_shmctl :: uintptr(195) +SYS_shmat :: uintptr(196) +SYS_shmdt :: uintptr(197) +SYS_socket :: uintptr(198) +SYS_socketpair :: uintptr(199) +SYS_bind :: uintptr(200) +SYS_listen :: uintptr(201) +SYS_accept :: uintptr(202) +SYS_connect :: uintptr(203) +SYS_getsockname :: uintptr(204) +SYS_getpeername :: uintptr(205) +SYS_sendto :: uintptr(206) +SYS_recvfrom :: uintptr(207) +SYS_setsockopt :: uintptr(208) +SYS_getsockopt :: uintptr(209) +SYS_shutdown :: uintptr(210) +SYS_sendmsg :: uintptr(211) +SYS_recvmsg :: uintptr(212) +SYS_readahead :: uintptr(213) +SYS_brk :: uintptr(214) +SYS_munmap :: uintptr(215) +SYS_mremap :: uintptr(216) +SYS_add_key :: uintptr(217) +SYS_request_key :: uintptr(218) +SYS_keyctl :: uintptr(219) +SYS_clone :: uintptr(220) +SYS_execve :: uintptr(221) +SYS_mmap :: uintptr(222) +SYS_fadvise64 :: uintptr(223) +SYS_swapon :: uintptr(224) +SYS_swapoff :: uintptr(225) +SYS_mprotect :: uintptr(226) +SYS_msync :: uintptr(227) +SYS_mlock :: uintptr(228) +SYS_munlock :: uintptr(229) +SYS_mlockall :: uintptr(230) +SYS_munlockall :: uintptr(231) +SYS_mincore :: uintptr(232) +SYS_madvise :: uintptr(233) +SYS_remap_file_pages :: uintptr(234) +SYS_mbind :: uintptr(235) +SYS_get_mempolicy :: uintptr(236) +SYS_set_mempolicy :: uintptr(237) +SYS_migrate_pages :: uintptr(238) +SYS_move_pages :: uintptr(239) +SYS_rt_tgsigqueueinfo :: uintptr(240) +SYS_perf_event_open :: uintptr(241) +SYS_accept4 :: uintptr(242) +SYS_recvmmsg :: uintptr(243) +SYS_wait4 :: uintptr(260) +SYS_prlimit64 :: uintptr(261) +SYS_fanotify_init :: uintptr(262) +SYS_fanotify_mark :: uintptr(263) +SYS_name_to_handle_at :: uintptr(264) +SYS_open_by_handle_at :: uintptr(265) +SYS_clock_adjtime :: uintptr(266) +SYS_syncfs :: uintptr(267) +SYS_setns :: uintptr(268) +SYS_sendmmsg :: uintptr(269) +SYS_process_vm_readv :: uintptr(270) +SYS_process_vm_writev :: uintptr(271) +SYS_kcmp :: uintptr(272) +SYS_finit_module :: uintptr(273) +SYS_sched_setattr :: uintptr(274) +SYS_sched_getattr :: uintptr(275) +SYS_renameat2 :: uintptr(276) +SYS_seccomp :: uintptr(277) +SYS_getrandom :: uintptr(278) +SYS_memfd_create :: uintptr(279) +SYS_bpf :: uintptr(280) +SYS_execveat :: uintptr(281) +SYS_userfaultfd :: uintptr(282) +SYS_membarrier :: uintptr(283) +SYS_mlock2 :: uintptr(284) +SYS_copy_file_range :: uintptr(285) +SYS_preadv2 :: uintptr(286) +SYS_pwritev2 :: uintptr(287) +SYS_pkey_mprotect :: uintptr(288) +SYS_pkey_alloc :: uintptr(289) +SYS_pkey_free :: uintptr(290) +SYS_statx :: uintptr(291) +SYS_io_pgetevents :: uintptr(292) +SYS_rseq :: uintptr(293) +SYS_kexec_file_load :: uintptr(294) +SYS_clock_gettime64 :: uintptr(403) +SYS_clock_settime64 :: uintptr(404) +SYS_clock_adjtime64 :: uintptr(405) +SYS_clock_getres_time64 :: uintptr(406) +SYS_clock_nanosleep_time64 :: uintptr(407) +SYS_timer_gettime64 :: uintptr(408) +SYS_timer_settime64 :: uintptr(409) +SYS_timerfd_gettime64 :: uintptr(410) +SYS_timerfd_settime64 :: uintptr(411) +SYS_utimensat_time64 :: uintptr(412) +SYS_pselect6_time64 :: uintptr(413) +SYS_ppoll_time64 :: uintptr(414) +SYS_io_pgetevents_time64 :: uintptr(416) +SYS_recvmmsg_time64 :: uintptr(417) +SYS_mq_timedsend_time64 :: uintptr(418) +SYS_mq_timedreceive_time64 :: uintptr(419) +SYS_semtimedop_time64 :: uintptr(420) +SYS_rt_sigtimedwait_time64 :: uintptr(421) +SYS_futex_time64 :: uintptr(422) +SYS_sched_rr_get_interval_time64 :: uintptr(423) +SYS_pidfd_send_signal :: uintptr(424) +SYS_io_uring_setup :: uintptr(425) +SYS_io_uring_enter :: uintptr(426) +SYS_io_uring_register :: uintptr(427) +SYS_open_tree :: uintptr(428) +SYS_move_mount :: uintptr(429) +SYS_fsopen :: uintptr(430) +SYS_fsconfig :: uintptr(431) +SYS_fsmount :: uintptr(432) +SYS_fspick :: uintptr(433) +SYS_pidfd_open :: uintptr(434) +SYS_clone3 :: uintptr(435) +SYS_close_range :: uintptr(436) +SYS_openat2 :: uintptr(437) +SYS_pidfd_getfd :: uintptr(438) +SYS_faccessat2 :: uintptr(439) +SYS_process_madvise :: uintptr(440) +SYS_epoll_pwait2 :: uintptr(441) +SYS_mount_setattr :: uintptr(442) +SYS_quotactl_fd :: uintptr(443) +SYS_landlock_create_ruleset :: uintptr(444) +SYS_landlock_add_rule :: uintptr(445) +SYS_landlock_restrict_self :: uintptr(446) +SYS_memfd_secret :: uintptr(447) +SYS_process_mrelease :: uintptr(448) +SYS_futex_waitv :: uintptr(449) +SYS_set_mempolicy_home_node :: uintptr(450) +SYS_cachestat :: uintptr(451) +SYS_fchmodat2 :: uintptr(452) diff --git a/core/sys/linux/types.odin b/core/sys/linux/types.odin index 288edf879..c78a5b576 100644 --- a/core/sys/linux/types.odin +++ b/core/sys/linux/types.odin @@ -120,6 +120,25 @@ when ODIN_ARCH == .amd64 { _: [3]uint, } } else when ODIN_ARCH == .arm64 { + _Arch_Stat :: struct { + dev: Dev, + ino: Inode, + mode: Mode, + nlink: u32, + uid: Uid, + gid: Gid, + rdev: Dev, + _: u64, + size: int, + blksize: i32, + _: i32, + blocks: int, + atime: Time_Spec, + mtime: Time_Spec, + ctime: Time_Spec, + _: [2]u32, + } +} else when ODIN_ARCH == .riscv64 { _Arch_Stat :: struct { dev: Dev, ino: Inode, @@ -927,7 +946,7 @@ when ODIN_ARCH == .i386 { nsems: uint, _: [2]uint, } -} else when ODIN_ARCH == .arm64 { +} else when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { _Arch_Semid_DS :: struct { perm: IPC_Perm, otime: int, @@ -1167,6 +1186,33 @@ when ODIN_ARCH == .arm32 { xmm_space: [32]uint, padding: [56]uint, } +} else when ODIN_ARCH == .riscv64 { + _Arch_User_Regs :: struct { + pc, ra, sp, gp, tp, + t0, t1, t2, + s0, s1, + a0, a1, a2, a3, a4, a5, a6, a7, + s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, + t3, t4, t5, t6: uint, + } + _Arch_User_FP_Regs :: struct #raw_union { + f_ext: struct { + f: [32]u32, + fcsr: u32, + }, + d_ext: struct { + f: [32]u64, + fcsr: u32, + }, + q_ext: struct { + using _: struct #align(16) { + f: [64]u64, + }, + fcsr: u32, + reserved: [3]u32, + }, + } + _Arch_User_FPX_Regs :: struct {} } /* diff --git a/core/sys/unix/syscalls_linux.odin b/core/sys/unix/syscalls_linux.odin index 038c16276..89ad2661f 100644 --- a/core/sys/unix/syscalls_linux.odin +++ b/core/sys/unix/syscalls_linux.odin @@ -1514,6 +1514,338 @@ when ODIN_ARCH == .amd64 { SYS_landlock_create_ruleset : uintptr : 444 SYS_landlock_add_rule : uintptr : 445 SYS_landlock_restrict_self : uintptr : 446 +} else when ODIN_ARCH == .riscv64 { + SYS_io_setup :: uintptr(0) + SYS_io_destroy :: uintptr(1) + SYS_io_submit :: uintptr(2) + SYS_io_cancel :: uintptr(3) + SYS_io_getevents :: uintptr(4) + SYS_setxattr :: uintptr(5) + SYS_lsetxattr :: uintptr(6) + SYS_fsetxattr :: uintptr(7) + SYS_getxattr :: uintptr(8) + SYS_lgetxattr :: uintptr(9) + SYS_fgetxattr :: uintptr(10) + SYS_listxattr :: uintptr(11) + SYS_llistxattr :: uintptr(12) + SYS_flistxattr :: uintptr(13) + SYS_removexattr :: uintptr(14) + SYS_lremovexattr :: uintptr(15) + SYS_fremovexattr :: uintptr(16) + SYS_getcwd :: uintptr(17) + SYS_lookup_dcookie :: uintptr(18) + SYS_eventfd2 :: uintptr(19) + SYS_epoll_create1 :: uintptr(20) + SYS_epoll_ctl :: uintptr(21) + SYS_epoll_pwait :: uintptr(22) + SYS_dup :: uintptr(23) + SYS_dup3 :: uintptr(24) + SYS_fcntl :: uintptr(25) + SYS_inotify_init1 :: uintptr(26) + SYS_inotify_add_watch :: uintptr(27) + SYS_inotify_rm_watch :: uintptr(28) + SYS_ioctl :: uintptr(29) + SYS_ioprio_set :: uintptr(30) + SYS_ioprio_get :: uintptr(31) + SYS_flock :: uintptr(32) + SYS_mknodat :: uintptr(33) + SYS_mkdirat :: uintptr(34) + SYS_unlinkat :: uintptr(35) + SYS_symlinkat :: uintptr(36) + SYS_linkat :: uintptr(37) + SYS_renameat :: uintptr(38) + SYS_umount2 :: uintptr(39) + SYS_mount :: uintptr(40) + SYS_pivot_root :: uintptr(41) + SYS_nfsservctl :: uintptr(42) + SYS_statfs :: uintptr(43) + SYS_fstatfs :: uintptr(44) + SYS_truncate :: uintptr(45) + SYS_ftruncate :: uintptr(46) + SYS_fallocate :: uintptr(47) + SYS_faccessat :: uintptr(48) + SYS_chdir :: uintptr(49) + SYS_fchdir :: uintptr(50) + SYS_chroot :: uintptr(51) + SYS_fchmod :: uintptr(52) + SYS_fchmodat :: uintptr(53) + SYS_fchownat :: uintptr(54) + SYS_fchown :: uintptr(55) + SYS_openat :: uintptr(56) + SYS_close :: uintptr(57) + SYS_vhangup :: uintptr(58) + SYS_pipe2 :: uintptr(59) + SYS_quotactl :: uintptr(60) + SYS_getdents64 :: uintptr(61) + SYS_lseek :: uintptr(62) + SYS_read :: uintptr(63) + SYS_write :: uintptr(64) + SYS_readv :: uintptr(65) + SYS_writev :: uintptr(66) + SYS_pread64 :: uintptr(67) + SYS_pwrite64 :: uintptr(68) + SYS_preadv :: uintptr(69) + SYS_pwritev :: uintptr(70) + SYS_sendfile :: uintptr(71) + SYS_pselect6 :: uintptr(72) + SYS_ppoll :: uintptr(73) + SYS_signalfd4 :: uintptr(74) + SYS_vmsplice :: uintptr(75) + SYS_splice :: uintptr(76) + SYS_tee :: uintptr(77) + SYS_readlinkat :: uintptr(78) + SYS_fstatat :: uintptr(79) + SYS_fstat :: uintptr(80) + SYS_sync :: uintptr(81) + SYS_fsync :: uintptr(82) + SYS_fdatasync :: uintptr(83) + SYS_sync_file_range2 :: uintptr(84) + SYS_sync_file_range :: uintptr(84) + SYS_timerfd_create :: uintptr(85) + SYS_timerfd_settime :: uintptr(86) + SYS_timerfd_gettime :: uintptr(87) + SYS_utimensat :: uintptr(88) + SYS_acct :: uintptr(89) + SYS_capget :: uintptr(90) + SYS_capset :: uintptr(91) + SYS_personality :: uintptr(92) + SYS_exit :: uintptr(93) + SYS_exit_group :: uintptr(94) + SYS_waitid :: uintptr(95) + SYS_set_tid_address :: uintptr(96) + SYS_unshare :: uintptr(97) + SYS_futex :: uintptr(98) + SYS_set_robust_list :: uintptr(99) + SYS_get_robust_list :: uintptr(100) + SYS_nanosleep :: uintptr(101) + SYS_getitimer :: uintptr(102) + SYS_setitimer :: uintptr(103) + SYS_kexec_load :: uintptr(104) + SYS_init_module :: uintptr(105) + SYS_delete_module :: uintptr(106) + SYS_timer_create :: uintptr(107) + SYS_timer_gettime :: uintptr(108) + SYS_timer_getoverrun :: uintptr(109) + SYS_timer_settime :: uintptr(110) + SYS_timer_delete :: uintptr(111) + SYS_clock_settime :: uintptr(112) + SYS_clock_gettime :: uintptr(113) + SYS_clock_getres :: uintptr(114) + SYS_clock_nanosleep :: uintptr(115) + SYS_syslog :: uintptr(116) + SYS_ptrace :: uintptr(117) + SYS_sched_setparam :: uintptr(118) + SYS_sched_setscheduler :: uintptr(119) + SYS_sched_getscheduler :: uintptr(120) + SYS_sched_getparam :: uintptr(121) + SYS_sched_setaffinity :: uintptr(122) + SYS_sched_getaffinity :: uintptr(123) + SYS_sched_yield :: uintptr(124) + SYS_sched_get_priority_max :: uintptr(125) + SYS_sched_get_priority_min :: uintptr(126) + SYS_sched_rr_get_interval :: uintptr(127) + SYS_restart_syscall :: uintptr(128) + SYS_kill :: uintptr(129) + SYS_tkill :: uintptr(130) + SYS_tgkill :: uintptr(131) + SYS_sigaltstack :: uintptr(132) + SYS_rt_sigsuspend :: uintptr(133) + SYS_rt_sigaction :: uintptr(134) + SYS_rt_sigprocmask :: uintptr(135) + SYS_rt_sigpending :: uintptr(136) + SYS_rt_sigtimedwait :: uintptr(137) + SYS_rt_sigqueueinfo :: uintptr(138) + SYS_rt_sigreturn :: uintptr(139) + SYS_setpriority :: uintptr(140) + SYS_getpriority :: uintptr(141) + SYS_reboot :: uintptr(142) + SYS_setregid :: uintptr(143) + SYS_setgid :: uintptr(144) + SYS_setreuid :: uintptr(145) + SYS_setuid :: uintptr(146) + SYS_setresuid :: uintptr(147) + SYS_getresuid :: uintptr(148) + SYS_setresgid :: uintptr(149) + SYS_getresgid :: uintptr(150) + SYS_setfsuid :: uintptr(151) + SYS_setfsgid :: uintptr(152) + SYS_times :: uintptr(153) + SYS_setpgid :: uintptr(154) + SYS_getpgid :: uintptr(155) + SYS_getsid :: uintptr(156) + SYS_setsid :: uintptr(157) + SYS_getgroups :: uintptr(158) + SYS_setgroups :: uintptr(159) + SYS_uname :: uintptr(160) + SYS_sethostname :: uintptr(161) + SYS_setdomainname :: uintptr(162) + SYS_getrlimit :: uintptr(163) + SYS_setrlimit :: uintptr(164) + SYS_getrusage :: uintptr(165) + SYS_umask :: uintptr(166) + SYS_prctl :: uintptr(167) + SYS_getcpu :: uintptr(168) + SYS_gettimeofday :: uintptr(169) + SYS_settimeofday :: uintptr(170) + SYS_adjtimex :: uintptr(171) + SYS_getpid :: uintptr(172) + SYS_getppid :: uintptr(173) + SYS_getuid :: uintptr(174) + SYS_geteuid :: uintptr(175) + SYS_getgid :: uintptr(176) + SYS_getegid :: uintptr(177) + SYS_gettid :: uintptr(178) + SYS_sysinfo :: uintptr(179) + SYS_mq_open :: uintptr(180) + SYS_mq_unlink :: uintptr(181) + SYS_mq_timedsend :: uintptr(182) + SYS_mq_timedreceive :: uintptr(183) + SYS_mq_notify :: uintptr(184) + SYS_mq_getsetattr :: uintptr(185) + SYS_msgget :: uintptr(186) + SYS_msgctl :: uintptr(187) + SYS_msgrcv :: uintptr(188) + SYS_msgsnd :: uintptr(189) + SYS_semget :: uintptr(190) + SYS_semctl :: uintptr(191) + SYS_semtimedop :: uintptr(192) + SYS_semop :: uintptr(193) + SYS_shmget :: uintptr(194) + SYS_shmctl :: uintptr(195) + SYS_shmat :: uintptr(196) + SYS_shmdt :: uintptr(197) + SYS_socket :: uintptr(198) + SYS_socketpair :: uintptr(199) + SYS_bind :: uintptr(200) + SYS_listen :: uintptr(201) + SYS_accept :: uintptr(202) + SYS_connect :: uintptr(203) + SYS_getsockname :: uintptr(204) + SYS_getpeername :: uintptr(205) + SYS_sendto :: uintptr(206) + SYS_recvfrom :: uintptr(207) + SYS_setsockopt :: uintptr(208) + SYS_getsockopt :: uintptr(209) + SYS_shutdown :: uintptr(210) + SYS_sendmsg :: uintptr(211) + SYS_recvmsg :: uintptr(212) + SYS_readahead :: uintptr(213) + SYS_brk :: uintptr(214) + SYS_munmap :: uintptr(215) + SYS_mremap :: uintptr(216) + SYS_add_key :: uintptr(217) + SYS_request_key :: uintptr(218) + SYS_keyctl :: uintptr(219) + SYS_clone :: uintptr(220) + SYS_execve :: uintptr(221) + SYS_mmap :: uintptr(222) + SYS_fadvise64 :: uintptr(223) + SYS_swapon :: uintptr(224) + SYS_swapoff :: uintptr(225) + SYS_mprotect :: uintptr(226) + SYS_msync :: uintptr(227) + SYS_mlock :: uintptr(228) + SYS_munlock :: uintptr(229) + SYS_mlockall :: uintptr(230) + SYS_munlockall :: uintptr(231) + SYS_mincore :: uintptr(232) + SYS_madvise :: uintptr(233) + SYS_remap_file_pages :: uintptr(234) + SYS_mbind :: uintptr(235) + SYS_get_mempolicy :: uintptr(236) + SYS_set_mempolicy :: uintptr(237) + SYS_migrate_pages :: uintptr(238) + SYS_move_pages :: uintptr(239) + SYS_rt_tgsigqueueinfo :: uintptr(240) + SYS_perf_event_open :: uintptr(241) + SYS_accept4 :: uintptr(242) + SYS_recvmmsg :: uintptr(243) + SYS_wait4 :: uintptr(260) + SYS_prlimit64 :: uintptr(261) + SYS_fanotify_init :: uintptr(262) + SYS_fanotify_mark :: uintptr(263) + SYS_name_to_handle_at :: uintptr(264) + SYS_open_by_handle_at :: uintptr(265) + SYS_clock_adjtime :: uintptr(266) + SYS_syncfs :: uintptr(267) + SYS_setns :: uintptr(268) + SYS_sendmmsg :: uintptr(269) + SYS_process_vm_readv :: uintptr(270) + SYS_process_vm_writev :: uintptr(271) + SYS_kcmp :: uintptr(272) + SYS_finit_module :: uintptr(273) + SYS_sched_setattr :: uintptr(274) + SYS_sched_getattr :: uintptr(275) + SYS_renameat2 :: uintptr(276) + SYS_seccomp :: uintptr(277) + SYS_getrandom :: uintptr(278) + SYS_memfd_create :: uintptr(279) + SYS_bpf :: uintptr(280) + SYS_execveat :: uintptr(281) + SYS_userfaultfd :: uintptr(282) + SYS_membarrier :: uintptr(283) + SYS_mlock2 :: uintptr(284) + SYS_copy_file_range :: uintptr(285) + SYS_preadv2 :: uintptr(286) + SYS_pwritev2 :: uintptr(287) + SYS_pkey_mprotect :: uintptr(288) + SYS_pkey_alloc :: uintptr(289) + SYS_pkey_free :: uintptr(290) + SYS_statx :: uintptr(291) + SYS_io_pgetevents :: uintptr(292) + SYS_rseq :: uintptr(293) + SYS_kexec_file_load :: uintptr(294) + SYS_clock_gettime64 :: uintptr(403) + SYS_clock_settime64 :: uintptr(404) + SYS_clock_adjtime64 :: uintptr(405) + SYS_clock_getres_time64 :: uintptr(406) + SYS_clock_nanosleep_time64 :: uintptr(407) + SYS_timer_gettime64 :: uintptr(408) + SYS_timer_settime64 :: uintptr(409) + SYS_timerfd_gettime64 :: uintptr(410) + SYS_timerfd_settime64 :: uintptr(411) + SYS_utimensat_time64 :: uintptr(412) + SYS_pselect6_time64 :: uintptr(413) + SYS_ppoll_time64 :: uintptr(414) + SYS_io_pgetevents_time64 :: uintptr(416) + SYS_recvmmsg_time64 :: uintptr(417) + SYS_mq_timedsend_time64 :: uintptr(418) + SYS_mq_timedreceive_time64 :: uintptr(419) + SYS_semtimedop_time64 :: uintptr(420) + SYS_rt_sigtimedwait_time64 :: uintptr(421) + SYS_futex_time64 :: uintptr(422) + SYS_sched_rr_get_interval_time64 :: uintptr(423) + SYS_pidfd_send_signal :: uintptr(424) + SYS_io_uring_setup :: uintptr(425) + SYS_io_uring_enter :: uintptr(426) + SYS_io_uring_register :: uintptr(427) + SYS_open_tree :: uintptr(428) + SYS_move_mount :: uintptr(429) + SYS_fsopen :: uintptr(430) + SYS_fsconfig :: uintptr(431) + SYS_fsmount :: uintptr(432) + SYS_fspick :: uintptr(433) + SYS_pidfd_open :: uintptr(434) + SYS_clone3 :: uintptr(435) + SYS_close_range :: uintptr(436) + SYS_openat2 :: uintptr(437) + SYS_pidfd_getfd :: uintptr(438) + SYS_faccessat2 :: uintptr(439) + SYS_process_madvise :: uintptr(440) + SYS_epoll_pwait2 :: uintptr(441) + SYS_mount_setattr :: uintptr(442) + SYS_quotactl_fd :: uintptr(443) + SYS_landlock_create_ruleset :: uintptr(444) + SYS_landlock_add_rule :: uintptr(445) + SYS_landlock_restrict_self :: uintptr(446) + SYS_memfd_secret :: uintptr(447) + SYS_process_mrelease :: uintptr(448) + SYS_futex_waitv :: uintptr(449) + SYS_set_mempolicy_home_node :: uintptr(450) + SYS_cachestat :: uintptr(451) + SYS_fchmodat2 :: uintptr(452) + + SIGCHLD :: 17 } else { #panic("Unsupported architecture") } @@ -1742,7 +2074,7 @@ sys_getrandom :: proc "contextless" (buf: [^]byte, buflen: uint, flags: int) -> } sys_open :: proc "contextless" (path: cstring, flags: int, mode: uint = 0o000) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_open, uintptr(rawptr(path)), uintptr(flags), uintptr(mode))) } else { // NOTE: arm64 does not have open return int(intrinsics.syscall(SYS_openat, AT_FDCWD, uintptr(rawptr(path)), uintptr(flags), uintptr(mode))) @@ -1762,7 +2094,7 @@ sys_read :: proc "contextless" (fd: int, buf: rawptr, size: uint) -> int { } sys_pread :: proc "contextless" (fd: int, buf: rawptr, size: uint, offset: i64) -> int { - when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 { + when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { return int(intrinsics.syscall(SYS_pread64, uintptr(fd), uintptr(buf), uintptr(size), uintptr(offset))) } else { low := uintptr(offset & 0xFFFFFFFF) @@ -1776,7 +2108,7 @@ sys_write :: proc "contextless" (fd: int, buf: rawptr, size: uint) -> int { } sys_pwrite :: proc "contextless" (fd: int, buf: rawptr, size: uint, offset: i64) -> int { - when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 { + when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { return int(intrinsics.syscall(SYS_pwrite64, uintptr(fd), uintptr(buf), uintptr(size), uintptr(offset))) } else { low := uintptr(offset & 0xFFFFFFFF) @@ -1786,7 +2118,7 @@ sys_pwrite :: proc "contextless" (fd: int, buf: rawptr, size: uint, offset: i64) } sys_lseek :: proc "contextless" (fd: int, offset: i64, whence: int) -> i64 { - when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 { + when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { return i64(intrinsics.syscall(SYS_lseek, uintptr(fd), uintptr(offset), uintptr(whence))) } else { low := uintptr(offset & 0xFFFFFFFF) @@ -1800,7 +2132,7 @@ sys_lseek :: proc "contextless" (fd: int, offset: i64, whence: int) -> i64 { sys_stat :: proc "contextless" (path: cstring, stat: rawptr) -> int { when ODIN_ARCH == .amd64 { return int(intrinsics.syscall(SYS_stat, uintptr(rawptr(path)), uintptr(stat))) - } else when ODIN_ARCH != .arm64 { + } else when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_stat64, uintptr(rawptr(path)), uintptr(stat))) } else { // NOTE: arm64 does not have stat return int(intrinsics.syscall(SYS_fstatat, AT_FDCWD, uintptr(rawptr(path)), uintptr(stat), 0)) @@ -1808,7 +2140,7 @@ sys_stat :: proc "contextless" (path: cstring, stat: rawptr) -> int { } sys_fstat :: proc "contextless" (fd: int, stat: rawptr) -> int { - when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 { + when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { return int(intrinsics.syscall(SYS_fstat, uintptr(fd), uintptr(stat))) } else { return int(intrinsics.syscall(SYS_fstat64, uintptr(fd), uintptr(stat))) @@ -1818,7 +2150,7 @@ sys_fstat :: proc "contextless" (fd: int, stat: rawptr) -> int { sys_lstat :: proc "contextless" (path: cstring, stat: rawptr) -> int { when ODIN_ARCH == .amd64 { return int(intrinsics.syscall(SYS_lstat, uintptr(rawptr(path)), uintptr(stat))) - } else when ODIN_ARCH != .arm64 { + } else when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_lstat64, uintptr(rawptr(path)), uintptr(stat))) } else { // NOTE: arm64 does not have any lstat return int(intrinsics.syscall(SYS_fstatat, AT_FDCWD, uintptr(rawptr(path)), uintptr(stat), AT_SYMLINK_NOFOLLOW)) @@ -1826,7 +2158,7 @@ sys_lstat :: proc "contextless" (path: cstring, stat: rawptr) -> int { } sys_readlink :: proc "contextless" (path: cstring, buf: rawptr, bufsiz: uint) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_readlink, uintptr(rawptr(path)), uintptr(buf), uintptr(bufsiz))) } else { // NOTE: arm64 does not have readlink return int(intrinsics.syscall(SYS_readlinkat, AT_FDCWD, uintptr(rawptr(path)), uintptr(buf), uintptr(bufsiz))) @@ -1834,7 +2166,7 @@ sys_readlink :: proc "contextless" (path: cstring, buf: rawptr, bufsiz: uint) -> } sys_symlink :: proc "contextless" (old_name: cstring, new_name: cstring) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_symlink, uintptr(rawptr(old_name)), uintptr(rawptr(new_name)))) } else { // NOTE: arm64 does not have symlink return int(intrinsics.syscall(SYS_symlinkat, uintptr(rawptr(old_name)), AT_FDCWD, uintptr(rawptr(new_name)))) @@ -1842,7 +2174,7 @@ sys_symlink :: proc "contextless" (old_name: cstring, new_name: cstring) -> int } sys_access :: proc "contextless" (path: cstring, mask: int) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_access, uintptr(rawptr(path)), uintptr(mask))) } else { // NOTE: arm64 does not have access return int(intrinsics.syscall(SYS_faccessat, AT_FDCWD, uintptr(rawptr(path)), uintptr(mask))) @@ -1862,7 +2194,7 @@ sys_fchdir :: proc "contextless" (fd: int) -> int { } sys_chmod :: proc "contextless" (path: cstring, mode: uint) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_chmod, uintptr(rawptr(path)), uintptr(mode))) } else { // NOTE: arm64 does not have chmod return int(intrinsics.syscall(SYS_fchmodat, AT_FDCWD, uintptr(rawptr(path)), uintptr(mode))) @@ -1874,7 +2206,7 @@ sys_fchmod :: proc "contextless" (fd: int, mode: uint) -> int { } sys_chown :: proc "contextless" (path: cstring, user: int, group: int) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH !=. riscv64 { return int(intrinsics.syscall(SYS_chown, uintptr(rawptr(path)), uintptr(user), uintptr(group))) } else { // NOTE: arm64 does not have chown return int(intrinsics.syscall(SYS_fchownat, AT_FDCWD, uintptr(rawptr(path)), uintptr(user), uintptr(group), 0)) @@ -1886,7 +2218,7 @@ sys_fchown :: proc "contextless" (fd: int, user: int, group: int) -> int { } sys_lchown :: proc "contextless" (path: cstring, user: int, group: int) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_lchown, uintptr(rawptr(path)), uintptr(user), uintptr(group))) } else { // NOTE: arm64 does not have lchown return int(intrinsics.syscall(SYS_fchownat, AT_FDCWD, uintptr(rawptr(path)), uintptr(user), uintptr(group), AT_SYMLINK_NOFOLLOW)) @@ -1894,7 +2226,7 @@ sys_lchown :: proc "contextless" (path: cstring, user: int, group: int) -> int { } sys_rename :: proc "contextless" (old, new: cstring) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_rename, uintptr(rawptr(old)), uintptr(rawptr(new)))) } else { // NOTE: arm64 does not have rename return int(intrinsics.syscall(SYS_renameat, AT_FDCWD, uintptr(rawptr(old)), uintptr(rawptr(new)))) @@ -1902,7 +2234,7 @@ sys_rename :: proc "contextless" (old, new: cstring) -> int { } sys_link :: proc "contextless" (old_name: cstring, new_name: cstring) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_link, uintptr(rawptr(old_name)), uintptr(rawptr(new_name)))) } else { // NOTE: arm64 does not have link return int(intrinsics.syscall(SYS_linkat, AT_FDCWD, uintptr(rawptr(old_name)), AT_FDCWD, uintptr(rawptr(new_name)), AT_SYMLINK_FOLLOW)) @@ -1910,7 +2242,7 @@ sys_link :: proc "contextless" (old_name: cstring, new_name: cstring) -> int { } sys_unlink :: proc "contextless" (path: cstring) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_unlink, uintptr(rawptr(path)))) } else { // NOTE: arm64 does not have unlink return int(intrinsics.syscall(SYS_unlinkat, AT_FDCWD, uintptr(rawptr(path)), 0)) @@ -1922,7 +2254,7 @@ sys_unlinkat :: proc "contextless" (dfd: int, path: cstring, flag: int = 0) -> i } sys_rmdir :: proc "contextless" (path: cstring) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_rmdir, uintptr(rawptr(path)))) } else { // NOTE: arm64 does not have rmdir return int(intrinsics.syscall(SYS_unlinkat, AT_FDCWD, uintptr(rawptr(path)), AT_REMOVEDIR)) @@ -1930,7 +2262,7 @@ sys_rmdir :: proc "contextless" (path: cstring) -> int { } sys_mkdir :: proc "contextless" (path: cstring, mode: uint) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_mkdir, uintptr(rawptr(path)), uintptr(mode))) } else { // NOTE: arm64 does not have mkdir return int(intrinsics.syscall(SYS_mkdirat, AT_FDCWD, uintptr(rawptr(path)), uintptr(mode))) @@ -1942,7 +2274,7 @@ sys_mkdirat :: proc "contextless" (dfd: int, path: cstring, mode: uint) -> int { } sys_mknod :: proc "contextless" (path: cstring, mode: uint, dev: int) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_mknod, uintptr(rawptr(path)), uintptr(mode), uintptr(dev))) } else { // NOTE: arm64 does not have mknod return int(intrinsics.syscall(SYS_mknodat, AT_FDCWD, uintptr(rawptr(path)), uintptr(mode), uintptr(dev))) @@ -1954,7 +2286,7 @@ sys_mknodat :: proc "contextless" (dfd: int, path: cstring, mode: uint, dev: int } sys_truncate :: proc "contextless" (path: cstring, length: i64) -> int { - when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 { + when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { return int(intrinsics.syscall(SYS_truncate, uintptr(rawptr(path)), uintptr(length))) } else { low := uintptr(length & 0xFFFFFFFF) @@ -1964,7 +2296,7 @@ sys_truncate :: proc "contextless" (path: cstring, length: i64) -> int { } sys_ftruncate :: proc "contextless" (fd: int, length: i64) -> int { - when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 { + when ODIN_ARCH == .amd64 || ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { return int(intrinsics.syscall(SYS_ftruncate, uintptr(fd), uintptr(length))) } else { low := uintptr(length & 0xFFFFFFFF) @@ -1982,7 +2314,7 @@ sys_getdents64 :: proc "contextless" (fd: int, dirent: rawptr, count: int) -> in } sys_fork :: proc "contextless" () -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_fork)) } else { return int(intrinsics.syscall(SYS_clone, SIGCHLD)) @@ -1992,7 +2324,7 @@ sys_pipe2 :: proc "contextless" (fds: rawptr, flags: int) -> int { return int(intrinsics.syscall(SYS_pipe2, uintptr(fds), uintptr(flags))) } sys_dup2 :: proc "contextless" (oldfd: int, newfd: int) -> int { - when ODIN_ARCH != .arm64 { + when ODIN_ARCH != .arm64 && ODIN_ARCH != .riscv64 { return int(intrinsics.syscall(SYS_dup2, uintptr(oldfd), uintptr(newfd))) } else { return int(intrinsics.syscall(SYS_dup3, uintptr(oldfd), uintptr(newfd), 0)) @@ -2076,7 +2408,7 @@ sys_fcntl :: proc "contextless" (fd: int, cmd: int, arg: int) -> int { sys_poll :: proc "contextless" (fds: rawptr, nfds: uint, timeout: int) -> int { // NOTE: specialcased here because `arm64` does not have `poll` - when ODIN_ARCH == .arm64 { + when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { seconds := i64(timeout / 1_000) nanoseconds := i64((timeout % 1000) * 1_000_000) timeout_spec := timespec{seconds, nanoseconds} diff --git a/misc/featuregen/README.md b/misc/featuregen/README.md index 22a798cca..82d95a2b6 100644 --- a/misc/featuregen/README.md +++ b/misc/featuregen/README.md @@ -5,7 +5,7 @@ for features regarding microarchitecture and target features of the compiler. It is not pretty! But LLVM has no way to query this information with their C API. -It generates these globals (intended for `src/build_settings.cpp`: +It generates these globals (intended for `src/build_settings_microarch.cpp`: - `target_microarch_list`: an array of strings indexed by the architecture, each string is a comma-seperated list of microarchitectures available on that architecture - `target_features_list`: an array of strings indexed by the architecture, each string is a comma-seperated list of target features available on that architecture @@ -23,6 +23,6 @@ does not impact much at all, the only thing it will do is make LLVM print a mess ## Usage 1. Make sure the table of architectures at the top of the python script is up-to-date (the triple can be any valid triple for the architecture) -1. `./build.sh` +1. `./build_featuregen.sh` 1. `python3 featuregen.py` 1. Copy the output into `src/build_settings.cpp` diff --git a/misc/featuregen/build_featuregen.sh b/misc/featuregen/build_featuregen.sh new file mode 100755 index 000000000..d68f29925 --- /dev/null +++ b/misc/featuregen/build_featuregen.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -ex + +$(llvm-config --bindir)/clang++ $(llvm-config --cxxflags --ldflags --libs) featuregen.cpp -o featuregen diff --git a/misc/featuregen/featuregen.py b/misc/featuregen/featuregen.py index da4cc68f5..ecc47f70c 100644 --- a/misc/featuregen/featuregen.py +++ b/misc/featuregen/featuregen.py @@ -4,12 +4,13 @@ import os import sys archs = [ - ("amd64", "linux_amd64", "x86_64-pc-linux-gnu", [], []), - ("i386", "linux_i386", "i386-pc-linux-gnu", [], []), - ("arm32", "linux_arm32", "arm-linux-gnu", [], []), - ("arm64", "linux_arm64", "aarch64-linux-elf", [], []), - ("wasm32", "js_wasm32", "wasm32-js-js", [], []), - ("wasm64p32", "js_wasm64p32","wasm32-js-js", [], []), + ("amd64", "linux_amd64", "x86_64-pc-linux-gnu", [], []), + ("i386", "linux_i386", "i386-pc-linux-gnu", [], []), + ("arm32", "linux_arm32", "arm-linux-gnu", [], []), + ("arm64", "linux_arm64", "aarch64-linux-elf", [], []), + ("wasm32", "js_wasm32", "wasm32-js-js", [], []), + ("wasm64p32", "js_wasm64p32", "wasm32-js-js", [], []), + ("riscv64", "linux_riscv64", "riscv64-linux-gnu", [], []), ]; SEEKING_CPUS = 0 @@ -78,7 +79,8 @@ print("\t// TargetArch_Invalid:") print('\tstr_lit(""),') for arch, target, triple, cpus, features in archs: print(f"\t// TargetArch_{arch}:") - print(f'\tstr_lit("{','.join(cpus)}"),') + cpus_str = ','.join(cpus) + print(f'\tstr_lit("{cpus_str}"),') print("};") print("") @@ -89,7 +91,8 @@ print("\t// TargetArch_Invalid:") print('\tstr_lit(""),') for arch, target, triple, cpus, features in archs: print(f"\t// TargetArch_{arch}:") - print(f'\tstr_lit("{','.join(features)}"),') + features_str = ','.join(features) + print(f'\tstr_lit("{features_str}"),') print("};") print("") diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 82523d736..3d56f4202 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -39,6 +39,7 @@ enum TargetArchKind : u16 { TargetArch_arm64, TargetArch_wasm32, TargetArch_wasm64p32, + TargetArch_riscv64, TargetArch_COUNT, }; @@ -104,6 +105,7 @@ gb_global String target_arch_names[TargetArch_COUNT] = { str_lit("arm64"), str_lit("wasm32"), str_lit("wasm64p32"), + str_lit("riscv64"), }; #include "build_settings_microarch.cpp" @@ -555,13 +557,18 @@ gb_global TargetMetrics target_linux_arm64 = { 8, 8, 16, 32, str_lit("aarch64-linux-elf"), }; - gb_global TargetMetrics target_linux_arm32 = { TargetOs_linux, TargetArch_arm32, 4, 4, 8, 16, str_lit("arm-unknown-linux-gnueabihf"), }; +gb_global TargetMetrics target_linux_riscv64 = { + TargetOs_linux, + TargetArch_riscv64, + 8, 8, 16, 32, + str_lit("riscv64-linux-gnu"), +}; gb_global TargetMetrics target_darwin_amd64 = { TargetOs_darwin, @@ -716,6 +723,12 @@ gb_global TargetMetrics target_freestanding_arm32 = { 4, 4, 8, 16, str_lit("arm-unknown-unknown-gnueabihf"), }; +gb_global TargetMetrics target_freestanding_riscv64 = { + TargetOs_freestanding, + TargetArch_riscv64, + 8, 8, 16, 32, + str_lit("riscv64-unknown-gnu"), +}; struct NamedTargetMetrics { @@ -733,6 +746,7 @@ gb_global NamedTargetMetrics named_targets[] = { { str_lit("linux_amd64"), &target_linux_amd64 }, { str_lit("linux_arm64"), &target_linux_arm64 }, { str_lit("linux_arm32"), &target_linux_arm32 }, + { str_lit("linux_riscv64"), &target_linux_riscv64 }, { str_lit("windows_i386"), &target_windows_i386 }, { str_lit("windows_amd64"), &target_windows_amd64 }, @@ -761,6 +775,8 @@ gb_global NamedTargetMetrics named_targets[] = { { str_lit("freestanding_arm64"), &target_freestanding_arm64 }, { str_lit("freestanding_arm32"), &target_freestanding_arm32 }, + + { str_lit("freestanding_riscv64"), &target_freestanding_riscv64 }, }; gb_global NamedTargetMetrics *selected_target_metrics; @@ -1631,6 +1647,8 @@ gb_internal void init_build_context(TargetMetrics *cross_target, Subtarget subta // Disallow on wasm bc->use_separate_modules = false; + } if(bc->metrics.arch == TargetArch_riscv64) { + bc->link_flags = str_lit("-target riscv64 "); } else { // NOTE: for targets other than darwin, we don't specify a `-target` link flag. // This is because we don't support cross-linking and clang is better at figuring diff --git a/src/build_settings_microarch.cpp b/src/build_settings_microarch.cpp index 02b507031..8f64d4026 100644 --- a/src/build_settings_microarch.cpp +++ b/src/build_settings_microarch.cpp @@ -3,17 +3,19 @@ gb_global String target_microarch_list[TargetArch_COUNT] = { // TargetArch_Invalid: str_lit(""), // TargetArch_amd64: - str_lit("alderlake,amdfam10,athlon,athlon-4,athlon-fx,athlon-mp,athlon-tbird,athlon-xp,athlon64,athlon64-sse3,atom,atom_sse4_2,atom_sse4_2_movbe,barcelona,bdver1,bdver2,bdver3,bdver4,bonnell,broadwell,btver1,btver2,c3,c3-2,cannonlake,cascadelake,cooperlake,core-avx-i,core-avx2,core2,core_2_duo_sse4_1,core_2_duo_ssse3,core_2nd_gen_avx,core_3rd_gen_avx,core_4th_gen_avx,core_4th_gen_avx_tsx,core_5th_gen_avx,core_5th_gen_avx_tsx,core_aes_pclmulqdq,core_i7_sse4_2,corei7,corei7-avx,emeraldrapids,generic,geode,goldmont,goldmont-plus,goldmont_plus,grandridge,graniterapids,graniterapids-d,graniterapids_d,haswell,i386,i486,i586,i686,icelake-client,icelake-server,icelake_client,icelake_server,ivybridge,k6,k6-2,k6-3,k8,k8-sse3,knl,knm,lakemont,meteorlake,mic_avx512,nehalem,nocona,opteron,opteron-sse3,penryn,pentium,pentium-m,pentium-mmx,pentium2,pentium3,pentium3m,pentium4,pentium4m,pentium_4,pentium_4_sse3,pentium_ii,pentium_iii,pentium_iii_no_xmm_regs,pentium_m,pentium_mmx,pentium_pro,pentiumpro,prescott,raptorlake,rocketlake,sandybridge,sapphirerapids,sierraforest,silvermont,skx,skylake,skylake-avx512,skylake_avx512,slm,tigerlake,tremont,westmere,winchip-c6,winchip2,x86-64,x86-64-v2,x86-64-v3,x86-64-v4,yonah,znver1,znver2,znver3,znver4"), + str_lit("alderlake,amdfam10,arrowlake,arrowlake-s,arrowlake_s,athlon,athlon-4,athlon-fx,athlon-mp,athlon-tbird,athlon-xp,athlon64,athlon64-sse3,atom,atom_sse4_2,atom_sse4_2_movbe,barcelona,bdver1,bdver2,bdver3,bdver4,bonnell,broadwell,btver1,btver2,c3,c3-2,cannonlake,cascadelake,clearwaterforest,cooperlake,core-avx-i,core-avx2,core2,core_2_duo_sse4_1,core_2_duo_ssse3,core_2nd_gen_avx,core_3rd_gen_avx,core_4th_gen_avx,core_4th_gen_avx_tsx,core_5th_gen_avx,core_5th_gen_avx_tsx,core_aes_pclmulqdq,core_i7_sse4_2,corei7,corei7-avx,emeraldrapids,generic,geode,goldmont,goldmont-plus,goldmont_plus,gracemont,grandridge,graniterapids,graniterapids-d,graniterapids_d,haswell,i386,i486,i586,i686,icelake-client,icelake-server,icelake_client,icelake_server,ivybridge,k6,k6-2,k6-3,k8,k8-sse3,knl,knm,lakemont,lunarlake,meteorlake,mic_avx512,nehalem,nocona,opteron,opteron-sse3,pantherlake,penryn,pentium,pentium-m,pentium-mmx,pentium2,pentium3,pentium3m,pentium4,pentium4m,pentium_4,pentium_4_sse3,pentium_ii,pentium_iii,pentium_iii_no_xmm_regs,pentium_m,pentium_mmx,pentium_pro,pentiumpro,prescott,raptorlake,rocketlake,sandybridge,sapphirerapids,sierraforest,silvermont,skx,skylake,skylake-avx512,skylake_avx512,slm,tigerlake,tremont,westmere,winchip-c6,winchip2,x86-64,x86-64-v2,x86-64-v3,x86-64-v4,yonah,znver1,znver2,znver3,znver4"), // TargetArch_i386: - str_lit("alderlake,amdfam10,athlon,athlon-4,athlon-fx,athlon-mp,athlon-tbird,athlon-xp,athlon64,athlon64-sse3,atom,atom_sse4_2,atom_sse4_2_movbe,barcelona,bdver1,bdver2,bdver3,bdver4,bonnell,broadwell,btver1,btver2,c3,c3-2,cannonlake,cascadelake,cooperlake,core-avx-i,core-avx2,core2,core_2_duo_sse4_1,core_2_duo_ssse3,core_2nd_gen_avx,core_3rd_gen_avx,core_4th_gen_avx,core_4th_gen_avx_tsx,core_5th_gen_avx,core_5th_gen_avx_tsx,core_aes_pclmulqdq,core_i7_sse4_2,corei7,corei7-avx,emeraldrapids,generic,geode,goldmont,goldmont-plus,goldmont_plus,grandridge,graniterapids,graniterapids-d,graniterapids_d,haswell,i386,i486,i586,i686,icelake-client,icelake-server,icelake_client,icelake_server,ivybridge,k6,k6-2,k6-3,k8,k8-sse3,knl,knm,lakemont,meteorlake,mic_avx512,nehalem,nocona,opteron,opteron-sse3,penryn,pentium,pentium-m,pentium-mmx,pentium2,pentium3,pentium3m,pentium4,pentium4m,pentium_4,pentium_4_sse3,pentium_ii,pentium_iii,pentium_iii_no_xmm_regs,pentium_m,pentium_mmx,pentium_pro,pentiumpro,prescott,raptorlake,rocketlake,sandybridge,sapphirerapids,sierraforest,silvermont,skx,skylake,skylake-avx512,skylake_avx512,slm,tigerlake,tremont,westmere,winchip-c6,winchip2,x86-64,x86-64-v2,x86-64-v3,x86-64-v4,yonah,znver1,znver2,znver3,znver4"), + str_lit("alderlake,amdfam10,arrowlake,arrowlake-s,arrowlake_s,athlon,athlon-4,athlon-fx,athlon-mp,athlon-tbird,athlon-xp,athlon64,athlon64-sse3,atom,atom_sse4_2,atom_sse4_2_movbe,barcelona,bdver1,bdver2,bdver3,bdver4,bonnell,broadwell,btver1,btver2,c3,c3-2,cannonlake,cascadelake,clearwaterforest,cooperlake,core-avx-i,core-avx2,core2,core_2_duo_sse4_1,core_2_duo_ssse3,core_2nd_gen_avx,core_3rd_gen_avx,core_4th_gen_avx,core_4th_gen_avx_tsx,core_5th_gen_avx,core_5th_gen_avx_tsx,core_aes_pclmulqdq,core_i7_sse4_2,corei7,corei7-avx,emeraldrapids,generic,geode,goldmont,goldmont-plus,goldmont_plus,gracemont,grandridge,graniterapids,graniterapids-d,graniterapids_d,haswell,i386,i486,i586,i686,icelake-client,icelake-server,icelake_client,icelake_server,ivybridge,k6,k6-2,k6-3,k8,k8-sse3,knl,knm,lakemont,lunarlake,meteorlake,mic_avx512,nehalem,nocona,opteron,opteron-sse3,pantherlake,penryn,pentium,pentium-m,pentium-mmx,pentium2,pentium3,pentium3m,pentium4,pentium4m,pentium_4,pentium_4_sse3,pentium_ii,pentium_iii,pentium_iii_no_xmm_regs,pentium_m,pentium_mmx,pentium_pro,pentiumpro,prescott,raptorlake,rocketlake,sandybridge,sapphirerapids,sierraforest,silvermont,skx,skylake,skylake-avx512,skylake_avx512,slm,tigerlake,tremont,westmere,winchip-c6,winchip2,x86-64,x86-64-v2,x86-64-v3,x86-64-v4,yonah,znver1,znver2,znver3,znver4"), // TargetArch_arm32: - str_lit("arm1020e,arm1020t,arm1022e,arm10e,arm10tdmi,arm1136j-s,arm1136jf-s,arm1156t2-s,arm1156t2f-s,arm1176jz-s,arm1176jzf-s,arm710t,arm720t,arm7tdmi,arm7tdmi-s,arm8,arm810,arm9,arm920,arm920t,arm922t,arm926ej-s,arm940t,arm946e-s,arm966e-s,arm968e-s,arm9e,arm9tdmi,cortex-a12,cortex-a15,cortex-a17,cortex-a32,cortex-a35,cortex-a5,cortex-a53,cortex-a55,cortex-a57,cortex-a7,cortex-a710,cortex-a72,cortex-a73,cortex-a75,cortex-a76,cortex-a76ae,cortex-a77,cortex-a78,cortex-a78c,cortex-a8,cortex-a9,cortex-m0,cortex-m0plus,cortex-m1,cortex-m23,cortex-m3,cortex-m33,cortex-m35p,cortex-m4,cortex-m55,cortex-m7,cortex-m85,cortex-r4,cortex-r4f,cortex-r5,cortex-r52,cortex-r7,cortex-r8,cortex-x1,cortex-x1c,cyclone,ep9312,exynos-m3,exynos-m4,exynos-m5,generic,iwmmxt,krait,kryo,mpcore,mpcorenovfp,neoverse-n1,neoverse-n2,neoverse-v1,sc000,sc300,strongarm,strongarm110,strongarm1100,strongarm1110,swift,xscale"), + str_lit("arm1020e,arm1020t,arm1022e,arm10e,arm10tdmi,arm1136j-s,arm1136jf-s,arm1156t2-s,arm1156t2f-s,arm1176jz-s,arm1176jzf-s,arm710t,arm720t,arm7tdmi,arm7tdmi-s,arm8,arm810,arm9,arm920,arm920t,arm922t,arm926ej-s,arm940t,arm946e-s,arm966e-s,arm968e-s,arm9e,arm9tdmi,cortex-a12,cortex-a15,cortex-a17,cortex-a32,cortex-a35,cortex-a5,cortex-a53,cortex-a55,cortex-a57,cortex-a7,cortex-a710,cortex-a72,cortex-a73,cortex-a75,cortex-a76,cortex-a76ae,cortex-a77,cortex-a78,cortex-a78c,cortex-a8,cortex-a9,cortex-m0,cortex-m0plus,cortex-m1,cortex-m23,cortex-m3,cortex-m33,cortex-m35p,cortex-m4,cortex-m52,cortex-m55,cortex-m7,cortex-m85,cortex-r4,cortex-r4f,cortex-r5,cortex-r52,cortex-r7,cortex-r8,cortex-x1,cortex-x1c,cyclone,ep9312,exynos-m3,exynos-m4,exynos-m5,generic,iwmmxt,krait,kryo,mpcore,mpcorenovfp,neoverse-n1,neoverse-n2,neoverse-v1,sc000,sc300,strongarm,strongarm110,strongarm1100,strongarm1110,swift,xscale"), // TargetArch_arm64: - str_lit("a64fx,ampere1,ampere1a,apple-a10,apple-a11,apple-a12,apple-a13,apple-a14,apple-a15,apple-a16,apple-a7,apple-a8,apple-a9,apple-latest,apple-m1,apple-m2,apple-s4,apple-s5,carmel,cortex-a34,cortex-a35,cortex-a510,cortex-a53,cortex-a55,cortex-a57,cortex-a65,cortex-a65ae,cortex-a710,cortex-a715,cortex-a72,cortex-a73,cortex-a75,cortex-a76,cortex-a76ae,cortex-a77,cortex-a78,cortex-a78c,cortex-r82,cortex-x1,cortex-x1c,cortex-x2,cortex-x3,cyclone,exynos-m3,exynos-m4,exynos-m5,falkor,generic,kryo,neoverse-512tvb,neoverse-e1,neoverse-n1,neoverse-n2,neoverse-v1,neoverse-v2,saphira,thunderx,thunderx2t99,thunderx3t110,thunderxt81,thunderxt83,thunderxt88,tsv110"), + str_lit("a64fx,ampere1,ampere1a,ampere1b,apple-a10,apple-a11,apple-a12,apple-a13,apple-a14,apple-a15,apple-a16,apple-a17,apple-a7,apple-a8,apple-a9,apple-latest,apple-m1,apple-m2,apple-m3,apple-s4,apple-s5,carmel,cortex-a34,cortex-a35,cortex-a510,cortex-a520,cortex-a53,cortex-a55,cortex-a57,cortex-a65,cortex-a65ae,cortex-a710,cortex-a715,cortex-a72,cortex-a720,cortex-a73,cortex-a75,cortex-a76,cortex-a76ae,cortex-a77,cortex-a78,cortex-a78c,cortex-r82,cortex-x1,cortex-x1c,cortex-x2,cortex-x3,cortex-x4,cyclone,exynos-m3,exynos-m4,exynos-m5,falkor,generic,kryo,neoverse-512tvb,neoverse-e1,neoverse-n1,neoverse-n2,neoverse-v1,neoverse-v2,saphira,thunderx,thunderx2t99,thunderx3t110,thunderxt81,thunderxt83,thunderxt88,tsv110"), // TargetArch_wasm32: str_lit("bleeding-edge,generic,mvp"), // TargetArch_wasm64p32: str_lit("bleeding-edge,generic,mvp"), + // TargetArch_riscv64: + str_lit("generic,generic-rv32,generic-rv64,rocket,rocket-rv32,rocket-rv64,sifive-7-series,sifive-e20,sifive-e21,sifive-e24,sifive-e31,sifive-e34,sifive-e76,sifive-p450,sifive-p670,sifive-s21,sifive-s51,sifive-s54,sifive-s76,sifive-u54,sifive-u74,sifive-x280,syntacore-scr1-base,syntacore-scr1-max,veyron-v1,xiangshan-nanhu"), }; // Generated with the featuregen script in `misc/featuregen` @@ -21,17 +23,19 @@ gb_global String target_features_list[TargetArch_COUNT] = { // TargetArch_Invalid: str_lit(""), // TargetArch_amd64: - str_lit("16bit-mode,32bit-mode,3dnow,3dnowa,64bit,64bit-mode,adx,aes,allow-light-256-bit,amx-bf16,amx-complex,amx-fp16,amx-int8,amx-tile,avx,avx2,avx512bf16,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512er,avx512f,avx512fp16,avx512ifma,avx512pf,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vp2intersect,avx512vpopcntdq,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,branchfusion,cldemote,clflushopt,clwb,clzero,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,ermsb,f16c,false-deps-getmant,false-deps-lzcnt-tzcnt,false-deps-mulc,false-deps-mullq,false-deps-perm,false-deps-popcnt,false-deps-range,fast-11bytenop,fast-15bytenop,fast-7bytenop,fast-bextr,fast-gather,fast-hops,fast-lzcnt,fast-movbe,fast-scalar-fsqrt,fast-scalar-shift-masks,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fast-vector-shift-masks,faster-shift-than-shuffle,fma,fma4,fsgsbase,fsrm,fxsr,gfni,harden-sls-ijmp,harden-sls-ret,hreset,idivl-to-divb,idivq-to-divl,invpcid,kl,lea-sp,lea-uses-ag,lvi-cfi,lvi-load-hardening,lwp,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,mwaitx,no-bypass-delay,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pad-short-functions,pclmul,pconfig,pku,popcnt,prefer-128-bit,prefer-256-bit,prefer-mask-registers,prefer-movmsk-over-vtest,prefer-no-gather,prefer-no-scatter,prefetchi,prefetchwt1,prfchw,ptwrite,raoint,rdpid,rdpru,rdrnd,rdseed,retpoline,retpoline-external-thunk,retpoline-indirect-branches,retpoline-indirect-calls,rtm,sahf,sbb-dep-breaking,serialize,seses,sgx,sha,sha512,shstk,slow-3ops-lea,slow-incdec,slow-lea,slow-pmaddwd,slow-pmulld,slow-shld,slow-two-mem-ops,slow-unaligned-mem-16,slow-unaligned-mem-32,sm3,sm4,soft-float,sse,sse-unaligned-mem,sse2,sse3,sse4.1,sse4.2,sse4a,ssse3,tagged-globals,tbm,tsxldtrk,tuning-fast-imm-vector-shift,uintr,use-glm-div-sqrt-costs,use-slm-arith-costs,vaes,vpclmulqdq,vzeroupper,waitpkg,wbnoinvd,widekl,x87,xop,xsave,xsavec,xsaveopt,xsaves"), + str_lit("16bit-mode,32bit-mode,3dnow,3dnowa,64bit,64bit-mode,adx,aes,allow-light-256-bit,amx-bf16,amx-complex,amx-fp16,amx-int8,amx-tile,avx,avx10.1-256,avx10.1-512,avx2,avx512bf16,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512er,avx512f,avx512fp16,avx512ifma,avx512pf,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vp2intersect,avx512vpopcntdq,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,branchfusion,ccmp,cf,cldemote,clflushopt,clwb,clzero,cmov,cmpccxadd,crc32,cx16,cx8,egpr,enqcmd,ermsb,evex512,f16c,false-deps-getmant,false-deps-lzcnt-tzcnt,false-deps-mulc,false-deps-mullq,false-deps-perm,false-deps-popcnt,false-deps-range,fast-11bytenop,fast-15bytenop,fast-7bytenop,fast-bextr,fast-gather,fast-hops,fast-lzcnt,fast-movbe,fast-scalar-fsqrt,fast-scalar-shift-masks,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fast-vector-shift-masks,faster-shift-than-shuffle,fma,fma4,fsgsbase,fsrm,fxsr,gfni,harden-sls-ijmp,harden-sls-ret,hreset,idivl-to-divb,idivq-to-divl,invpcid,kl,lea-sp,lea-uses-ag,lvi-cfi,lvi-load-hardening,lwp,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,mwaitx,ndd,no-bypass-delay,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pad-short-functions,pclmul,pconfig,pku,popcnt,ppx,prefer-128-bit,prefer-256-bit,prefer-mask-registers,prefer-movmsk-over-vtest,prefer-no-gather,prefer-no-scatter,prefetchi,prefetchwt1,prfchw,ptwrite,push2pop2,raoint,rdpid,rdpru,rdrnd,rdseed,retpoline,retpoline-external-thunk,retpoline-indirect-branches,retpoline-indirect-calls,rtm,sahf,sbb-dep-breaking,serialize,seses,sgx,sha,sha512,shstk,slow-3ops-lea,slow-incdec,slow-lea,slow-pmaddwd,slow-pmulld,slow-shld,slow-two-mem-ops,slow-unaligned-mem-16,slow-unaligned-mem-32,sm3,sm4,soft-float,sse,sse-unaligned-mem,sse2,sse3,sse4.1,sse4.2,sse4a,ssse3,tagged-globals,tbm,tsxldtrk,tuning-fast-imm-vector-shift,uintr,use-glm-div-sqrt-costs,use-slm-arith-costs,usermsr,vaes,vpclmulqdq,vzeroupper,waitpkg,wbnoinvd,widekl,x87,xop,xsave,xsavec,xsaveopt,xsaves"), // TargetArch_i386: - str_lit("16bit-mode,32bit-mode,3dnow,3dnowa,64bit,64bit-mode,adx,aes,allow-light-256-bit,amx-bf16,amx-complex,amx-fp16,amx-int8,amx-tile,avx,avx2,avx512bf16,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512er,avx512f,avx512fp16,avx512ifma,avx512pf,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vp2intersect,avx512vpopcntdq,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,branchfusion,cldemote,clflushopt,clwb,clzero,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,ermsb,f16c,false-deps-getmant,false-deps-lzcnt-tzcnt,false-deps-mulc,false-deps-mullq,false-deps-perm,false-deps-popcnt,false-deps-range,fast-11bytenop,fast-15bytenop,fast-7bytenop,fast-bextr,fast-gather,fast-hops,fast-lzcnt,fast-movbe,fast-scalar-fsqrt,fast-scalar-shift-masks,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fast-vector-shift-masks,faster-shift-than-shuffle,fma,fma4,fsgsbase,fsrm,fxsr,gfni,harden-sls-ijmp,harden-sls-ret,hreset,idivl-to-divb,idivq-to-divl,invpcid,kl,lea-sp,lea-uses-ag,lvi-cfi,lvi-load-hardening,lwp,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,mwaitx,no-bypass-delay,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pad-short-functions,pclmul,pconfig,pku,popcnt,prefer-128-bit,prefer-256-bit,prefer-mask-registers,prefer-movmsk-over-vtest,prefer-no-gather,prefer-no-scatter,prefetchi,prefetchwt1,prfchw,ptwrite,raoint,rdpid,rdpru,rdrnd,rdseed,retpoline,retpoline-external-thunk,retpoline-indirect-branches,retpoline-indirect-calls,rtm,sahf,sbb-dep-breaking,serialize,seses,sgx,sha,sha512,shstk,slow-3ops-lea,slow-incdec,slow-lea,slow-pmaddwd,slow-pmulld,slow-shld,slow-two-mem-ops,slow-unaligned-mem-16,slow-unaligned-mem-32,sm3,sm4,soft-float,sse,sse-unaligned-mem,sse2,sse3,sse4.1,sse4.2,sse4a,ssse3,tagged-globals,tbm,tsxldtrk,tuning-fast-imm-vector-shift,uintr,use-glm-div-sqrt-costs,use-slm-arith-costs,vaes,vpclmulqdq,vzeroupper,waitpkg,wbnoinvd,widekl,x87,xop,xsave,xsavec,xsaveopt,xsaves"), + str_lit("16bit-mode,32bit-mode,3dnow,3dnowa,64bit,64bit-mode,adx,aes,allow-light-256-bit,amx-bf16,amx-complex,amx-fp16,amx-int8,amx-tile,avx,avx10.1-256,avx10.1-512,avx2,avx512bf16,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512er,avx512f,avx512fp16,avx512ifma,avx512pf,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vp2intersect,avx512vpopcntdq,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,branchfusion,ccmp,cf,cldemote,clflushopt,clwb,clzero,cmov,cmpccxadd,crc32,cx16,cx8,egpr,enqcmd,ermsb,evex512,f16c,false-deps-getmant,false-deps-lzcnt-tzcnt,false-deps-mulc,false-deps-mullq,false-deps-perm,false-deps-popcnt,false-deps-range,fast-11bytenop,fast-15bytenop,fast-7bytenop,fast-bextr,fast-gather,fast-hops,fast-lzcnt,fast-movbe,fast-scalar-fsqrt,fast-scalar-shift-masks,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fast-vector-shift-masks,faster-shift-than-shuffle,fma,fma4,fsgsbase,fsrm,fxsr,gfni,harden-sls-ijmp,harden-sls-ret,hreset,idivl-to-divb,idivq-to-divl,invpcid,kl,lea-sp,lea-uses-ag,lvi-cfi,lvi-load-hardening,lwp,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,mwaitx,ndd,no-bypass-delay,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pad-short-functions,pclmul,pconfig,pku,popcnt,ppx,prefer-128-bit,prefer-256-bit,prefer-mask-registers,prefer-movmsk-over-vtest,prefer-no-gather,prefer-no-scatter,prefetchi,prefetchwt1,prfchw,ptwrite,push2pop2,raoint,rdpid,rdpru,rdrnd,rdseed,retpoline,retpoline-external-thunk,retpoline-indirect-branches,retpoline-indirect-calls,rtm,sahf,sbb-dep-breaking,serialize,seses,sgx,sha,sha512,shstk,slow-3ops-lea,slow-incdec,slow-lea,slow-pmaddwd,slow-pmulld,slow-shld,slow-two-mem-ops,slow-unaligned-mem-16,slow-unaligned-mem-32,sm3,sm4,soft-float,sse,sse-unaligned-mem,sse2,sse3,sse4.1,sse4.2,sse4a,ssse3,tagged-globals,tbm,tsxldtrk,tuning-fast-imm-vector-shift,uintr,use-glm-div-sqrt-costs,use-slm-arith-costs,usermsr,vaes,vpclmulqdq,vzeroupper,waitpkg,wbnoinvd,widekl,x87,xop,xsave,xsavec,xsaveopt,xsaves"), // TargetArch_arm32: - str_lit("32bit,8msecext,a12,a15,a17,a32,a35,a5,a53,a55,a57,a7,a72,a73,a75,a76,a77,a78c,a8,a9,aapcs-frame-chain,aapcs-frame-chain-leaf,aclass,acquire-release,aes,armv4,armv4t,armv5t,armv5te,armv5tej,armv6,armv6-m,armv6j,armv6k,armv6kz,armv6s-m,armv6t2,armv7-a,armv7-m,armv7-r,armv7e-m,armv7k,armv7s,armv7ve,armv8-a,armv8-m.base,armv8-m.main,armv8-r,armv8.1-a,armv8.1-m.main,armv8.2-a,armv8.3-a,armv8.4-a,armv8.5-a,armv8.6-a,armv8.7-a,armv8.8-a,armv8.9-a,armv9-a,armv9.1-a,armv9.2-a,armv9.3-a,armv9.4-a,atomics-32,avoid-movs-shop,avoid-partial-cpsr,bf16,big-endian-instructions,cde,cdecp0,cdecp1,cdecp2,cdecp3,cdecp4,cdecp5,cdecp6,cdecp7,cheap-predicable-cpsr,clrbhb,cortex-a710,cortex-a78,cortex-x1,cortex-x1c,crc,crypto,d32,db,dfb,disable-postra-scheduler,dont-widen-vmovs,dotprod,dsp,execute-only,expand-fp-mlx,exynos,fix-cmse-cve-2021-35465,fix-cortex-a57-aes-1742098,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp16fml,fp64,fpao,fpregs,fpregs16,fpregs64,fullfp16,fuse-aes,fuse-literals,harden-sls-blr,harden-sls-nocomdat,harden-sls-retbr,hwdiv,hwdiv-arm,i8mm,iwmmxt,iwmmxt2,krait,kryo,lob,long-calls,loop-align,m3,m7,mclass,mp,muxed-units,mve,mve.fp,mve1beat,mve2beat,mve4beat,nacl-trap,neon,neon-fpmovs,neonfp,neoverse-v1,no-branch-predictor,no-bti-at-return-twice,no-movt,no-neg-immediates,noarm,nonpipelined-vfp,pacbti,perfmon,prefer-ishst,prefer-vmovsr,prof-unpr,r4,r5,r52,r7,ras,rclass,read-tp-tpidrprw,read-tp-tpidruro,read-tp-tpidrurw,reserve-r9,ret-addr-stack,sb,sha2,slow-fp-brcc,slow-load-D-subreg,slow-odd-reg,slow-vdup32,slow-vgetlni32,slowfpvfmx,slowfpvmlx,soft-float,splat-vfp-neon,strict-align,swift,thumb-mode,thumb2,trustzone,use-mipipeliner,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8.1a,v8.1m.main,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8.7a,v8.8a,v8.9a,v8m,v8m.main,v9.1a,v9.2a,v9.3a,v9.4a,v9a,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vldn-align,vmlx-forwarding,vmlx-hazards,wide-stride-vfp,xscale,zcz"), + str_lit("32bit,8msecext,a12,a15,a17,a32,a35,a5,a53,a55,a57,a7,a72,a73,a75,a76,a77,a78c,a8,a9,aapcs-frame-chain,aapcs-frame-chain-leaf,aclass,acquire-release,aes,armv4,armv4t,armv5t,armv5te,armv5tej,armv6,armv6-m,armv6j,armv6k,armv6kz,armv6s-m,armv6t2,armv7-a,armv7-m,armv7-r,armv7e-m,armv7k,armv7s,armv7ve,armv8-a,armv8-m.base,armv8-m.main,armv8-r,armv8.1-a,armv8.1-m.main,armv8.2-a,armv8.3-a,armv8.4-a,armv8.5-a,armv8.6-a,armv8.7-a,armv8.8-a,armv8.9-a,armv9-a,armv9.1-a,armv9.2-a,armv9.3-a,armv9.4-a,armv9.5-a,atomics-32,avoid-movs-shop,avoid-partial-cpsr,bf16,big-endian-instructions,cde,cdecp0,cdecp1,cdecp2,cdecp3,cdecp4,cdecp5,cdecp6,cdecp7,cheap-predicable-cpsr,clrbhb,cortex-a710,cortex-a78,cortex-x1,cortex-x1c,crc,crypto,d32,db,dfb,disable-postra-scheduler,dont-widen-vmovs,dotprod,dsp,execute-only,expand-fp-mlx,exynos,fix-cmse-cve-2021-35465,fix-cortex-a57-aes-1742098,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp16fml,fp64,fpao,fpregs,fpregs16,fpregs64,fullfp16,fuse-aes,fuse-literals,harden-sls-blr,harden-sls-nocomdat,harden-sls-retbr,hwdiv,hwdiv-arm,i8mm,iwmmxt,iwmmxt2,krait,kryo,lob,long-calls,loop-align,m3,m7,mclass,mp,muxed-units,mve,mve.fp,mve1beat,mve2beat,mve4beat,nacl-trap,neon,neon-fpmovs,neonfp,neoverse-v1,no-branch-predictor,no-bti-at-return-twice,no-movt,no-neg-immediates,noarm,nonpipelined-vfp,pacbti,perfmon,prefer-ishst,prefer-vmovsr,prof-unpr,r4,r5,r52,r7,ras,rclass,read-tp-tpidrprw,read-tp-tpidruro,read-tp-tpidrurw,reserve-r9,ret-addr-stack,sb,sha2,slow-fp-brcc,slow-load-D-subreg,slow-odd-reg,slow-vdup32,slow-vgetlni32,slowfpvfmx,slowfpvmlx,soft-float,splat-vfp-neon,strict-align,swift,thumb-mode,thumb2,trustzone,use-mipipeliner,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8.1a,v8.1m.main,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8.7a,v8.8a,v8.9a,v8m,v8m.main,v9.1a,v9.2a,v9.3a,v9.4a,v9.5a,v9a,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vldn-align,vmlx-forwarding,vmlx-hazards,wide-stride-vfp,xscale,zcz"), // TargetArch_arm64: - str_lit("CONTEXTIDREL2,a35,a510,a53,a55,a57,a64fx,a65,a710,a715,a72,a73,a75,a76,a77,a78,a78c,aes,aggressive-fma,all,alternate-sextload-cvt-f32-pattern,altnzcv,am,ampere1,ampere1a,amvs,apple-a10,apple-a11,apple-a12,apple-a13,apple-a14,apple-a15,apple-a16,apple-a7,apple-a7-sysreg,arith-bcc-fusion,arith-cbz-fusion,ascend-store-address,b16b16,balance-fp-ops,bf16,brbe,bti,call-saved-x10,call-saved-x11,call-saved-x12,call-saved-x13,call-saved-x14,call-saved-x15,call-saved-x18,call-saved-x8,call-saved-x9,carmel,ccdp,ccidx,ccpp,chk,clrbhb,cmp-bcc-fusion,complxnum,cortex-r82,cortex-x1,cortex-x2,cortex-x3,crc,crypto,cssc,custom-cheap-as-move,d128,disable-latency-sched-heuristic,dit,dotprod,ecv,el2vmsa,el3,enable-select-opt,ete,exynos-cheap-as-move,exynosm3,exynosm4,f32mm,f64mm,falkor,fgt,fix-cortex-a53-835769,flagm,fmv,force-32bit-jump-tables,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-addsub-2reg-const1,fuse-adrp-add,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,gcs,harden-sls-blr,harden-sls-nocomdat,harden-sls-retbr,hbc,hcx,i8mm,ite,jsconv,kryo,lor,ls64,lse,lse128,lse2,lsl-fast,mec,mops,mpam,mte,neon,neoverse512tvb,neoversee1,neoversen1,neoversen2,neoversev1,neoversev2,nmi,no-bti-at-return-twice,no-neg-immediates,no-sve-fp-ld1r,no-zcz-fp,nv,outline-atomics,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,predres,prfm-slc-target,rand,ras,rasv2,rcpc,rcpc-immo,rcpc3,rdm,reserve-x1,reserve-x10,reserve-x11,reserve-x12,reserve-x13,reserve-x14,reserve-x15,reserve-x18,reserve-x2,reserve-x20,reserve-x21,reserve-x22,reserve-x23,reserve-x24,reserve-x25,reserve-x26,reserve-x27,reserve-x28,reserve-x3,reserve-x30,reserve-x4,reserve-x5,reserve-x6,reserve-x7,reserve-x9,rme,saphira,sb,sel2,sha2,sha3,slow-misaligned-128store,slow-paired-128,slow-strqro-store,sm4,sme,sme-f16f16,sme-f64f64,sme-i16i64,sme2,sme2p1,spe,spe-eef,specres2,specrestrict,ssbs,strict-align,sve,sve2,sve2-aes,sve2-bitperm,sve2-sha3,sve2-sm4,sve2p1,tagged-globals,the,thunderx,thunderx2t99,thunderx3t110,thunderxt81,thunderxt83,thunderxt88,tlb-rmi,tme,tpidr-el1,tpidr-el2,tpidr-el3,tpidrro-el0,tracev8.4,trbe,tsv110,uaops,use-experimental-zeroing-pseudos,use-postra-scheduler,use-reciprocal-square-root,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8.7a,v8.8a,v8.9a,v8a,v8r,v9.1a,v9.2a,v9.3a,v9.4a,v9a,vh,wfxt,xs,zcm,zcz,zcz-fp-workaround,zcz-gp"), + str_lit("CONTEXTIDREL2,a35,a510,a520,a53,a55,a57,a64fx,a65,a710,a715,a72,a720,a73,a75,a76,a77,a78,a78c,addr-lsl-fast,aes,aggressive-fma,all,alternate-sextload-cvt-f32-pattern,altnzcv,alu-lsl-fast,am,ampere1,ampere1a,ampere1b,amvs,apple-a10,apple-a11,apple-a12,apple-a13,apple-a14,apple-a15,apple-a16,apple-a17,apple-a7,apple-a7-sysreg,arith-bcc-fusion,arith-cbz-fusion,ascend-store-address,b16b16,balance-fp-ops,bf16,brbe,bti,call-saved-x10,call-saved-x11,call-saved-x12,call-saved-x13,call-saved-x14,call-saved-x15,call-saved-x18,call-saved-x8,call-saved-x9,carmel,ccdp,ccidx,ccpp,chk,clrbhb,cmp-bcc-fusion,complxnum,cortex-r82,cortex-x1,cortex-x2,cortex-x3,cortex-x4,cpa,crc,crypto,cssc,d128,disable-latency-sched-heuristic,disable-ldp,disable-stp,dit,dotprod,ecv,el2vmsa,el3,enable-select-opt,ete,exynos-cheap-as-move,exynosm3,exynosm4,f32mm,f64mm,falkor,faminmax,fgt,fix-cortex-a53-835769,flagm,fmv,force-32bit-jump-tables,fp-armv8,fp16fml,fp8,fp8dot2,fp8dot4,fp8fma,fpmr,fptoint,fullfp16,fuse-address,fuse-addsub-2reg-const1,fuse-adrp-add,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,gcs,harden-sls-blr,harden-sls-nocomdat,harden-sls-retbr,hbc,hcx,i8mm,ite,jsconv,kryo,ldp-aligned-only,lor,ls64,lse,lse128,lse2,lut,mec,mops,mpam,mte,neon,neoverse512tvb,neoversee1,neoversen1,neoversen2,neoversev1,neoversev2,nmi,no-bti-at-return-twice,no-neg-immediates,no-sve-fp-ld1r,no-zcz-fp,nv,outline-atomics,pan,pan-rwv,pauth,pauth-lr,perfmon,predictable-select-expensive,predres,prfm-slc-target,rand,ras,rasv2,rcpc,rcpc-immo,rcpc3,rdm,reserve-x1,reserve-x10,reserve-x11,reserve-x12,reserve-x13,reserve-x14,reserve-x15,reserve-x18,reserve-x2,reserve-x20,reserve-x21,reserve-x22,reserve-x23,reserve-x24,reserve-x25,reserve-x26,reserve-x27,reserve-x28,reserve-x3,reserve-x30,reserve-x4,reserve-x5,reserve-x6,reserve-x7,reserve-x9,rme,saphira,sb,sel2,sha2,sha3,slow-misaligned-128store,slow-paired-128,slow-strqro-store,sm4,sme,sme-f16f16,sme-f64f64,sme-f8f16,sme-f8f32,sme-fa64,sme-i16i64,sme-lutv2,sme2,sme2p1,spe,spe-eef,specres2,specrestrict,ssbs,ssve-fp8dot2,ssve-fp8dot4,ssve-fp8fma,store-pair-suppress,stp-aligned-only,strict-align,sve,sve2,sve2-aes,sve2-bitperm,sve2-sha3,sve2-sm4,sve2p1,tagged-globals,the,thunderx,thunderx2t99,thunderx3t110,thunderxt81,thunderxt83,thunderxt88,tlb-rmi,tlbiw,tme,tpidr-el1,tpidr-el2,tpidr-el3,tpidrro-el0,tracev8.4,trbe,tsv110,uaops,use-experimental-zeroing-pseudos,use-postra-scheduler,use-reciprocal-square-root,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8.7a,v8.8a,v8.9a,v8a,v8r,v9.1a,v9.2a,v9.3a,v9.4a,v9.5a,v9a,vh,wfxt,xs,zcm,zcz,zcz-fp-workaround,zcz-gp"), // TargetArch_wasm32: - str_lit("atomics,bulk-memory,exception-handling,extended-const,multivalue,mutable-globals,nontrapping-fptoint,reference-types,relaxed-simd,sign-ext,simd128,tail-call"), + str_lit("atomics,bulk-memory,exception-handling,extended-const,multimemory,multivalue,mutable-globals,nontrapping-fptoint,reference-types,relaxed-simd,sign-ext,simd128,tail-call"), // TargetArch_wasm64p32: - str_lit("atomics,bulk-memory,exception-handling,extended-const,multivalue,mutable-globals,nontrapping-fptoint,reference-types,relaxed-simd,sign-ext,simd128,tail-call"), + str_lit("atomics,bulk-memory,exception-handling,extended-const,multimemory,multivalue,mutable-globals,nontrapping-fptoint,reference-types,relaxed-simd,sign-ext,simd128,tail-call"), + // TargetArch_riscv64: + str_lit("32bit,64bit,a,auipc-addi-fusion,c,conditional-cmv-fusion,d,dlen-factor-2,e,experimental,experimental-zacas,experimental-zcmop,experimental-zfbfmin,experimental-zicfilp,experimental-zicfiss,experimental-zimop,experimental-ztso,experimental-zvfbfmin,experimental-zvfbfwma,f,fast-unaligned-access,forced-atomics,h,i,ld-add-fusion,lui-addi-fusion,m,no-default-unroll,no-optimized-zero-stride-load,no-rvc-hints,relax,reserve-x1,reserve-x10,reserve-x11,reserve-x12,reserve-x13,reserve-x14,reserve-x15,reserve-x16,reserve-x17,reserve-x18,reserve-x19,reserve-x2,reserve-x20,reserve-x21,reserve-x22,reserve-x23,reserve-x24,reserve-x25,reserve-x26,reserve-x27,reserve-x28,reserve-x29,reserve-x3,reserve-x30,reserve-x31,reserve-x4,reserve-x5,reserve-x6,reserve-x7,reserve-x8,reserve-x9,save-restore,seq-cst-trailing-fence,shifted-zextw-fusion,short-forward-branch-opt,sifive7,smaia,smepmp,ssaia,svinval,svnapot,svpbmt,tagged-globals,unaligned-scalar-mem,use-postra-scheduler,v,ventana-veyron,xcvalu,xcvbi,xcvbitmanip,xcvelw,xcvmac,xcvmem,xcvsimd,xsfvcp,xsfvfnrclipxfqf,xsfvfwmaccqqq,xsfvqmaccdod,xsfvqmaccqoq,xtheadba,xtheadbb,xtheadbs,xtheadcmo,xtheadcondmov,xtheadfmemidx,xtheadmac,xtheadmemidx,xtheadmempair,xtheadsync,xtheadvdot,xventanacondops,za128rs,za64rs,zawrs,zba,zbb,zbc,zbkb,zbkc,zbkx,zbs,zca,zcb,zcd,zce,zcf,zcmp,zcmt,zdinx,zexth-fusion,zextw-fusion,zfa,zfh,zfhmin,zfinx,zhinx,zhinxmin,zic64b,zicbom,zicbop,zicboz,ziccamoa,ziccif,zicclsm,ziccrse,zicntr,zicond,zicsr,zifencei,zihintntl,zihintpause,zihpm,zk,zkn,zknd,zkne,zknh,zkr,zks,zksed,zksh,zkt,zmmul,zvbb,zvbc,zve32f,zve32x,zve64d,zve64f,zve64x,zvfh,zvfhmin,zvkb,zvkg,zvkn,zvknc,zvkned,zvkng,zvknha,zvknhb,zvks,zvksc,zvksed,zvksg,zvksh,zvkt,zvl1024b,zvl128b,zvl16384b,zvl2048b,zvl256b,zvl32768b,zvl32b,zvl4096b,zvl512b,zvl64b,zvl65536b,zvl8192b"), }; // Generated with the featuregen script in `misc/featuregen` @@ -39,17 +43,19 @@ gb_global int target_microarch_counts[TargetArch_COUNT] = { // TargetArch_Invalid: 0, // TargetArch_amd64: - 120, + 127, // TargetArch_i386: - 120, + 127, // TargetArch_arm32: - 90, + 91, // TargetArch_arm64: - 63, + 69, // TargetArch_wasm32: 3, // TargetArch_wasm64p32: 3, + // TargetArch_riscv64: + 26, }; // Generated with the featuregen script in `misc/featuregen` @@ -57,6 +63,9 @@ gb_global MicroarchFeatureList microarch_features_list[] = { // TargetArch_amd64: { str_lit("alderlake"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("amdfam10"), str_lit("3dnow,3dnowa,64bit,64bit-mode,cmov,cx16,cx8,fast-scalar-shift-masks,fxsr,lzcnt,mmx,nopl,popcnt,prfchw,sahf,sbb-dep-breaking,slow-shld,sse,sse2,sse3,sse4a,vzeroupper,x87") }, + { str_lit("arrowlake"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, + { str_lit("arrowlake-s"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, + { str_lit("arrowlake_s"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("athlon"), str_lit("3dnow,3dnowa,64bit-mode,cmov,cx8,mmx,nopl,slow-shld,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, { str_lit("athlon-4"), str_lit("3dnow,3dnowa,64bit-mode,cmov,cx8,fxsr,mmx,nopl,slow-shld,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, { str_lit("athlon-fx"), str_lit("3dnow,3dnowa,64bit,64bit-mode,cmov,cx8,fast-scalar-shift-masks,fxsr,mmx,nopl,sbb-dep-breaking,slow-shld,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, @@ -81,6 +90,7 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("c3-2"), str_lit("64bit-mode,cmov,cx8,fxsr,mmx,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, { str_lit("cannonlake"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avx512bw,avx512cd,avx512dq,avx512f,avx512ifma,avx512vbmi,avx512vl,bmi,bmi2,clflushopt,cmov,crc32,cx16,cx8,ermsb,evex512,f16c,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pku,popcnt,prefer-256-bit,prfchw,rdrnd,rdseed,sahf,sha,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("cascadelake"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avx512bw,avx512cd,avx512dq,avx512f,avx512vl,avx512vnni,bmi,bmi2,clflushopt,clwb,cmov,crc32,cx16,cx8,ermsb,evex512,f16c,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,faster-shift-than-shuffle,fma,fsgsbase,fxsr,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pku,popcnt,prefer-256-bit,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, + { str_lit("clearwaterforest"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prefetchi,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,usermsr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("cooperlake"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avx512bf16,avx512bw,avx512cd,avx512dq,avx512f,avx512vl,avx512vnni,bmi,bmi2,clflushopt,clwb,cmov,crc32,cx16,cx8,ermsb,evex512,f16c,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,faster-shift-than-shuffle,fma,fsgsbase,fxsr,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pku,popcnt,prefer-256-bit,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("core-avx-i"), str_lit("64bit,64bit-mode,avx,cmov,crc32,cx16,cx8,f16c,false-deps-popcnt,fast-15bytenop,fast-scalar-fsqrt,fast-shld-rotate,fsgsbase,fxsr,idivq-to-divl,macrofusion,mmx,no-bypass-delay-mov,nopl,pclmul,popcnt,rdrnd,sahf,slow-3ops-lea,slow-unaligned-mem-32,sse,sse2,sse3,sse4.1,sse4.2,ssse3,vzeroupper,x87,xsave,xsaveopt") }, { str_lit("core-avx2"), str_lit("64bit,64bit-mode,allow-light-256-bit,avx,avx2,bmi,bmi2,cmov,crc32,cx16,cx8,ermsb,f16c,false-deps-lzcnt-tzcnt,false-deps-popcnt,fast-15bytenop,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fma,fsgsbase,fxsr,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,popcnt,rdrnd,sahf,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,vzeroupper,x87,xsave,xsaveopt") }, @@ -103,6 +113,7 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("goldmont"), str_lit("64bit,64bit-mode,aes,clflushopt,cmov,crc32,cx16,cx8,false-deps-popcnt,fast-movbe,fsgsbase,fxsr,mmx,movbe,no-bypass-delay,nopl,pclmul,popcnt,prfchw,rdrnd,rdseed,sahf,sha,slow-incdec,slow-lea,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,use-glm-div-sqrt-costs,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("goldmont-plus"), str_lit("64bit,64bit-mode,aes,clflushopt,cmov,crc32,cx16,cx8,fast-movbe,fsgsbase,fxsr,mmx,movbe,no-bypass-delay,nopl,pclmul,popcnt,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,sha,slow-incdec,slow-lea,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,use-glm-div-sqrt-costs,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("goldmont_plus"), str_lit("64bit,64bit-mode,aes,clflushopt,cmov,crc32,cx16,cx8,fast-movbe,fsgsbase,fxsr,mmx,movbe,no-bypass-delay,nopl,pclmul,popcnt,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,sha,slow-incdec,slow-lea,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,use-glm-div-sqrt-costs,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, + { str_lit("gracemont"), str_lit("64bit,64bit-mode,adx,aes,avx,avx2,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,f16c,false-deps-popcnt,fast-15bytenop,fast-scalar-fsqrt,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivl-to-divb,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,nopl,pclmul,pconfig,pku,popcnt,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("grandridge"), str_lit("64bit,64bit-mode,adx,aes,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,fast-movbe,fma,fsgsbase,fxsr,gfni,hreset,invpcid,kl,lzcnt,mmx,movbe,movdir64b,movdiri,no-bypass-delay,nopl,pclmul,pconfig,pku,popcnt,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-incdec,slow-lea,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,uintr,use-glm-div-sqrt-costs,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("graniterapids"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,amx-bf16,amx-fp16,amx-int8,amx-tile,avx,avx2,avx512bf16,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512fp16,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,enqcmd,ermsb,evex512,f16c,false-deps-getmant,false-deps-mulc,false-deps-mullq,false-deps-perm,false-deps-range,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fsrm,fxsr,gfni,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-256-bit,prefetchi,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tsxldtrk,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,wbnoinvd,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("graniterapids-d"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,amx-bf16,amx-complex,amx-fp16,amx-int8,amx-tile,avx,avx2,avx512bf16,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512fp16,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,enqcmd,ermsb,evex512,f16c,false-deps-getmant,false-deps-mulc,false-deps-mullq,false-deps-perm,false-deps-range,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fsrm,fxsr,gfni,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-256-bit,prefetchi,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tsxldtrk,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,wbnoinvd,x87,xsave,xsavec,xsaveopt,xsaves") }, @@ -125,12 +136,14 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("knl"), str_lit("64bit,64bit-mode,adx,aes,avx,avx2,avx512cd,avx512er,avx512f,avx512pf,bmi,bmi2,cmov,crc32,cx16,cx8,evex512,f16c,fast-gather,fast-movbe,fma,fsgsbase,fxsr,idivq-to-divl,lzcnt,mmx,movbe,nopl,pclmul,popcnt,prefer-mask-registers,prefetchwt1,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,slow-incdec,slow-pmaddwd,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,x87,xsave,xsaveopt") }, { str_lit("knm"), str_lit("64bit,64bit-mode,adx,aes,avx,avx2,avx512cd,avx512er,avx512f,avx512pf,avx512vpopcntdq,bmi,bmi2,cmov,crc32,cx16,cx8,evex512,f16c,fast-gather,fast-movbe,fma,fsgsbase,fxsr,idivq-to-divl,lzcnt,mmx,movbe,nopl,pclmul,popcnt,prefer-mask-registers,prefetchwt1,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,slow-incdec,slow-pmaddwd,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,x87,xsave,xsaveopt") }, { str_lit("lakemont"), str_lit("64bit-mode,cx8,slow-unaligned-mem-16,sse,sse2,vzeroupper") }, + { str_lit("lunarlake"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("meteorlake"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("mic_avx512"), str_lit("64bit,64bit-mode,adx,aes,avx,avx2,avx512cd,avx512er,avx512f,avx512pf,bmi,bmi2,cmov,crc32,cx16,cx8,evex512,f16c,fast-gather,fast-movbe,fma,fsgsbase,fxsr,idivq-to-divl,lzcnt,mmx,movbe,nopl,pclmul,popcnt,prefer-mask-registers,prefetchwt1,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,slow-incdec,slow-pmaddwd,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,x87,xsave,xsaveopt") }, { str_lit("nehalem"), str_lit("64bit,64bit-mode,cmov,crc32,cx16,cx8,fxsr,macrofusion,mmx,no-bypass-delay-mov,nopl,popcnt,sahf,sse,sse2,sse3,sse4.1,sse4.2,ssse3,vzeroupper,x87") }, { str_lit("nocona"), str_lit("64bit,64bit-mode,cmov,cx16,cx8,fxsr,mmx,nopl,slow-unaligned-mem-16,sse,sse2,sse3,vzeroupper,x87") }, { str_lit("opteron"), str_lit("3dnow,3dnowa,64bit,64bit-mode,cmov,cx8,fast-scalar-shift-masks,fxsr,mmx,nopl,sbb-dep-breaking,slow-shld,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, { str_lit("opteron-sse3"), str_lit("3dnow,3dnowa,64bit,64bit-mode,cmov,cx16,cx8,fast-scalar-shift-masks,fxsr,mmx,nopl,sbb-dep-breaking,slow-shld,slow-unaligned-mem-16,sse,sse2,sse3,vzeroupper,x87") }, + { str_lit("pantherlake"), str_lit("64bit,64bit-mode,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prefetchi,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("penryn"), str_lit("64bit,64bit-mode,cmov,cx16,cx8,fxsr,macrofusion,mmx,nopl,sahf,slow-unaligned-mem-16,sse,sse2,sse3,sse4.1,ssse3,vzeroupper,x87") }, { str_lit("pentium"), str_lit("64bit-mode,cx8,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, { str_lit("pentium-m"), str_lit("64bit-mode,cmov,cx8,fxsr,mmx,nopl,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, @@ -178,6 +191,9 @@ gb_global MicroarchFeatureList microarch_features_list[] = { // TargetArch_i386: { str_lit("alderlake"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("amdfam10"), str_lit("32bit-mode,3dnow,3dnowa,64bit,cmov,cx16,cx8,fast-scalar-shift-masks,fxsr,lzcnt,mmx,nopl,popcnt,prfchw,sahf,sbb-dep-breaking,slow-shld,sse,sse2,sse3,sse4a,vzeroupper,x87") }, + { str_lit("arrowlake"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, + { str_lit("arrowlake-s"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, + { str_lit("arrowlake_s"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("athlon"), str_lit("32bit-mode,3dnow,3dnowa,cmov,cx8,mmx,nopl,slow-shld,slow-unaligned-mem-16,vzeroupper,x87") }, { str_lit("athlon-4"), str_lit("32bit-mode,3dnow,3dnowa,cmov,cx8,fxsr,mmx,nopl,slow-shld,slow-unaligned-mem-16,sse,vzeroupper,x87") }, { str_lit("athlon-fx"), str_lit("32bit-mode,3dnow,3dnowa,64bit,cmov,cx8,fast-scalar-shift-masks,fxsr,mmx,nopl,sbb-dep-breaking,slow-shld,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, @@ -202,6 +218,7 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("c3-2"), str_lit("32bit-mode,cmov,cx8,fxsr,mmx,slow-unaligned-mem-16,sse,vzeroupper,x87") }, { str_lit("cannonlake"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avx512bw,avx512cd,avx512dq,avx512f,avx512ifma,avx512vbmi,avx512vl,bmi,bmi2,clflushopt,cmov,crc32,cx16,cx8,ermsb,evex512,f16c,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pku,popcnt,prefer-256-bit,prfchw,rdrnd,rdseed,sahf,sha,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("cascadelake"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avx512bw,avx512cd,avx512dq,avx512f,avx512vl,avx512vnni,bmi,bmi2,clflushopt,clwb,cmov,crc32,cx16,cx8,ermsb,evex512,f16c,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,faster-shift-than-shuffle,fma,fsgsbase,fxsr,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pku,popcnt,prefer-256-bit,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, + { str_lit("clearwaterforest"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prefetchi,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,usermsr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("cooperlake"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avx512bf16,avx512bw,avx512cd,avx512dq,avx512f,avx512vl,avx512vnni,bmi,bmi2,clflushopt,clwb,cmov,crc32,cx16,cx8,ermsb,evex512,f16c,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,faster-shift-than-shuffle,fma,fsgsbase,fxsr,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pku,popcnt,prefer-256-bit,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("core-avx-i"), str_lit("32bit-mode,64bit,avx,cmov,crc32,cx16,cx8,f16c,false-deps-popcnt,fast-15bytenop,fast-scalar-fsqrt,fast-shld-rotate,fsgsbase,fxsr,idivq-to-divl,macrofusion,mmx,no-bypass-delay-mov,nopl,pclmul,popcnt,rdrnd,sahf,slow-3ops-lea,slow-unaligned-mem-32,sse,sse2,sse3,sse4.1,sse4.2,ssse3,vzeroupper,x87,xsave,xsaveopt") }, { str_lit("core-avx2"), str_lit("32bit-mode,64bit,allow-light-256-bit,avx,avx2,bmi,bmi2,cmov,crc32,cx16,cx8,ermsb,f16c,false-deps-lzcnt-tzcnt,false-deps-popcnt,fast-15bytenop,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fma,fsgsbase,fxsr,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,popcnt,rdrnd,sahf,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,vzeroupper,x87,xsave,xsaveopt") }, @@ -224,6 +241,7 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("goldmont"), str_lit("32bit-mode,64bit,aes,clflushopt,cmov,crc32,cx16,cx8,false-deps-popcnt,fast-movbe,fsgsbase,fxsr,mmx,movbe,no-bypass-delay,nopl,pclmul,popcnt,prfchw,rdrnd,rdseed,sahf,sha,slow-incdec,slow-lea,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,use-glm-div-sqrt-costs,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("goldmont-plus"), str_lit("32bit-mode,64bit,aes,clflushopt,cmov,crc32,cx16,cx8,fast-movbe,fsgsbase,fxsr,mmx,movbe,no-bypass-delay,nopl,pclmul,popcnt,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,sha,slow-incdec,slow-lea,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,use-glm-div-sqrt-costs,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("goldmont_plus"), str_lit("32bit-mode,64bit,aes,clflushopt,cmov,crc32,cx16,cx8,fast-movbe,fsgsbase,fxsr,mmx,movbe,no-bypass-delay,nopl,pclmul,popcnt,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,sha,slow-incdec,slow-lea,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,use-glm-div-sqrt-costs,vzeroupper,x87,xsave,xsavec,xsaveopt,xsaves") }, + { str_lit("gracemont"), str_lit("32bit-mode,64bit,adx,aes,avx,avx2,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,f16c,false-deps-popcnt,fast-15bytenop,fast-scalar-fsqrt,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivl-to-divb,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,nopl,pclmul,pconfig,pku,popcnt,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("grandridge"), str_lit("32bit-mode,64bit,adx,aes,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,fast-movbe,fma,fsgsbase,fxsr,gfni,hreset,invpcid,kl,lzcnt,mmx,movbe,movdir64b,movdiri,no-bypass-delay,nopl,pclmul,pconfig,pku,popcnt,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-incdec,slow-lea,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,uintr,use-glm-div-sqrt-costs,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("graniterapids"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,amx-bf16,amx-fp16,amx-int8,amx-tile,avx,avx2,avx512bf16,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512fp16,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,enqcmd,ermsb,evex512,f16c,false-deps-getmant,false-deps-mulc,false-deps-mullq,false-deps-perm,false-deps-range,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fsrm,fxsr,gfni,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-256-bit,prefetchi,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tsxldtrk,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,wbnoinvd,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("graniterapids-d"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,amx-bf16,amx-complex,amx-fp16,amx-int8,amx-tile,avx,avx2,avx512bf16,avx512bitalg,avx512bw,avx512cd,avx512dq,avx512f,avx512fp16,avx512ifma,avx512vbmi,avx512vbmi2,avx512vl,avx512vnni,avx512vpopcntdq,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,enqcmd,ermsb,evex512,f16c,false-deps-getmant,false-deps-mulc,false-deps-mullq,false-deps-perm,false-deps-range,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fsrm,fxsr,gfni,idivq-to-divl,invpcid,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-256-bit,prefetchi,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tsxldtrk,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,wbnoinvd,x87,xsave,xsavec,xsaveopt,xsaves") }, @@ -246,12 +264,14 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("knl"), str_lit("32bit-mode,64bit,adx,aes,avx,avx2,avx512cd,avx512er,avx512f,avx512pf,bmi,bmi2,cmov,crc32,cx16,cx8,evex512,f16c,fast-gather,fast-movbe,fma,fsgsbase,fxsr,idivq-to-divl,lzcnt,mmx,movbe,nopl,pclmul,popcnt,prefer-mask-registers,prefetchwt1,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,slow-incdec,slow-pmaddwd,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,x87,xsave,xsaveopt") }, { str_lit("knm"), str_lit("32bit-mode,64bit,adx,aes,avx,avx2,avx512cd,avx512er,avx512f,avx512pf,avx512vpopcntdq,bmi,bmi2,cmov,crc32,cx16,cx8,evex512,f16c,fast-gather,fast-movbe,fma,fsgsbase,fxsr,idivq-to-divl,lzcnt,mmx,movbe,nopl,pclmul,popcnt,prefer-mask-registers,prefetchwt1,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,slow-incdec,slow-pmaddwd,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,x87,xsave,xsaveopt") }, { str_lit("lakemont"), str_lit("32bit-mode,cx8,slow-unaligned-mem-16,vzeroupper") }, + { str_lit("lunarlake"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("meteorlake"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avxvnni,bmi,bmi2,cldemote,clflushopt,clwb,cmov,crc32,cx16,cx8,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,shstk,slow-3ops-lea,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("mic_avx512"), str_lit("32bit-mode,64bit,adx,aes,avx,avx2,avx512cd,avx512er,avx512f,avx512pf,bmi,bmi2,cmov,crc32,cx16,cx8,evex512,f16c,fast-gather,fast-movbe,fma,fsgsbase,fxsr,idivq-to-divl,lzcnt,mmx,movbe,nopl,pclmul,popcnt,prefer-mask-registers,prefetchwt1,prfchw,rdrnd,rdseed,sahf,slow-3ops-lea,slow-incdec,slow-pmaddwd,slow-two-mem-ops,sse,sse2,sse3,sse4.1,sse4.2,ssse3,x87,xsave,xsaveopt") }, { str_lit("nehalem"), str_lit("32bit-mode,64bit,cmov,crc32,cx16,cx8,fxsr,macrofusion,mmx,no-bypass-delay-mov,nopl,popcnt,sahf,sse,sse2,sse3,sse4.1,sse4.2,ssse3,vzeroupper,x87") }, { str_lit("nocona"), str_lit("32bit-mode,64bit,cmov,cx16,cx8,fxsr,mmx,nopl,slow-unaligned-mem-16,sse,sse2,sse3,vzeroupper,x87") }, { str_lit("opteron"), str_lit("32bit-mode,3dnow,3dnowa,64bit,cmov,cx8,fast-scalar-shift-masks,fxsr,mmx,nopl,sbb-dep-breaking,slow-shld,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, { str_lit("opteron-sse3"), str_lit("32bit-mode,3dnow,3dnowa,64bit,cmov,cx16,cx8,fast-scalar-shift-masks,fxsr,mmx,nopl,sbb-dep-breaking,slow-shld,slow-unaligned-mem-16,sse,sse2,sse3,vzeroupper,x87") }, + { str_lit("pantherlake"), str_lit("32bit-mode,64bit,adx,aes,allow-light-256-bit,avx,avx2,avxifma,avxneconvert,avxvnni,avxvnniint16,avxvnniint8,bmi,bmi2,cldemote,clflushopt,clwb,cmov,cmpccxadd,crc32,cx16,cx8,enqcmd,f16c,false-deps-perm,false-deps-popcnt,fast-15bytenop,fast-gather,fast-scalar-fsqrt,fast-shld-rotate,fast-variable-crosslane-shuffle,fast-variable-perlane-shuffle,fast-vector-fsqrt,fma,fsgsbase,fxsr,gfni,hreset,idivq-to-divl,invpcid,kl,lzcnt,macrofusion,mmx,movbe,movdir64b,movdiri,no-bypass-delay-blend,no-bypass-delay-mov,no-bypass-delay-shuffle,nopl,pclmul,pconfig,pku,popcnt,prefer-movmsk-over-vtest,prefetchi,prfchw,ptwrite,rdpid,rdrnd,rdseed,sahf,serialize,sha,sha512,shstk,slow-3ops-lea,sm3,sm4,sse,sse2,sse3,sse4.1,sse4.2,ssse3,tuning-fast-imm-vector-shift,uintr,vaes,vpclmulqdq,vzeroupper,waitpkg,widekl,x87,xsave,xsavec,xsaveopt,xsaves") }, { str_lit("penryn"), str_lit("32bit-mode,64bit,cmov,cx16,cx8,fxsr,macrofusion,mmx,nopl,sahf,slow-unaligned-mem-16,sse,sse2,sse3,sse4.1,ssse3,vzeroupper,x87") }, { str_lit("pentium"), str_lit("32bit-mode,cx8,slow-unaligned-mem-16,vzeroupper,x87") }, { str_lit("pentium-m"), str_lit("32bit-mode,cmov,cx8,fxsr,mmx,nopl,slow-unaligned-mem-16,sse,sse2,vzeroupper,x87") }, @@ -325,16 +345,16 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("arm968e-s"), str_lit("armv5te,v4t,v5t,v5te") }, { str_lit("arm9e"), str_lit("armv5te,v4t,v5t,v5te") }, { str_lit("arm9tdmi"), str_lit("armv4t,v4t") }, - { str_lit("cortex-a12"), str_lit("a12,aclass,armv7-a,avoid-partial-cpsr,d32,db,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,ret-addr-stack,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vmlx-forwarding") }, - { str_lit("cortex-a15"), str_lit("a15,aclass,armv7-a,avoid-partial-cpsr,d32,db,dont-widen-vmovs,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,muxed-units,neon,perfmon,ret-addr-stack,splat-vfp-neon,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vldn-align") }, - { str_lit("cortex-a17"), str_lit("a17,aclass,armv7-a,avoid-partial-cpsr,d32,db,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,ret-addr-stack,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vmlx-forwarding") }, + { str_lit("cortex-a12"), str_lit("a12,aclass,armv7-a,avoid-partial-cpsr,d32,db,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,perfmon,ret-addr-stack,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vmlx-forwarding") }, + { str_lit("cortex-a15"), str_lit("a15,aclass,armv7-a,avoid-partial-cpsr,d32,db,dont-widen-vmovs,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,muxed-units,perfmon,ret-addr-stack,splat-vfp-neon,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vldn-align") }, + { str_lit("cortex-a17"), str_lit("a17,aclass,armv7-a,avoid-partial-cpsr,d32,db,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,perfmon,ret-addr-stack,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vmlx-forwarding") }, { str_lit("cortex-a32"), str_lit("aclass,acquire-release,aes,armv8-a,crc,crypto,d32,db,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, { str_lit("cortex-a35"), str_lit("a35,aclass,acquire-release,aes,armv8-a,crc,crypto,d32,db,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, - { str_lit("cortex-a5"), str_lit("a5,aclass,armv7-a,d32,db,dsp,fp16,fp64,fpregs,fpregs64,mp,neon,perfmon,ret-addr-stack,slow-fp-brcc,slowfpvfmx,slowfpvmlx,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,vmlx-forwarding") }, + { str_lit("cortex-a5"), str_lit("a5,aclass,armv7-a,d32,db,dsp,fp16,fp64,fpregs,fpregs64,mp,perfmon,ret-addr-stack,slow-fp-brcc,slowfpvfmx,slowfpvmlx,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,vmlx-forwarding") }, { str_lit("cortex-a53"), str_lit("a53,aclass,acquire-release,aes,armv8-a,crc,crypto,d32,db,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpao,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, { str_lit("cortex-a55"), str_lit("a55,aclass,acquire-release,aes,armv8.2-a,crc,crypto,d32,db,dotprod,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,ras,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8.1a,v8.2a,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, { str_lit("cortex-a57"), str_lit("a57,aclass,acquire-release,aes,armv8-a,avoid-partial-cpsr,cheap-predicable-cpsr,crc,crypto,d32,db,dsp,fix-cortex-a57-aes-1742098,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpao,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, - { str_lit("cortex-a7"), str_lit("a7,aclass,armv7-a,d32,db,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,ret-addr-stack,slow-fp-brcc,slowfpvfmx,slowfpvmlx,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vmlx-forwarding,vmlx-hazards") }, + { str_lit("cortex-a7"), str_lit("a7,aclass,armv7-a,d32,db,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,perfmon,ret-addr-stack,slow-fp-brcc,slowfpvfmx,slowfpvmlx,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,vmlx-forwarding,vmlx-hazards") }, { str_lit("cortex-a710"), str_lit("aclass,acquire-release,armv9-a,bf16,cortex-a710,crc,d32,db,dotprod,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp16fml,fp64,fpregs,fpregs16,fpregs64,fullfp16,hwdiv,hwdiv-arm,i8mm,mp,neon,perfmon,ras,sb,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8m,v9a,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, { str_lit("cortex-a72"), str_lit("a72,aclass,acquire-release,aes,armv8-a,crc,crypto,d32,db,dsp,fix-cortex-a57-aes-1742098,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, { str_lit("cortex-a73"), str_lit("a73,aclass,acquire-release,aes,armv8-a,crc,crypto,d32,db,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,perfmon,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, @@ -344,8 +364,8 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("cortex-a77"), str_lit("a77,aclass,acquire-release,aes,armv8.2-a,crc,crypto,d32,db,dotprod,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs16,fpregs64,fullfp16,hwdiv,hwdiv-arm,mp,neon,perfmon,ras,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8.1a,v8.2a,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, { str_lit("cortex-a78"), str_lit("aclass,acquire-release,aes,armv8.2-a,cortex-a78,crc,crypto,d32,db,dotprod,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs16,fpregs64,fullfp16,hwdiv,hwdiv-arm,mp,neon,perfmon,ras,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8.1a,v8.2a,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, { str_lit("cortex-a78c"), str_lit("a78c,aclass,acquire-release,aes,armv8.2-a,crc,crypto,d32,db,dotprod,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs16,fpregs64,fullfp16,hwdiv,hwdiv-arm,mp,neon,perfmon,ras,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8.1a,v8.2a,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, - { str_lit("cortex-a8"), str_lit("a8,aclass,armv7-a,d32,db,dsp,fp64,fpregs,fpregs64,neon,nonpipelined-vfp,perfmon,ret-addr-stack,slow-fp-brcc,slowfpvfmx,slowfpvmlx,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vmlx-forwarding,vmlx-hazards") }, - { str_lit("cortex-a9"), str_lit("a9,aclass,armv7-a,avoid-partial-cpsr,d32,db,dsp,expand-fp-mlx,fp16,fp64,fpregs,fpregs64,mp,muxed-units,neon,neon-fpmovs,perfmon,prefer-vmovsr,ret-addr-stack,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vldn-align,vmlx-forwarding,vmlx-hazards") }, + { str_lit("cortex-a8"), str_lit("a8,aclass,armv7-a,d32,db,dsp,fp64,fpregs,fpregs64,nonpipelined-vfp,perfmon,ret-addr-stack,slow-fp-brcc,slowfpvfmx,slowfpvmlx,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vmlx-forwarding,vmlx-hazards") }, + { str_lit("cortex-a9"), str_lit("a9,aclass,armv7-a,avoid-partial-cpsr,d32,db,dsp,expand-fp-mlx,fp16,fp64,fpregs,fpregs64,mp,muxed-units,neon-fpmovs,perfmon,prefer-vmovsr,ret-addr-stack,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vldn-align,vmlx-forwarding,vmlx-hazards") }, { str_lit("cortex-m0"), str_lit("armv6-m,db,mclass,no-branch-predictor,noarm,strict-align,thumb-mode,v4t,v5t,v5te,v6,v6m") }, { str_lit("cortex-m0plus"), str_lit("armv6-m,db,mclass,no-branch-predictor,noarm,strict-align,thumb-mode,v4t,v5t,v5te,v6,v6m") }, { str_lit("cortex-m1"), str_lit("armv6-m,db,mclass,no-branch-predictor,noarm,strict-align,thumb-mode,v4t,v5t,v5te,v6,v6m") }, @@ -354,6 +374,7 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("cortex-m33"), str_lit("8msecext,acquire-release,armv8-m.main,db,dsp,fix-cmse-cve-2021-35465,fp-armv8d16sp,fp16,fpregs,hwdiv,loop-align,mclass,no-branch-predictor,noarm,slowfpvfmx,slowfpvmlx,thumb-mode,thumb2,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,v8m.main,vfp2sp,vfp3d16sp,vfp4d16sp") }, { str_lit("cortex-m35p"), str_lit("8msecext,acquire-release,armv8-m.main,db,dsp,fix-cmse-cve-2021-35465,fp-armv8d16sp,fp16,fpregs,hwdiv,loop-align,mclass,no-branch-predictor,noarm,slowfpvfmx,slowfpvmlx,thumb-mode,thumb2,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,v8m.main,vfp2sp,vfp3d16sp,vfp4d16sp") }, { str_lit("cortex-m4"), str_lit("armv7e-m,db,dsp,fp16,fpregs,hwdiv,loop-align,mclass,no-branch-predictor,noarm,slowfpvfmx,slowfpvmlx,thumb-mode,thumb2,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2sp,vfp3d16sp,vfp4d16sp") }, + { str_lit("cortex-m52"), str_lit("8msecext,acquire-release,armv8.1-m.main,db,dsp,fp-armv8d16,fp-armv8d16sp,fp16,fp64,fpregs,fpregs16,fpregs64,fullfp16,hwdiv,lob,loop-align,mclass,mve,mve.fp,mve1beat,no-branch-predictor,noarm,pacbti,ras,slowfpvmlx,thumb-mode,thumb2,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8.1m.main,v8m,v8m.main,vfp2,vfp2sp,vfp3d16,vfp3d16sp,vfp4d16,vfp4d16sp") }, { str_lit("cortex-m55"), str_lit("8msecext,acquire-release,armv8.1-m.main,db,dsp,fix-cmse-cve-2021-35465,fp-armv8d16,fp-armv8d16sp,fp16,fp64,fpregs,fpregs16,fpregs64,fullfp16,hwdiv,lob,loop-align,mclass,mve,mve.fp,no-branch-predictor,noarm,ras,slowfpvmlx,thumb-mode,thumb2,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8.1m.main,v8m,v8m.main,vfp2,vfp2sp,vfp3d16,vfp3d16sp,vfp4d16,vfp4d16sp") }, { str_lit("cortex-m7"), str_lit("armv7e-m,db,dsp,fp-armv8d16,fp-armv8d16sp,fp16,fp64,fpregs,fpregs64,hwdiv,m7,mclass,noarm,thumb-mode,thumb2,use-mipipeliner,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3d16,vfp3d16sp,vfp4d16,vfp4d16sp") }, { str_lit("cortex-m85"), str_lit("8msecext,acquire-release,armv8.1-m.main,db,dsp,fp-armv8d16,fp-armv8d16sp,fp16,fp64,fpregs,fpregs16,fpregs64,fullfp16,hwdiv,lob,mclass,mve,mve.fp,noarm,pacbti,ras,thumb-mode,thumb2,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8.1m.main,v8m,v8m.main,vfp2,vfp2sp,vfp3d16,vfp3d16sp,vfp4d16,vfp4d16sp") }, @@ -372,7 +393,7 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("exynos-m5"), str_lit("aclass,acquire-release,aes,armv8.2-a,crc,crypto,d32,db,dont-widen-vmovs,dotprod,dsp,expand-fp-mlx,exynos,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs16,fpregs64,fullfp16,fuse-aes,fuse-literals,hwdiv,hwdiv-arm,mp,neon,perfmon,prof-unpr,ras,ret-addr-stack,sha2,slow-fp-brcc,slow-vdup32,slow-vgetlni32,slowfpvfmx,slowfpvmlx,splat-vfp-neon,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8.1a,v8.2a,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization,wide-stride-vfp,zcz") }, { str_lit("generic"), str_lit("") }, { str_lit("iwmmxt"), str_lit("armv5te,v4t,v5t,v5te") }, - { str_lit("krait"), str_lit("aclass,armv7-a,avoid-partial-cpsr,d32,db,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,krait,muxed-units,neon,perfmon,ret-addr-stack,thumb2,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,vldn-align,vmlx-forwarding") }, + { str_lit("krait"), str_lit("aclass,armv7-a,avoid-partial-cpsr,d32,db,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,krait,muxed-units,perfmon,ret-addr-stack,thumb2,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,vldn-align,vmlx-forwarding") }, { str_lit("kryo"), str_lit("aclass,acquire-release,aes,armv8-a,crc,crypto,d32,db,dsp,fp-armv8,fp-armv8d16,fp-armv8d16sp,fp-armv8sp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,kryo,mp,neon,perfmon,sha2,thumb2,trustzone,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,virtualization") }, { str_lit("mpcore"), str_lit("armv6k,fp64,fpregs,fpregs64,slowfpvmlx,v4t,v5t,v5te,v6,v6k,vfp2,vfp2sp") }, { str_lit("mpcorenovfp"), str_lit("armv6k,v4t,v5t,v5te,v6,v6k") }, @@ -385,12 +406,13 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("strongarm110"), str_lit("armv4") }, { str_lit("strongarm1100"), str_lit("armv4") }, { str_lit("strongarm1110"), str_lit("armv4") }, - { str_lit("swift"), str_lit("aclass,armv7-a,avoid-movs-shop,avoid-partial-cpsr,d32,db,disable-postra-scheduler,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neon,neonfp,perfmon,prefer-ishst,prof-unpr,ret-addr-stack,slow-load-D-subreg,slow-odd-reg,slow-vdup32,slow-vgetlni32,slowfpvfmx,slowfpvmlx,swift,thumb2,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,vmlx-hazards,wide-stride-vfp") }, + { str_lit("swift"), str_lit("aclass,armv7-a,avoid-movs-shop,avoid-partial-cpsr,d32,db,disable-postra-scheduler,dsp,fp16,fp64,fpregs,fpregs64,hwdiv,hwdiv-arm,mp,neonfp,perfmon,prefer-ishst,prof-unpr,ret-addr-stack,slow-load-D-subreg,slow-odd-reg,slow-vdup32,slow-vgetlni32,slowfpvfmx,slowfpvmlx,swift,thumb2,use-misched,v4t,v5t,v5te,v6,v6k,v6m,v6t2,v7,v7clrex,v8m,vfp2,vfp2sp,vfp3,vfp3d16,vfp3d16sp,vfp3sp,vfp4,vfp4d16,vfp4d16sp,vfp4sp,vmlx-hazards,wide-stride-vfp") }, { str_lit("xscale"), str_lit("armv5te,v4t,v5t,v5te") }, // TargetArch_arm64: { str_lit("a64fx"), str_lit("CONTEXTIDREL2,a64fx,aggressive-fma,arith-bcc-fusion,ccpp,complxnum,crc,el2vmsa,el3,fp-armv8,fullfp16,lor,lse,neon,pan,pan-rwv,perfmon,predictable-select-expensive,ras,rdm,sha2,store-pair-suppress,sve,uaops,use-postra-scheduler,v8.1a,v8.2a,v8a,vh") }, { str_lit("ampere1"), str_lit("CONTEXTIDREL2,addr-lsl-fast,aes,aggressive-fma,altnzcv,alu-lsl-fast,am,ampere1,amvs,arith-bcc-fusion,bf16,bti,ccdp,ccidx,ccpp,cmp-bcc-fusion,complxnum,crc,dit,dotprod,ecv,el2vmsa,el3,fgt,flagm,fp-armv8,fptoint,fuse-address,fuse-aes,fuse-literals,i8mm,jsconv,ldp-aligned-only,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,rand,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,stp-aligned-only,tlb-rmi,tracev8.4,uaops,use-postra-scheduler,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8a,vh") }, { str_lit("ampere1a"), str_lit("CONTEXTIDREL2,addr-lsl-fast,aes,aggressive-fma,altnzcv,alu-lsl-fast,am,ampere1a,amvs,arith-bcc-fusion,bf16,bti,ccdp,ccidx,ccpp,cmp-bcc-fusion,complxnum,crc,dit,dotprod,ecv,el2vmsa,el3,fgt,flagm,fp-armv8,fptoint,fuse-address,fuse-aes,fuse-literals,i8mm,jsconv,ldp-aligned-only,lor,lse,lse2,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predres,rand,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,sm4,specrestrict,ssbs,store-pair-suppress,stp-aligned-only,tlb-rmi,tracev8.4,uaops,use-postra-scheduler,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8a,vh") }, + { str_lit("ampere1b"), str_lit("CONTEXTIDREL2,addr-lsl-fast,aes,aggressive-fma,altnzcv,alu-lsl-fast,am,ampere1b,amvs,arith-bcc-fusion,bf16,bti,ccdp,ccidx,ccpp,cmp-bcc-fusion,complxnum,crc,cssc,dit,dotprod,ecv,el2vmsa,el3,enable-select-opt,fgt,flagm,fp-armv8,fptoint,fullfp16,fuse-address,fuse-adrp-add,fuse-aes,fuse-literals,hcx,i8mm,jsconv,ldp-aligned-only,lor,lse,lse2,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,predres,rand,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,sm4,specrestrict,ssbs,store-pair-suppress,stp-aligned-only,tlb-rmi,tracev8.4,uaops,use-postra-scheduler,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8.7a,v8a,vh,wfxt,xs") }, { str_lit("apple-a10"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,apple-a10,arith-bcc-fusion,arith-cbz-fusion,crc,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fuse-aes,fuse-crypto-eor,lor,neon,pan,perfmon,rdm,sha2,store-pair-suppress,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-a11"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,apple-a11,arith-bcc-fusion,arith-cbz-fusion,ccpp,crc,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fullfp16,fuse-aes,fuse-crypto-eor,lor,lse,neon,pan,pan-rwv,perfmon,ras,rdm,sha2,store-pair-suppress,uaops,v8.1a,v8.2a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-a12"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,apple-a12,arith-bcc-fusion,arith-cbz-fusion,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fullfp16,fuse-aes,fuse-crypto-eor,jsconv,lor,lse,neon,pan,pan-rwv,pauth,perfmon,ras,rcpc,rdm,sha2,store-pair-suppress,uaops,v8.1a,v8.2a,v8.3a,v8a,vh,zcm,zcz,zcz-gp") }, @@ -398,18 +420,21 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("apple-a14"), str_lit("CONTEXTIDREL2,aes,aggressive-fma,alternate-sextload-cvt-f32-pattern,altnzcv,am,apple-a14,arith-bcc-fusion,arith-cbz-fusion,ccdp,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,dit,dotprod,el2vmsa,el3,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-adrp-add,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,jsconv,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,tlb-rmi,tracev8.4,uaops,v8.1a,v8.2a,v8.3a,v8.4a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-a15"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,altnzcv,am,amvs,apple-a15,arith-bcc-fusion,arith-cbz-fusion,bf16,bti,ccdp,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,dit,dotprod,ecv,el2vmsa,el3,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,i8mm,jsconv,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,tlb-rmi,tracev8.4,uaops,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-a16"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,altnzcv,am,amvs,apple-a16,arith-bcc-fusion,arith-cbz-fusion,bf16,bti,ccdp,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,dit,dotprod,ecv,el2vmsa,el3,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,hcx,i8mm,jsconv,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,tlb-rmi,tracev8.4,uaops,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8a,vh,zcm,zcz,zcz-gp") }, + { str_lit("apple-a17"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,altnzcv,am,amvs,apple-a17,arith-bcc-fusion,arith-cbz-fusion,bf16,bti,ccdp,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,dit,dotprod,ecv,el2vmsa,el3,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,hcx,i8mm,jsconv,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,tlb-rmi,tracev8.4,uaops,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-a7"), str_lit("aes,alternate-sextload-cvt-f32-pattern,apple-a7,apple-a7-sysreg,arith-bcc-fusion,arith-cbz-fusion,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fuse-aes,fuse-crypto-eor,neon,perfmon,sha2,store-pair-suppress,v8a,zcm,zcz,zcz-fp-workaround,zcz-gp") }, { str_lit("apple-a8"), str_lit("aes,alternate-sextload-cvt-f32-pattern,apple-a7,apple-a7-sysreg,arith-bcc-fusion,arith-cbz-fusion,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fuse-aes,fuse-crypto-eor,neon,perfmon,sha2,store-pair-suppress,v8a,zcm,zcz,zcz-fp-workaround,zcz-gp") }, { str_lit("apple-a9"), str_lit("aes,alternate-sextload-cvt-f32-pattern,apple-a7,apple-a7-sysreg,arith-bcc-fusion,arith-cbz-fusion,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fuse-aes,fuse-crypto-eor,neon,perfmon,sha2,store-pair-suppress,v8a,zcm,zcz,zcz-fp-workaround,zcz-gp") }, { str_lit("apple-latest"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,altnzcv,am,amvs,apple-a16,arith-bcc-fusion,arith-cbz-fusion,bf16,bti,ccdp,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,dit,dotprod,ecv,el2vmsa,el3,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,hcx,i8mm,jsconv,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,tlb-rmi,tracev8.4,uaops,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-m1"), str_lit("CONTEXTIDREL2,aes,aggressive-fma,alternate-sextload-cvt-f32-pattern,altnzcv,am,apple-a14,arith-bcc-fusion,arith-cbz-fusion,ccdp,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,dit,dotprod,el2vmsa,el3,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-adrp-add,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,jsconv,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,tlb-rmi,tracev8.4,uaops,v8.1a,v8.2a,v8.3a,v8.4a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-m2"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,altnzcv,am,amvs,apple-a15,arith-bcc-fusion,arith-cbz-fusion,bf16,bti,ccdp,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,dit,dotprod,ecv,el2vmsa,el3,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,i8mm,jsconv,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,tlb-rmi,tracev8.4,uaops,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8a,vh,zcm,zcz,zcz-gp") }, + { str_lit("apple-m3"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,altnzcv,am,amvs,apple-a16,arith-bcc-fusion,arith-cbz-fusion,bf16,bti,ccdp,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,dit,dotprod,ecv,el2vmsa,el3,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-address,fuse-aes,fuse-arith-logic,fuse-crypto-eor,fuse-csel,fuse-literals,hcx,i8mm,jsconv,lor,lse,lse2,mpam,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,sb,sel2,sha2,sha3,specrestrict,ssbs,store-pair-suppress,tlb-rmi,tracev8.4,uaops,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-s4"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,apple-a12,arith-bcc-fusion,arith-cbz-fusion,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fullfp16,fuse-aes,fuse-crypto-eor,jsconv,lor,lse,neon,pan,pan-rwv,pauth,perfmon,ras,rcpc,rdm,sha2,store-pair-suppress,uaops,v8.1a,v8.2a,v8.3a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("apple-s5"), str_lit("CONTEXTIDREL2,aes,alternate-sextload-cvt-f32-pattern,apple-a12,arith-bcc-fusion,arith-cbz-fusion,ccidx,ccpp,complxnum,crc,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fullfp16,fuse-aes,fuse-crypto-eor,jsconv,lor,lse,neon,pan,pan-rwv,pauth,perfmon,ras,rcpc,rdm,sha2,store-pair-suppress,uaops,v8.1a,v8.2a,v8.3a,v8a,vh,zcm,zcz,zcz-gp") }, { str_lit("carmel"), str_lit("CONTEXTIDREL2,aes,carmel,ccpp,crc,crypto,el2vmsa,el3,fp-armv8,fullfp16,lor,lse,neon,pan,pan-rwv,ras,rdm,sha2,uaops,v8.1a,v8.2a,v8a,vh") }, { str_lit("cortex-a34"), str_lit("a35,aes,crc,crypto,el2vmsa,el3,fp-armv8,neon,perfmon,sha2,v8a") }, { str_lit("cortex-a35"), str_lit("a35,aes,crc,crypto,el2vmsa,el3,fp-armv8,neon,perfmon,sha2,v8a") }, { str_lit("cortex-a510"), str_lit("CONTEXTIDREL2,a510,altnzcv,am,bf16,bti,ccdp,ccidx,ccpp,complxnum,crc,dit,dotprod,el2vmsa,el3,ete,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-adrp-add,fuse-aes,i8mm,jsconv,lor,lse,lse2,mec,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,rme,sb,sel2,specrestrict,ssbs,sve,sve2,sve2-bitperm,tlb-rmi,tracev8.4,trbe,uaops,use-postra-scheduler,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8a,v9a,vh") }, + { str_lit("cortex-a520"), str_lit("CONTEXTIDREL2,a520,altnzcv,am,amvs,bf16,bti,ccdp,ccidx,ccpp,complxnum,crc,dit,dotprod,ecv,el2vmsa,el3,ete,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-adrp-add,fuse-aes,hcx,i8mm,jsconv,lor,lse,lse2,mec,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predres,ras,rcpc,rcpc-immo,rdm,rme,sb,sel2,specrestrict,ssbs,sve,sve2,sve2-bitperm,tlb-rmi,tracev8.4,trbe,uaops,use-postra-scheduler,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8.7a,v8a,v9.1a,v9.2a,v9a,vh,wfxt,xs") }, { str_lit("cortex-a53"), str_lit("a53,aes,balance-fp-ops,crc,crypto,el2vmsa,el3,fp-armv8,fuse-adrp-add,fuse-aes,neon,perfmon,sha2,use-postra-scheduler,v8a") }, { str_lit("cortex-a55"), str_lit("CONTEXTIDREL2,a55,aes,ccpp,crc,crypto,dotprod,el2vmsa,el3,fp-armv8,fullfp16,fuse-address,fuse-adrp-add,fuse-aes,lor,lse,neon,pan,pan-rwv,perfmon,ras,rcpc,rdm,sha2,uaops,use-postra-scheduler,v8.1a,v8.2a,v8a,vh") }, { str_lit("cortex-a57"), str_lit("a57,aes,balance-fp-ops,crc,crypto,el2vmsa,el3,enable-select-opt,fp-armv8,fuse-adrp-add,fuse-aes,fuse-literals,neon,perfmon,predictable-select-expensive,sha2,use-postra-scheduler,v8a") }, @@ -418,6 +443,7 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("cortex-a710"), str_lit("CONTEXTIDREL2,a710,addr-lsl-fast,altnzcv,alu-lsl-fast,am,bf16,bti,ccdp,ccidx,ccpp,cmp-bcc-fusion,complxnum,crc,dit,dotprod,el2vmsa,el3,enable-select-opt,ete,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-adrp-add,fuse-aes,i8mm,jsconv,lor,lse,lse2,mec,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,predres,ras,rcpc,rcpc-immo,rdm,rme,sb,sel2,specrestrict,ssbs,sve,sve2,sve2-bitperm,tlb-rmi,tracev8.4,trbe,uaops,use-postra-scheduler,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8a,v9a,vh") }, { str_lit("cortex-a715"), str_lit("CONTEXTIDREL2,a715,addr-lsl-fast,altnzcv,alu-lsl-fast,am,bf16,bti,ccdp,ccidx,ccpp,cmp-bcc-fusion,complxnum,crc,dit,dotprod,el2vmsa,el3,enable-select-opt,ete,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-adrp-add,fuse-aes,i8mm,jsconv,lor,lse,lse2,mec,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,predres,ras,rcpc,rcpc-immo,rdm,rme,sb,sel2,spe,specrestrict,ssbs,sve,sve2,sve2-bitperm,tlb-rmi,tracev8.4,trbe,uaops,use-postra-scheduler,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8a,v9a,vh") }, { str_lit("cortex-a72"), str_lit("a72,aes,crc,crypto,el2vmsa,el3,enable-select-opt,fp-armv8,fuse-adrp-add,fuse-aes,fuse-literals,neon,perfmon,predictable-select-expensive,sha2,v8a") }, + { str_lit("cortex-a720"), str_lit("CONTEXTIDREL2,a720,addr-lsl-fast,altnzcv,alu-lsl-fast,am,amvs,bf16,bti,ccdp,ccidx,ccpp,cmp-bcc-fusion,complxnum,crc,dit,dotprod,ecv,el2vmsa,el3,enable-select-opt,ete,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-adrp-add,fuse-aes,hcx,i8mm,jsconv,lor,lse,lse2,mec,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,predres,ras,rcpc,rcpc-immo,rdm,rme,sb,sel2,spe,spe-eef,specrestrict,ssbs,sve,sve2,sve2-bitperm,tlb-rmi,tracev8.4,trbe,uaops,use-postra-scheduler,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8.7a,v8a,v9.1a,v9.2a,v9a,vh,wfxt,xs") }, { str_lit("cortex-a73"), str_lit("a73,aes,crc,crypto,el2vmsa,el3,enable-select-opt,fp-armv8,fuse-adrp-add,fuse-aes,neon,perfmon,predictable-select-expensive,sha2,v8a") }, { str_lit("cortex-a75"), str_lit("CONTEXTIDREL2,a75,aes,ccpp,crc,crypto,dotprod,el2vmsa,el3,enable-select-opt,fp-armv8,fullfp16,fuse-adrp-add,fuse-aes,lor,lse,neon,pan,pan-rwv,perfmon,predictable-select-expensive,ras,rcpc,rdm,sha2,uaops,v8.1a,v8.2a,v8a,vh") }, { str_lit("cortex-a76"), str_lit("CONTEXTIDREL2,a76,addr-lsl-fast,aes,alu-lsl-fast,ccpp,crc,crypto,dotprod,el2vmsa,el3,enable-select-opt,fp-armv8,fullfp16,fuse-adrp-add,fuse-aes,lor,lse,neon,pan,pan-rwv,perfmon,predictable-select-expensive,ras,rcpc,rdm,sha2,ssbs,uaops,v8.1a,v8.2a,v8a,vh") }, @@ -430,6 +456,7 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("cortex-x1c"), str_lit("CONTEXTIDREL2,addr-lsl-fast,aes,alu-lsl-fast,ccpp,cmp-bcc-fusion,cortex-x1,crc,crypto,dotprod,el2vmsa,el3,enable-select-opt,flagm,fp-armv8,fullfp16,fuse-adrp-add,fuse-aes,lor,lse,lse2,neon,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,ras,rcpc,rcpc-immo,rdm,sha2,spe,ssbs,uaops,use-postra-scheduler,v8.1a,v8.2a,v8a,vh") }, { str_lit("cortex-x2"), str_lit("CONTEXTIDREL2,addr-lsl-fast,altnzcv,alu-lsl-fast,am,bf16,bti,ccdp,ccidx,ccpp,cmp-bcc-fusion,complxnum,cortex-x2,crc,dit,dotprod,el2vmsa,el3,enable-select-opt,ete,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-adrp-add,fuse-aes,i8mm,jsconv,lor,lse,lse2,mec,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,predres,ras,rcpc,rcpc-immo,rdm,rme,sb,sel2,specrestrict,ssbs,sve,sve2,sve2-bitperm,tlb-rmi,tracev8.4,trbe,uaops,use-postra-scheduler,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8a,v9a,vh") }, { str_lit("cortex-x3"), str_lit("CONTEXTIDREL2,addr-lsl-fast,altnzcv,alu-lsl-fast,am,bf16,bti,ccdp,ccidx,ccpp,complxnum,cortex-x3,crc,dit,dotprod,el2vmsa,el3,enable-select-opt,ete,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-adrp-add,fuse-aes,i8mm,jsconv,lor,lse,lse2,mec,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,predres,ras,rcpc,rcpc-immo,rdm,rme,sb,sel2,spe,specrestrict,ssbs,sve,sve2,sve2-bitperm,tlb-rmi,tracev8.4,trbe,uaops,use-postra-scheduler,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8a,v9a,vh") }, + { str_lit("cortex-x4"), str_lit("CONTEXTIDREL2,addr-lsl-fast,altnzcv,alu-lsl-fast,am,amvs,bf16,bti,ccdp,ccidx,ccpp,complxnum,cortex-x4,crc,dit,dotprod,ecv,el2vmsa,el3,enable-select-opt,ete,fgt,flagm,fp-armv8,fp16fml,fptoint,fullfp16,fuse-adrp-add,fuse-aes,hcx,i8mm,jsconv,lor,lse,lse2,mec,mpam,mte,neon,nv,pan,pan-rwv,pauth,perfmon,predictable-select-expensive,predres,ras,rcpc,rcpc-immo,rdm,rme,sb,sel2,spe,spe-eef,specrestrict,ssbs,sve,sve2,sve2-bitperm,tlb-rmi,tracev8.4,trbe,uaops,use-postra-scheduler,use-scalar-inc-vl,v8.1a,v8.2a,v8.3a,v8.4a,v8.5a,v8.6a,v8.7a,v8a,v9.1a,v9.2a,v9a,vh,wfxt,xs") }, { str_lit("cyclone"), str_lit("aes,alternate-sextload-cvt-f32-pattern,apple-a7,apple-a7-sysreg,arith-bcc-fusion,arith-cbz-fusion,crypto,disable-latency-sched-heuristic,el2vmsa,el3,fp-armv8,fuse-aes,fuse-crypto-eor,neon,perfmon,sha2,store-pair-suppress,v8a,zcm,zcz,zcz-fp-workaround,zcz-gp") }, { str_lit("exynos-m3"), str_lit("addr-lsl-fast,aes,alu-lsl-fast,crc,crypto,el2vmsa,el3,exynos-cheap-as-move,exynosm3,force-32bit-jump-tables,fp-armv8,fuse-address,fuse-adrp-add,fuse-aes,fuse-csel,fuse-literals,neon,perfmon,predictable-select-expensive,sha2,store-pair-suppress,use-postra-scheduler,v8a") }, { str_lit("exynos-m4"), str_lit("CONTEXTIDREL2,addr-lsl-fast,aes,alu-lsl-fast,arith-bcc-fusion,arith-cbz-fusion,ccpp,crc,crypto,dotprod,el2vmsa,el3,exynos-cheap-as-move,exynosm4,force-32bit-jump-tables,fp-armv8,fullfp16,fuse-address,fuse-adrp-add,fuse-aes,fuse-arith-logic,fuse-csel,fuse-literals,lor,lse,neon,pan,pan-rwv,perfmon,ras,rdm,sha2,store-pair-suppress,uaops,use-postra-scheduler,v8.1a,v8.2a,v8a,vh,zcz,zcz-gp") }, @@ -459,4 +486,31 @@ gb_global MicroarchFeatureList microarch_features_list[] = { { str_lit("bleeding-edge"), str_lit("atomics,bulk-memory,mutable-globals,nontrapping-fptoint,sign-ext,simd128,tail-call") }, { str_lit("generic"), str_lit("mutable-globals,sign-ext") }, { str_lit("mvp"), str_lit("") }, -}; \ No newline at end of file + // TargetArch_riscv64: + { str_lit("generic"), str_lit("64bit") }, + { str_lit("generic-rv32"), str_lit("32bit") }, + { str_lit("generic-rv64"), str_lit("64bit") }, + { str_lit("rocket"), str_lit("") }, + { str_lit("rocket-rv32"), str_lit("32bit,zicsr,zifencei") }, + { str_lit("rocket-rv64"), str_lit("64bit,zicsr,zifencei") }, + { str_lit("sifive-7-series"), str_lit("no-default-unroll,short-forward-branch-opt,sifive7") }, + { str_lit("sifive-e20"), str_lit("32bit,c,m,zicsr,zifencei") }, + { str_lit("sifive-e21"), str_lit("32bit,a,c,m,zicsr,zifencei") }, + { str_lit("sifive-e24"), str_lit("32bit,a,c,f,m,zicsr,zifencei") }, + { str_lit("sifive-e31"), str_lit("32bit,a,c,m,zicsr,zifencei") }, + { str_lit("sifive-e34"), str_lit("32bit,a,c,f,m,zicsr,zifencei") }, + { str_lit("sifive-e76"), str_lit("32bit,a,c,f,m,no-default-unroll,short-forward-branch-opt,sifive7,zicsr,zifencei") }, + { str_lit("sifive-p450"), str_lit("64bit,a,auipc-addi-fusion,c,conditional-cmv-fusion,d,f,fast-unaligned-access,lui-addi-fusion,m,no-default-unroll,za64rs,zba,zbb,zbs,zfhmin,zic64b,zicbom,zicbop,zicboz,ziccamoa,ziccif,zicclsm,ziccrse,zicsr,zifencei,zihintntl,zihintpause,zihpm") }, + { str_lit("sifive-p670"), str_lit("64bit,a,auipc-addi-fusion,c,conditional-cmv-fusion,d,f,fast-unaligned-access,lui-addi-fusion,m,no-default-unroll,v,za64rs,zba,zbb,zbs,zfhmin,zic64b,zicbom,zicbop,zicboz,ziccamoa,ziccif,zicclsm,ziccrse,zicsr,zifencei,zihintntl,zihintpause,zihpm,zvbb,zvbc,zve32f,zve32x,zve64d,zve64f,zve64x,zvkb,zvkg,zvkn,zvknc,zvkned,zvkng,zvknhb,zvks,zvksc,zvksed,zvksg,zvksh,zvkt,zvl128b,zvl32b,zvl64b") }, + { str_lit("sifive-s21"), str_lit("64bit,a,c,m,zicsr,zifencei") }, + { str_lit("sifive-s51"), str_lit("64bit,a,c,m,zicsr,zifencei") }, + { str_lit("sifive-s54"), str_lit("64bit,a,c,d,f,m,zicsr,zifencei") }, + { str_lit("sifive-s76"), str_lit("64bit,a,c,d,f,m,no-default-unroll,short-forward-branch-opt,sifive7,zicsr,zifencei,zihintpause") }, + { str_lit("sifive-u54"), str_lit("64bit,a,c,d,f,m,zicsr,zifencei") }, + { str_lit("sifive-u74"), str_lit("64bit,a,c,d,f,m,no-default-unroll,short-forward-branch-opt,sifive7,zicsr,zifencei") }, + { str_lit("sifive-x280"), str_lit("64bit,a,c,d,dlen-factor-2,f,m,no-default-unroll,short-forward-branch-opt,sifive7,v,zba,zbb,zfh,zfhmin,zicsr,zifencei,zve32f,zve32x,zve64d,zve64f,zve64x,zvfh,zvfhmin,zvl128b,zvl256b,zvl32b,zvl512b,zvl64b") }, + { str_lit("syntacore-scr1-base"), str_lit("32bit,c,no-default-unroll,zicsr,zifencei") }, + { str_lit("syntacore-scr1-max"), str_lit("32bit,c,m,no-default-unroll,zicsr,zifencei") }, + { str_lit("veyron-v1"), str_lit("64bit,a,auipc-addi-fusion,c,d,f,ld-add-fusion,lui-addi-fusion,m,shifted-zextw-fusion,ventana-veyron,xventanacondops,zba,zbb,zbc,zbs,zexth-fusion,zextw-fusion,zicbom,zicbop,zicboz,zicntr,zicsr,zifencei,zihintpause,zihpm") }, + { str_lit("xiangshan-nanhu"), str_lit("64bit,a,c,d,f,m,svinval,zba,zbb,zbc,zbkb,zbkc,zbkx,zbs,zicbom,zicboz,zicsr,zifencei,zkn,zknd,zkne,zknh,zksed,zksh") }, +}; diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index e5282f63e..1c4b88101 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -155,6 +155,11 @@ gb_internal bool does_require_msgSend_stret(Type *return_type) { return false; } + // No objc here so this doesn't matter, right? + if (build_context.metrics.arch == TargetArch_riscv64) { + return false; + } + // if (build_context.metrics.arch == TargetArch_arm32) { // i64 struct_limit = type_size_of(t_uintptr); // // NOTE(bill): This is technically wrong diff --git a/src/checker.cpp b/src/checker.cpp index c8eaf0acc..b24a7afdb 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -1039,6 +1039,7 @@ gb_internal void init_universal(void) { {"arm64", TargetArch_arm64}, {"wasm32", TargetArch_wasm32}, {"wasm64p32", TargetArch_wasm64p32}, + {"riscv64", TargetArch_riscv64}, }; auto fields = add_global_enum_type(str_lit("Odin_Arch_Type"), values, gb_count_of(values)); diff --git a/src/linker.cpp b/src/linker.cpp index 046e72d0e..faca28932 100644 --- a/src/linker.cpp +++ b/src/linker.cpp @@ -388,6 +388,12 @@ gb_internal i32 linker_stage(LinkerData *gen) { } else { timings_start_section(timings, str_lit("ld-link")); + // Link using `clang`, unless overridden by `ODIN_CLANG_PATH` environment variable. + const char* clang_path = gb_get_env("ODIN_CLANG_PATH", permanent_allocator()); + if (clang_path == NULL) { + clang_path = "clang"; + } + // NOTE(vassvik): get cwd, for used for local shared libs linking, since those have to be relative to the exe char cwd[256]; #if !defined(GB_SYSTEM_WINDOWS) @@ -458,7 +464,20 @@ gb_internal i32 linker_stage(LinkerData *gen) { } #endif // GB_ARCH_*_BIT - if (is_osx) { + if (build_context.metrics.arch == TargetArch_riscv64) { + result = system_exec_command_line_app("clang", + "%s \"%.*s\" " + "-c -o \"%.*s\" " + "-target %.*s -march=rv64gc " + "%.*s " + "", + clang_path, + LIT(asm_file), + LIT(obj_file), + LIT(build_context.metrics.target_triplet), + LIT(build_context.extra_assembler_flags) + ); + } else if (is_osx) { // `as` comes with MacOS. result = system_exec_command_line_app("as", "as \"%.*s\" " @@ -592,7 +611,7 @@ gb_internal i32 linker_stage(LinkerData *gen) { link_settings = gb_string_appendc(link_settings, "-Wl,-fini,'_odin_exit_point' "); } - } else if (build_context.metrics.os != TargetOs_openbsd && build_context.metrics.os != TargetOs_haiku) { + } else if (build_context.metrics.os != TargetOs_openbsd && build_context.metrics.os != TargetOs_haiku && build_context.metrics.arch != TargetArch_riscv64) { // OpenBSD and Haiku default to PIE executable. do not pass -no-pie for it. link_settings = gb_string_appendc(link_settings, "-no-pie "); } @@ -635,12 +654,6 @@ gb_internal i32 linker_stage(LinkerData *gen) { } } - // Link using `clang`, unless overridden by `ODIN_CLANG_PATH` environment variable. - const char* clang_path = gb_get_env("ODIN_CLANG_PATH", permanent_allocator()); - if (clang_path == NULL) { - clang_path = "clang"; - } - gbString link_command_line = gb_string_make(heap_allocator(), clang_path); defer (gb_string_free(link_command_line)); diff --git a/src/llvm_abi.cpp b/src/llvm_abi.cpp index c21cd0a46..0837e40bf 100644 --- a/src/llvm_abi.cpp +++ b/src/llvm_abi.cpp @@ -332,7 +332,8 @@ gb_internal i64 lb_alignof(LLVMTypeRef type) { } -#define LB_ABI_INFO(name) lbFunctionType *name(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count, LLVMTypeRef return_type, bool return_is_defined, bool return_is_tuple, ProcCallingConvention calling_convention, Type *original_type) +#define LB_ABI_INFO(name) lbFunctionType *name(lbModule *m, LLVMTypeRef *arg_types, unsigned arg_count, LLVMTypeRef return_type, bool return_is_defined, bool return_is_tuple, ProcCallingConvention calling_convention, Type *original_type) +#define LB_ABI_INFO_CTX(name) lbFunctionType *name(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count, LLVMTypeRef return_type, bool return_is_defined, bool return_is_tuple, ProcCallingConvention calling_convention, Type *original_type) typedef LB_ABI_INFO(lbAbiInfoType); #define LB_ABI_COMPUTE_RETURN_TYPE(name) lbArgType name(lbFunctionType *ft, LLVMContextRef c, LLVMTypeRef return_type, bool return_is_defined, bool return_is_tuple) @@ -379,7 +380,7 @@ namespace lbAbi386 { gb_internal Array compute_arg_types(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count); gb_internal LB_ABI_COMPUTE_RETURN_TYPE(compute_return_type); - gb_internal LB_ABI_INFO(abi_info) { + gb_internal LB_ABI_INFO_CTX(abi_info) { lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->args = compute_arg_types(c, arg_types, arg_count); @@ -460,7 +461,7 @@ namespace lbAbiAmd64Win64 { gb_internal Array compute_arg_types(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count); gb_internal LB_ABI_COMPUTE_RETURN_TYPE(compute_return_type); - gb_internal LB_ABI_INFO(abi_info) { + gb_internal LB_ABI_INFO_CTX(abi_info) { lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->args = compute_arg_types(c, arg_types, arg_count); @@ -570,7 +571,7 @@ namespace lbAbiAmd64SysV { gb_internal Array classify(LLVMTypeRef t); gb_internal LLVMTypeRef llreg(LLVMContextRef c, Array const ®_classes, LLVMTypeRef type); - gb_internal LB_ABI_INFO(abi_info) { + gb_internal LB_ABI_INFO_CTX(abi_info) { lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->calling_convention = calling_convention; @@ -1008,7 +1009,7 @@ namespace lbAbiArm64 { gb_internal LB_ABI_COMPUTE_RETURN_TYPE(compute_return_type); gb_internal bool is_homogenous_aggregate(LLVMContextRef c, LLVMTypeRef type, LLVMTypeRef *base_type_, unsigned *member_count_); - gb_internal LB_ABI_INFO(abi_info) { + gb_internal LB_ABI_INFO_CTX(abi_info) { lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->args = compute_arg_types(c, arg_types, arg_count); @@ -1242,7 +1243,7 @@ namespace lbAbiWasm { enum {MAX_DIRECT_STRUCT_SIZE = 32}; - gb_internal LB_ABI_INFO(abi_info) { + gb_internal LB_ABI_INFO_CTX(abi_info) { lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->calling_convention = calling_convention; @@ -1407,7 +1408,7 @@ namespace lbAbiArm32 { gb_internal Array compute_arg_types(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count, ProcCallingConvention calling_convention); gb_internal lbArgType compute_return_type(LLVMContextRef c, LLVMTypeRef return_type, bool return_is_defined); - gb_internal LB_ABI_INFO(abi_info) { + gb_internal LB_ABI_INFO_CTX(abi_info) { lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->args = compute_arg_types(c, arg_types, arg_count, calling_convention); @@ -1485,8 +1486,256 @@ namespace lbAbiArm32 { } }; +namespace lbAbiRiscv64 { + + gb_internal bool is_register(LLVMTypeRef type) { + LLVMTypeKind kind = LLVMGetTypeKind(type); + switch (kind) { + case LLVMIntegerTypeKind: + case LLVMHalfTypeKind: + case LLVMFloatTypeKind: + case LLVMDoubleTypeKind: + case LLVMPointerTypeKind: + return true; + } + return false; + } + + gb_internal bool is_float(LLVMTypeRef type) { + LLVMTypeKind kind = LLVMGetTypeKind(type); + switch (kind) { + case LLVMHalfTypeKind: + case LLVMFloatTypeKind: + case LLVMDoubleTypeKind: + return true; + default: + return false; + } + } + + gb_internal lbArgType non_struct(LLVMContextRef c, LLVMTypeRef type) { + LLVMAttributeRef attr = nullptr; + LLVMTypeRef i1 = LLVMInt1TypeInContext(c); + if (type == i1) { + attr = lb_create_enum_attribute(c, "zeroext"); + } + return lb_arg_type_direct(type, nullptr, nullptr, attr); + } + + gb_internal void flatten(lbModule *m, Array *fields, LLVMTypeRef type, bool with_padding) { + LLVMTypeKind kind = LLVMGetTypeKind(type); + switch (kind) { + case LLVMStructTypeKind: { + if (LLVMIsPackedStruct(type)) { + array_add(fields, type); + break; + } + + if (!with_padding) { + auto field_remapping = map_get(&m->struct_field_remapping, cast(void *)type); + if (field_remapping) { + auto remap = *field_remapping; + for_array(i, remap) { + flatten(m, fields, LLVMStructGetTypeAtIndex(type, remap[i]), with_padding); + } + break; + } else { + debugf("no field mapping for type: %s\n", LLVMPrintTypeToString(type)); + } + } + + unsigned elem_count = LLVMCountStructElementTypes(type); + for (unsigned i = 0; i < elem_count; i += 1) { + flatten(m, fields, LLVMStructGetTypeAtIndex(type, i), with_padding); + } + break; + } + case LLVMArrayTypeKind: { + unsigned len = LLVMGetArrayLength(type); + LLVMTypeRef elem = OdinLLVMGetArrayElementType(type); + for (unsigned i = 0; i < len; i += 1) { + flatten(m, fields, elem, with_padding); + } + break; + } + default: + array_add(fields, type); + } + } + + gb_internal lbArgType compute_arg_type(lbModule *m, LLVMTypeRef type, int *gprs_left, int *fprs_left, Type *odin_type) { + LLVMContextRef c = m->ctx; + + int xlen = 8; // 8 byte int register size for riscv64. + + // NOTE: we are requiring both of these to be enabled so we can just hard-code 8. + // int flen = 0; + // if (check_target_feature_is_enabled(str_lit("d"), nullptr)) { + // flen = 8; // Double precision floats are enabled. + // } else if (check_target_feature_is_enabled(str_lit("f"), nullptr)) { + // flen = 4; // Single precision floats are enabled. + // } + int flen = 8; + + LLVMTypeKind kind = LLVMGetTypeKind(type); + i64 size = lb_sizeof(type); + + if (size == 0) { + return lb_arg_type_direct(type, LLVMStructTypeInContext(c, nullptr, 0, false), nullptr, nullptr); + } + + LLVMTypeRef orig_type = type; + + // Flatten down the type so it is easier to check all the ABI conditions. + // Note that we also need to remove all implicit padding fields Odin adds so we keep ABI + // compatibility for struct declarations. + if (kind == LLVMStructTypeKind && size <= gb_max(2*xlen, 2*flen)) { + Array fields = array_make(temporary_allocator(), 0, LLVMCountStructElementTypes(type)); + flatten(m, &fields, type, false); + + if (fields.count == 1) { + type = fields[0]; + } else { + type = LLVMStructTypeInContext(c, fields.data, cast(unsigned)fields.count, false); + } + + kind = LLVMGetTypeKind(type); + size = lb_sizeof(type); + GB_ASSERT_MSG(size == lb_sizeof(orig_type), "flattened: %s of size %d, original: %s of size %d", LLVMPrintTypeToString(type), size, LLVMPrintTypeToString(orig_type), lb_sizeof(orig_type)); + } + + if (is_float(type) && size <= flen && *fprs_left >= 1) { + *fprs_left -= 1; + return non_struct(c, orig_type); + } + + if (kind == LLVMStructTypeKind && size <= 2*flen) { + unsigned elem_count = LLVMCountStructElementTypes(type); + if (elem_count == 2) { + LLVMTypeRef ty1 = LLVMStructGetTypeAtIndex(type, 0); + i64 ty1s = lb_sizeof(ty1); + LLVMTypeRef ty2 = LLVMStructGetTypeAtIndex(type, 1); + i64 ty2s = lb_sizeof(ty2); + + if (is_float(ty1) && is_float(ty2) && ty1s <= flen && ty2s <= flen && *fprs_left >= 2) { + *fprs_left -= 2; + return lb_arg_type_direct(orig_type, type, nullptr, nullptr); + } + + if (is_float(ty1) && is_register(ty2) && ty1s <= flen && ty2s <= xlen && *fprs_left >= 1 && *gprs_left >= 1) { + *fprs_left -= 1; + *gprs_left -= 1; + return lb_arg_type_direct(orig_type, type, nullptr, nullptr); + } + + if (is_register(ty1) && is_float(ty2) && ty1s <= xlen && ty2s <= flen && *gprs_left >= 1 && *fprs_left >= 1) { + *fprs_left -= 1; + *gprs_left -= 1; + return lb_arg_type_direct(orig_type, type, nullptr, nullptr); + } + } + } + + // At this point all the cases for floating point registers are exhausted, fit it into + // integer registers or the stack. + // LLVM automatically handles putting args on the stack so we don't check the amount of registers that are left here. + + if (size <= xlen) { + *gprs_left -= 1; + if (is_register(type)) { + return non_struct(c, orig_type); + } else { + return lb_arg_type_direct(orig_type, LLVMIntTypeInContext(c, cast(unsigned)(size*8)), nullptr, nullptr); + } + } else if (size <= 2*xlen) { + LLVMTypeRef *fields = gb_alloc_array(temporary_allocator(), LLVMTypeRef, 2); + fields[0] = LLVMIntTypeInContext(c, cast(unsigned)(xlen*8)); + fields[1] = LLVMIntTypeInContext(c, cast(unsigned)((size-xlen)*8)); + + *gprs_left -= 2; + return lb_arg_type_direct(orig_type, LLVMStructTypeInContext(c, fields, 2, false), nullptr, nullptr); + } else { + return lb_arg_type_indirect(orig_type, nullptr); + } + } + + gb_internal Array compute_arg_types(lbModule *m, LLVMTypeRef *arg_types, unsigned arg_count, ProcCallingConvention calling_convention, Type *odin_type, int *gprs, int *fprs) { + auto args = array_make(lb_function_type_args_allocator(), arg_count); + + for (unsigned i = 0; i < arg_count; i++) { + LLVMTypeRef type = arg_types[i]; + args[i] = compute_arg_type(m, type, gprs, fprs, odin_type); + } + + return args; + } + + gb_internal lbArgType compute_return_type(lbFunctionType *ft, lbModule *m, LLVMTypeRef return_type, bool return_is_defined, bool return_is_tuple, Type *odin_type, int *agprs) { + LLVMContextRef c = m->ctx; + + if (!return_is_defined) { + return lb_arg_type_direct(LLVMVoidTypeInContext(c)); + } + + // There are two registers for return types. + int gprs = 2; + int fprs = 2; + lbArgType ret = compute_arg_type(m, return_type, &gprs, &fprs, odin_type); + + // Return didn't fit into the return registers, so caller allocates and it is returned via + // an out-pointer. + if (ret.kind == lbArg_Indirect) { + + // Transform multiple return into out pointers if possible. + if (return_is_tuple) { + if (lb_is_type_kind(return_type, LLVMStructTypeKind)) { + int field_count = cast(int)LLVMCountStructElementTypes(return_type); + if (field_count > 1 && field_count <= *agprs) { + ft->original_arg_count = ft->args.count; + ft->multiple_return_original_type = return_type; + + for (int i = 0; i < field_count-1; i++) { + LLVMTypeRef field_type = LLVMStructGetTypeAtIndex(return_type, i); + LLVMTypeRef field_pointer_type = LLVMPointerType(field_type, 0); + lbArgType ret_partial = lb_arg_type_direct(field_pointer_type); + array_add(&ft->args, ret_partial); + *agprs -= 1; + } + GB_ASSERT(*agprs >= 0); + + // override the return type for the last field + LLVMTypeRef new_return_type = LLVMStructGetTypeAtIndex(return_type, field_count-1); + return compute_return_type(ft, m, new_return_type, true, false, odin_type, agprs); + } + } + } + + LLVMAttributeRef attr = lb_create_enum_attribute_with_type(c, "sret", ret.type); + return lb_arg_type_indirect(ret.type, attr); + } + + return ret; + } + + gb_internal LB_ABI_INFO(abi_info) { + lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); + ft->ctx = m->ctx; + ft->calling_convention = calling_convention; + + int gprs = 8; + int fprs = 8; + + ft->args = compute_arg_types(m, arg_types, arg_count, calling_convention, original_type, &gprs, &fprs); + ft->ret = compute_return_type(ft, m, return_type, return_is_defined, return_is_tuple, original_type, &gprs); + + return ft; + } +} + gb_internal LB_ABI_INFO(lb_get_abi_info_internal) { + LLVMContextRef c = m->ctx; + switch (calling_convention) { case ProcCC_None: case ProcCC_InlineAsm: @@ -1534,6 +1783,8 @@ gb_internal LB_ABI_INFO(lb_get_abi_info_internal) { return lbAbiWasm::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); case TargetArch_wasm64p32: return lbAbiWasm::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + case TargetArch_riscv64: + return lbAbiRiscv64::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); } GB_PANIC("Unsupported ABI"); @@ -1543,7 +1794,7 @@ gb_internal LB_ABI_INFO(lb_get_abi_info_internal) { gb_internal LB_ABI_INFO(lb_get_abi_info) { lbFunctionType *ft = lb_get_abi_info_internal( - c, + m, arg_types, arg_count, return_type, return_is_defined, ALLOW_SPLIT_MULTI_RETURNS && return_is_tuple && is_calling_convention_odin(calling_convention), @@ -1555,7 +1806,7 @@ gb_internal LB_ABI_INFO(lb_get_abi_info) { // This is to make it consistent when and how it is handled if (calling_convention == ProcCC_Odin) { // append the `context` pointer - lbArgType context_param = lb_arg_type_direct(LLVMPointerType(LLVMInt8TypeInContext(c), 0)); + lbArgType context_param = lb_arg_type_direct(LLVMPointerType(LLVMInt8TypeInContext(m->ctx), 0)); array_add(&ft->args, context_param); } diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 72ba12516..f852636a6 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -40,7 +40,10 @@ String get_default_microarchitecture() { default_march = str_lit("x86-64-v2"); } } + } else if (build_context.metrics.arch == TargetArch_riscv64) { + default_march = str_lit("generic-rv64"); } + return default_march; } @@ -65,13 +68,33 @@ gb_internal String get_default_features() { } String microarch = get_final_microarchitecture(); + + // NOTE(laytan): for riscv64 to work properly with Odin, we need to enforce some features. + // and we also overwrite the generic target to include more features so we don't default to + // a potato feature set. + if (bc->metrics.arch == TargetArch_riscv64) { + if (microarch == str_lit("generic-rv64")) { + // This is what clang does by default (on -march=rv64gc for General Computing), seems good to also default to. + String features = str_lit("64bit,a,c,d,f,m,relax,zicsr,zifencei"); + + // Update the features string so LLVM uses it later. + if (bc->target_features_string.len > 0) { + bc->target_features_string = concatenate3_strings(permanent_allocator(), features, str_lit(","), bc->target_features_string); + } else { + bc->target_features_string = features; + } + + return features; + } + } + for (int i = off; i < off+target_microarch_counts[bc->metrics.arch]; i += 1) { if (microarch_features_list[i].microarch == microarch) { return microarch_features_list[i].features; } } - GB_PANIC("unknown microarch"); + GB_PANIC("unknown microarch: %.*s", LIT(microarch)); return {}; } @@ -3030,6 +3053,12 @@ gb_internal bool lb_generate_code(lbGenerator *gen) { // Always use PIC for OpenBSD and Haiku: they default to PIE reloc_mode = LLVMRelocPIC; } + + if (build_context.metrics.arch == TargetArch_riscv64) { + // NOTE(laytan): didn't seem to work without this. + reloc_mode = LLVMRelocPIC; + } + break; case RelocMode_Static: reloc_mode = LLVMRelocStatic; diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index b5338297e..842a1cbc8 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -1787,7 +1787,7 @@ gb_internal LLVMTypeRef lb_type_internal_for_procedures_raw(lbModule *m, Type *t } } GB_ASSERT(param_index == param_count); - lbFunctionType *ft = lb_get_abi_info(m->ctx, params, param_count, ret, ret != nullptr, return_is_tuple, type->Proc.calling_convention, type); + lbFunctionType *ft = lb_get_abi_info(m, params, param_count, ret, ret != nullptr, return_is_tuple, type->Proc.calling_convention, type); { for_array(j, ft->args) { auto arg = ft->args[j]; @@ -2114,6 +2114,12 @@ gb_internal LLVMTypeRef lb_type_internal(lbModule *m, Type *type) { llvm_type = LLVMStructCreateNamed(ctx, name); map_set(&m->types, type, llvm_type); lb_clone_struct_type(llvm_type, lb_type(m, base)); + + if (base->kind == Type_Struct) { + map_set(&m->struct_field_remapping, cast(void *)llvm_type, lb_get_struct_remapping(m, base)); + map_set(&m->struct_field_remapping, cast(void *)type, lb_get_struct_remapping(m, base)); + } + return llvm_type; } } diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index 3326b4041..e850d3364 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -2877,6 +2877,31 @@ gb_internal lbValue lb_build_builtin_proc(lbProcedure *p, Ast *expr, TypeAndValu LLVMValueRef inline_asm = nullptr; switch (build_context.metrics.arch) { + case TargetArch_riscv64: + { + GB_ASSERT(arg_count <= 7); + + char asm_string[] = "ecall"; + gbString constraints = gb_string_make(heap_allocator(), "={a0}"); + for (unsigned i = 0; i < arg_count; i++) { + constraints = gb_string_appendc(constraints, ",{"); + static char const *regs[] = { + "a7", + "a0", + "a1", + "a2", + "a3", + "a4", + "a5", + "a6" + }; + constraints = gb_string_appendc(constraints, regs[i]); + constraints = gb_string_appendc(constraints, "}"); + } + + inline_asm = llvm_get_inline_asm(func_type, make_string_c(asm_string), make_string_c(constraints)); + } + break; case TargetArch_amd64: { GB_ASSERT(arg_count <= 7); diff --git a/src/main.cpp b/src/main.cpp index 77758b929..5131bdc21 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3245,6 +3245,15 @@ int main(int arg_count, char const **arg_ptr) { } } + // NOTE(laytan): on riscv64 we want to enforce some features. + if (build_context.metrics.arch == TargetArch_riscv64) { + String disabled; + if (!check_target_feature_is_enabled(str_lit("64bit,f,d,m"), &disabled)) { // 64bit, floats, doubles, integer multiplication. + gb_printf_err("missing required target feature: \"%.*s\", enable it by setting a different -microarch or explicitly adding it through -target-features\n", LIT(disabled)); + gb_exit(1); + } + } + if (build_context.show_debug_messages) { debugf("Selected microarch: %.*s\n", LIT(march)); debugf("Default microarch features: %.*s\n", LIT(default_features)); From 5b22bfa2b7e599b8107ecdb7aa50117a62ba1292 Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Tue, 20 Aug 2024 14:06:25 +0200 Subject: [PATCH 28/43] unify LB_ABI_INFO and LB_ABI_INFO_CTX --- src/llvm_abi.cpp | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/llvm_abi.cpp b/src/llvm_abi.cpp index 0837e40bf..aa5c4dc60 100644 --- a/src/llvm_abi.cpp +++ b/src/llvm_abi.cpp @@ -333,7 +333,6 @@ gb_internal i64 lb_alignof(LLVMTypeRef type) { #define LB_ABI_INFO(name) lbFunctionType *name(lbModule *m, LLVMTypeRef *arg_types, unsigned arg_count, LLVMTypeRef return_type, bool return_is_defined, bool return_is_tuple, ProcCallingConvention calling_convention, Type *original_type) -#define LB_ABI_INFO_CTX(name) lbFunctionType *name(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count, LLVMTypeRef return_type, bool return_is_defined, bool return_is_tuple, ProcCallingConvention calling_convention, Type *original_type) typedef LB_ABI_INFO(lbAbiInfoType); #define LB_ABI_COMPUTE_RETURN_TYPE(name) lbArgType name(lbFunctionType *ft, LLVMContextRef c, LLVMTypeRef return_type, bool return_is_defined, bool return_is_tuple) @@ -380,7 +379,8 @@ namespace lbAbi386 { gb_internal Array compute_arg_types(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count); gb_internal LB_ABI_COMPUTE_RETURN_TYPE(compute_return_type); - gb_internal LB_ABI_INFO_CTX(abi_info) { + gb_internal LB_ABI_INFO(abi_info) { + LLVMContextRef c = m->ctx; lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->args = compute_arg_types(c, arg_types, arg_count); @@ -461,7 +461,8 @@ namespace lbAbiAmd64Win64 { gb_internal Array compute_arg_types(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count); gb_internal LB_ABI_COMPUTE_RETURN_TYPE(compute_return_type); - gb_internal LB_ABI_INFO_CTX(abi_info) { + gb_internal LB_ABI_INFO(abi_info) { + LLVMContextRef c = m->ctx; lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->args = compute_arg_types(c, arg_types, arg_count); @@ -571,7 +572,8 @@ namespace lbAbiAmd64SysV { gb_internal Array classify(LLVMTypeRef t); gb_internal LLVMTypeRef llreg(LLVMContextRef c, Array const ®_classes, LLVMTypeRef type); - gb_internal LB_ABI_INFO_CTX(abi_info) { + gb_internal LB_ABI_INFO(abi_info) { + LLVMContextRef c = m->ctx; lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->calling_convention = calling_convention; @@ -1009,7 +1011,8 @@ namespace lbAbiArm64 { gb_internal LB_ABI_COMPUTE_RETURN_TYPE(compute_return_type); gb_internal bool is_homogenous_aggregate(LLVMContextRef c, LLVMTypeRef type, LLVMTypeRef *base_type_, unsigned *member_count_); - gb_internal LB_ABI_INFO_CTX(abi_info) { + gb_internal LB_ABI_INFO(abi_info) { + LLVMContextRef c = m->ctx; lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->args = compute_arg_types(c, arg_types, arg_count); @@ -1243,7 +1246,8 @@ namespace lbAbiWasm { enum {MAX_DIRECT_STRUCT_SIZE = 32}; - gb_internal LB_ABI_INFO_CTX(abi_info) { + gb_internal LB_ABI_INFO(abi_info) { + LLVMContextRef c = m->ctx; lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->calling_convention = calling_convention; @@ -1408,7 +1412,8 @@ namespace lbAbiArm32 { gb_internal Array compute_arg_types(LLVMContextRef c, LLVMTypeRef *arg_types, unsigned arg_count, ProcCallingConvention calling_convention); gb_internal lbArgType compute_return_type(LLVMContextRef c, LLVMTypeRef return_type, bool return_is_defined); - gb_internal LB_ABI_INFO_CTX(abi_info) { + gb_internal LB_ABI_INFO(abi_info) { + LLVMContextRef c = m->ctx; lbFunctionType *ft = gb_alloc_item(permanent_allocator(), lbFunctionType); ft->ctx = c; ft->args = compute_arg_types(c, arg_types, arg_count, calling_convention); @@ -1756,33 +1761,33 @@ gb_internal LB_ABI_INFO(lb_get_abi_info_internal) { } case ProcCC_Win64: GB_ASSERT(build_context.metrics.arch == TargetArch_amd64); - return lbAbiAmd64Win64::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiAmd64Win64::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); case ProcCC_SysV: GB_ASSERT(build_context.metrics.arch == TargetArch_amd64); - return lbAbiAmd64SysV::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiAmd64SysV::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); } switch (build_context.metrics.arch) { case TargetArch_amd64: if (build_context.metrics.os == TargetOs_windows) { - return lbAbiAmd64Win64::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiAmd64Win64::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); } else if (build_context.metrics.abi == TargetABI_Win64) { - return lbAbiAmd64Win64::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiAmd64Win64::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); } else if (build_context.metrics.abi == TargetABI_SysV) { - return lbAbiAmd64SysV::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiAmd64SysV::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); } else { - return lbAbiAmd64SysV::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiAmd64SysV::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); } case TargetArch_i386: - return lbAbi386::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbi386::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); case TargetArch_arm32: - return lbAbiArm32::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiArm32::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); case TargetArch_arm64: - return lbAbiArm64::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiArm64::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); case TargetArch_wasm32: - return lbAbiWasm::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiWasm::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); case TargetArch_wasm64p32: - return lbAbiWasm::abi_info(c, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); + return lbAbiWasm::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); case TargetArch_riscv64: return lbAbiRiscv64::abi_info(m, arg_types, arg_count, return_type, return_is_defined, return_is_tuple, calling_convention, original_type); } From 6b4b0cea5dd0b451a283f919e36fa6c060f2f236 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 20 Aug 2024 22:12:47 +0200 Subject: [PATCH 29/43] Add table-driven (in)definite article to some errors. --- src/check_expr.cpp | 22 +++++++++++++++------- src/error.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index f4d5cc5a4..2540230df 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -1071,16 +1071,19 @@ gb_internal void check_assignment(CheckerContext *c, Operand *operand, Type *typ return; } + // Grab definite or indefinite article matching `context_name`, or "" if not found. + String article = error_article(context_name); + if (is_type_untyped(operand->type)) { Type *target_type = type; if (type == nullptr || is_type_any(type)) { if (type == nullptr && is_type_untyped_uninit(operand->type)) { - error(operand->expr, "Use of --- in %.*s", LIT(context_name)); + error(operand->expr, "Use of --- in %.*s%.*s", LIT(article), LIT(context_name)); operand->mode = Addressing_Invalid; return; } if (type == nullptr && is_type_untyped_nil(operand->type)) { - error(operand->expr, "Use of untyped nil in %.*s", LIT(context_name)); + error(operand->expr, "Use of untyped nil in %.*s%.*s", LIT(article), LIT(context_name)); operand->mode = Addressing_Invalid; return; } @@ -1135,9 +1138,10 @@ gb_internal void check_assignment(CheckerContext *c, Operand *operand, Type *typ // TODO(bill): is this a good enough error message? error(operand->expr, - "Cannot assign overloaded procedure group '%s' to '%s' in %.*s", + "Cannot assign overloaded procedure group '%s' to '%s' in %.*s%.*s", expr_str, op_type_str, + LIT(article), LIT(context_name)); operand->mode = Addressing_Invalid; } @@ -1163,20 +1167,23 @@ gb_internal void check_assignment(CheckerContext *c, Operand *operand, Type *typ switch (operand->mode) { case Addressing_Builtin: error(operand->expr, - "Cannot assign built-in procedure '%s' in %.*s", + "Cannot assign built-in procedure '%s' to %.*s%.*s", expr_str, + LIT(article), LIT(context_name)); break; case Addressing_Type: if (is_type_polymorphic(operand->type)) { error(operand->expr, - "Cannot assign '%s' which is a polymorphic type in %.*s", + "Cannot assign '%s' — a polymorphic type — to %.*s%.*s", op_type_str, + LIT(article), LIT(context_name)); } else { error(operand->expr, - "Cannot assign '%s' which is a type in %.*s", + "Cannot assign '%s' — a type — to %.*s%.*s", op_type_str, + LIT(article), LIT(context_name)); } break; @@ -1203,10 +1210,11 @@ gb_internal void check_assignment(CheckerContext *c, Operand *operand, Type *typ ERROR_BLOCK(); error(operand->expr, - "Cannot assign value '%s' of type '%s%s' to '%s%s' in %.*s", + "Cannot assign value '%s' of type '%s%s' to '%s%s' in %.*s%.*s", expr_str, op_type_str, op_type_extra, type_str, type_extra, + LIT(article), LIT(context_name)); check_assignment_error_suggestion(c, operand, type); diff --git a/src/error.cpp b/src/error.cpp index f95123f15..1492b00c7 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -820,6 +820,35 @@ gb_internal int error_value_cmp(void const *a, void const *b) { return token_pos_cmp(x->pos, y->pos); } +gb_global String error_article_table[][2] = { + {str_lit("a "), str_lit("bit_set literal")}, + {str_lit("a "), str_lit("constant declaration")}, + {str_lit("a "), str_lit("dynamiic array literal")}, + {str_lit("a "), str_lit("map index")}, + {str_lit("a "), str_lit("map literal")}, + {str_lit("a "), str_lit("matrix literal")}, + {str_lit("a "), str_lit("polymorphic type argument")}, + {str_lit("a "), str_lit("procedure argument")}, + {str_lit("a "), str_lit("simd vector literal")}, + {str_lit("a "), str_lit("slice literal")}, + {str_lit("a "), str_lit("structure literal")}, + {str_lit("a "), str_lit("variable declaration")}, + {str_lit("an "), str_lit("'any' literal")}, + {str_lit("an "), str_lit("array literal")}, + {str_lit("an "), str_lit("enumerated array literal")}, + +}; + +// Returns definite or indefinite article matching `context_name`, or "" if not found. +gb_internal String error_article(String context_name) { + for (int i = 0; i < gb_count_of(error_article_table); i += 1) { + if (context_name == error_article_table[i][1]) { + return error_article_table[i][0]; + } + } + return str_lit(""); +} + gb_internal bool errors_already_printed = false; gb_internal void print_all_errors(void) { From bbe4c32e32a2ca3f34ff0efbf7791815f84e0655 Mon Sep 17 00:00:00 2001 From: Yeongju Kang Date: Wed, 21 Aug 2024 08:25:14 +0900 Subject: [PATCH 30/43] changed signature of clock_getres --- core/sys/linux/sys.odin | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/sys/linux/sys.odin b/core/sys/linux/sys.odin index 8aa2f09cc..15f5c9d4c 100644 --- a/core/sys/linux/sys.odin +++ b/core/sys/linux/sys.odin @@ -2428,9 +2428,10 @@ clock_gettime :: proc "contextless" (clock: Clock_Id) -> (ts: Time_Spec, err: Er Finds the resolution of the specified clock. Available since Linux 2.6. */ -clock_getres :: proc "contextless" (clock: Clock_Id, res: ^Time_Spec) -> (Errno) { - ret := syscall(SYS_clock_getres, clock, res) - return Errno(-ret) +clock_getres :: proc "contextless" (clock: Clock_Id) -> (res: Time_Spec, err: Errno) { + ret := syscall(SYS_clock_getres, clock, &res) + err = Errno(-ret) + return } /* From 1bcc0742234d5cf1066e73384bd618ddab792e86 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 21 Aug 2024 14:18:12 +0200 Subject: [PATCH 31/43] dash --- src/check_expr.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 2540230df..95b0872ac 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -1175,13 +1175,13 @@ gb_internal void check_assignment(CheckerContext *c, Operand *operand, Type *typ case Addressing_Type: if (is_type_polymorphic(operand->type)) { error(operand->expr, - "Cannot assign '%s' — a polymorphic type — to %.*s%.*s", + "Cannot assign '%s' - a polymorphic type - to %.*s%.*s", op_type_str, LIT(article), LIT(context_name)); } else { error(operand->expr, - "Cannot assign '%s' — a type — to %.*s%.*s", + "Cannot assign '%s' - a type - to %.*s%.*s", op_type_str, LIT(article), LIT(context_name)); From c0125f3192a453195cad108bf8155c64be78f197 Mon Sep 17 00:00:00 2001 From: Laytan Date: Wed, 21 Aug 2024 14:41:23 +0200 Subject: [PATCH 32/43] correct the riscv64 stat structs --- core/os/os_linux.odin | 21 +-------------------- core/sys/linux/types.odin | 21 +-------------------- 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/core/os/os_linux.odin b/core/os/os_linux.odin index f1b3720c6..2f7a5ac43 100644 --- a/core/os/os_linux.odin +++ b/core/os/os_linux.odin @@ -262,7 +262,7 @@ Unix_File_Time :: struct { nanoseconds: i64, } -when ODIN_ARCH == .arm64 { +when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { OS_Stat :: struct { device_id: u64, // ID of device containing file serial: u64, // File serial number @@ -284,25 +284,6 @@ when ODIN_ARCH == .arm64 { _reserved: [2]i32, } #assert(size_of(OS_Stat) == 128) -} else when ODIN_ARCH == .riscv64 { - OS_Stat :: struct { - device_id: u64, - serial: u64, - mode: u32, - nlink: u32, - uid: u32, - gid: u32, - rdev: u64, - _: u64, - size: i64, - block_size: i32, - _: i32, - blocks: i64, - last_access: Unix_File_Time, - modified: Unix_File_Time, - status_change: Unix_File_Time, - _: [3]uint, - } } else { OS_Stat :: struct { device_id: u64, // ID of device containing file diff --git a/core/sys/linux/types.odin b/core/sys/linux/types.odin index c78a5b576..3f873f96c 100644 --- a/core/sys/linux/types.odin +++ b/core/sys/linux/types.odin @@ -119,7 +119,7 @@ when ODIN_ARCH == .amd64 { ctime: Time_Spec, _: [3]uint, } -} else when ODIN_ARCH == .arm64 { +} else when ODIN_ARCH == .arm64 || ODIN_ARCH == .riscv64 { _Arch_Stat :: struct { dev: Dev, ino: Inode, @@ -138,25 +138,6 @@ when ODIN_ARCH == .amd64 { ctime: Time_Spec, _: [2]u32, } -} else when ODIN_ARCH == .riscv64 { - _Arch_Stat :: struct { - dev: Dev, - ino: Inode, - mode: Mode, - nlink: u32, - uid: Uid, - gid: Gid, - rdev: Dev, - _: u64, - size: int, - blksize: i32, - _: i32, - blocks: int, - atime: Time_Spec, - mtime: Time_Spec, - ctime: Time_Spec, - _: [3]uint, - } } else { _Arch_Stat :: struct { dev: Dev, From c77e8ca4014d704190773d2254c618b089d612a7 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 21 Aug 2024 14:06:04 +0100 Subject: [PATCH 33/43] Fix tests --- tests/core/text/regex/test_core_text_regex.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/text/regex/test_core_text_regex.odin b/tests/core/text/regex/test_core_text_regex.odin index d99130584..dfc9224a8 100644 --- a/tests/core/text/regex/test_core_text_regex.odin +++ b/tests/core/text/regex/test_core_text_regex.odin @@ -50,7 +50,7 @@ check_expression_with_flags :: proc(t: ^testing.T, pattern: string, flags: regex } } -check_expression :: proc(t: ^testing.T, pattern, haystack: string, needles: ..string, extra_flags: regex.Flags = {}, loc := #caller_location) { +check_expression :: proc(t: ^testing.T, pattern, haystack: string, needles: ..string, extra_flags := regex.Flags{}, loc := #caller_location) { check_expression_with_flags(t, pattern, { .Global } + extra_flags, haystack, ..needles, loc = loc) check_expression_with_flags(t, pattern, { .Global, .No_Optimization } + extra_flags, @@ -61,7 +61,7 @@ check_expression :: proc(t: ^testing.T, pattern, haystack: string, needles: ..st haystack, ..needles, loc = loc) } -expect_error :: proc(t: ^testing.T, pattern: string, expected_error: typeid, flags: regex.Flags = {}, loc := #caller_location) { +expect_error :: proc(t: ^testing.T, pattern: string, expected_error: typeid, flags := regex.Flags{}, loc := #caller_location) { rex, err := regex.create(pattern, flags) regex.destroy(rex) From 63cd9a031a76220dc31ef8c009b47ace9fc2b0fd Mon Sep 17 00:00:00 2001 From: Laytan Date: Wed, 21 Aug 2024 15:11:16 +0200 Subject: [PATCH 34/43] fix variadic parameter with default value error check --- src/check_type.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/check_type.cpp b/src/check_type.cpp index 83b6600c8..3767f7666 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -1781,6 +1781,11 @@ gb_internal Type *check_get_params(CheckerContext *ctx, Scope *scope, Ast *_para success = false; } + if (default_value != nullptr) { + error(type_expr, "A variadic parameter may not have a default value"); + success = false; + } + GB_ASSERT(original_type_expr->kind == Ast_Ellipsis); type_expr = ast_array_type(type_expr->file(), original_type_expr->Ellipsis.token, nullptr, type_expr); } @@ -1819,8 +1824,6 @@ gb_internal Type *check_get_params(CheckerContext *ctx, Scope *scope, Ast *_para if (default_value != nullptr) { if (type_expr != nullptr && type_expr->kind == Ast_TypeidType) { error(type_expr, "A type parameter may not have a default value"); - } else if (is_variadic) { - error(type_expr, "A variadic parameter may not have a default value"); } else { param_value = handle_parameter_value(ctx, type, nullptr, default_value, true); } From 07aedb0b89c5ac4fa2607f446a7f3994159e11c3 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 21 Aug 2024 18:42:11 +0200 Subject: [PATCH 35/43] , --- src/check_expr.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/check_expr.cpp b/src/check_expr.cpp index 95b0872ac..9cdba2710 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -1175,13 +1175,13 @@ gb_internal void check_assignment(CheckerContext *c, Operand *operand, Type *typ case Addressing_Type: if (is_type_polymorphic(operand->type)) { error(operand->expr, - "Cannot assign '%s' - a polymorphic type - to %.*s%.*s", + "Cannot assign '%s', a polymorphic type, to %.*s%.*s", op_type_str, LIT(article), LIT(context_name)); } else { error(operand->expr, - "Cannot assign '%s' - a type - to %.*s%.*s", + "Cannot assign '%s', a type, to %.*s%.*s", op_type_str, LIT(article), LIT(context_name)); From 58e5078b6686250b86d654432fea034b730ca36e Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Thu, 22 Aug 2024 14:17:45 +0200 Subject: [PATCH 36/43] add riscv to simd.IS_EMULATED --- core/simd/simd.odin | 1 + 1 file changed, 1 insertion(+) diff --git a/core/simd/simd.odin b/core/simd/simd.odin index e7b1803c4..01d11dfbe 100644 --- a/core/simd/simd.odin +++ b/core/simd/simd.odin @@ -8,6 +8,7 @@ import "base:intrinsics" IS_EMULATED :: true when (ODIN_ARCH == .amd64 || ODIN_ARCH == .i386) && !intrinsics.has_target_feature("sse2") else true when (ODIN_ARCH == .arm64 || ODIN_ARCH == .arm32) && !intrinsics.has_target_feature("neon") else true when (ODIN_ARCH == .wasm64p32 || ODIN_ARCH == .wasm32) && !intrinsics.has_target_feature("simd128") else + true when (ODIN_ARCH == .riscv64) && !intrinsics.has_target_feature("v") else false // 128-bit vector aliases From b9043db434319dedd518dbab45e5cad9d0df486f Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Thu, 22 Aug 2024 22:59:13 +0200 Subject: [PATCH 37/43] os2: make platform error more ergonomic by making it an alias --- core/os/os2/errors.odin | 2 +- core/os/os2/errors_linux.odin | 2 ++ core/os/os2/errors_posix.odin | 2 ++ core/os/os2/errors_windows.odin | 4 +++- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/core/os/os2/errors.odin b/core/os/os2/errors.odin index f90baa699..8a2163634 100644 --- a/core/os/os2/errors.odin +++ b/core/os/os2/errors.odin @@ -30,7 +30,7 @@ General_Error :: enum u32 { Unsupported, } -Platform_Error :: enum i32 {None=0} +Platform_Error :: _Platform_Error Error :: union #shared_nil { General_Error, diff --git a/core/os/os2/errors_linux.odin b/core/os/os2/errors_linux.odin index 7f28d1c41..29815bf79 100644 --- a/core/os/os2/errors_linux.odin +++ b/core/os/os2/errors_linux.odin @@ -3,6 +3,8 @@ package os2 import "core:sys/linux" +_Platform_Error :: linux.Errno + @(rodata) _errno_strings := [linux.Errno]string{ .NONE = "", diff --git a/core/os/os2/errors_posix.odin b/core/os/os2/errors_posix.odin index 7143557a6..9e3424a4a 100644 --- a/core/os/os2/errors_posix.odin +++ b/core/os/os2/errors_posix.odin @@ -4,6 +4,8 @@ package os2 import "core:sys/posix" +_Platform_Error :: posix.Errno + _error_string :: proc(errno: i32) -> string { return string(posix.strerror(posix.Errno(errno))) } diff --git a/core/os/os2/errors_windows.odin b/core/os/os2/errors_windows.odin index 6421d26ee..6748c1167 100644 --- a/core/os/os2/errors_windows.odin +++ b/core/os/os2/errors_windows.odin @@ -5,6 +5,8 @@ import "base:runtime" import "core:slice" import win32 "core:sys/windows" +_Platform_Error :: win32.System_Error + _error_string :: proc(errno: i32) -> string { e := win32.DWORD(errno) if e == 0 { @@ -68,4 +70,4 @@ _get_platform_error :: proc() -> Error { // fallthrough } return Platform_Error(err) -} \ No newline at end of file +} From a66520ba57a4d6376c757f64086dba831aae1294 Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Thu, 22 Aug 2024 23:00:01 +0200 Subject: [PATCH 38/43] os2: process API for Darwin and most of it for BSDs --- core/os/os2/process_linux.odin | 11 +- core/os/os2/process_posix.odin | 340 ++++++++++++++++++-------- core/os/os2/process_posix_darwin.odin | 64 +++-- core/os/os2/process_posix_other.odin | 13 + core/sys/darwin/proc.odin | 24 ++ core/sys/kqueue/kqueue.odin | 256 +++++++++++++++++++ 6 files changed, 587 insertions(+), 121 deletions(-) create mode 100644 core/sys/kqueue/kqueue.odin diff --git a/core/os/os2/process_linux.odin b/core/os/os2/process_linux.odin index 60dc29189..40406bad5 100644 --- a/core/os/os2/process_linux.odin +++ b/core/os/os2/process_linux.odin @@ -576,10 +576,13 @@ _process_start :: proc(desc: Process_Desc) -> (process: Process, err: Error) { success_byte: [1]u8 linux.write(child_pipe_fds[WRITE], success_byte[:]) - if errno = linux.execveat(exe_fd, "", &cargs[0], env, {.AT_EMPTY_PATH}); errno != .NONE { - write_errno_to_parent_and_abort(child_pipe_fds[WRITE], errno) - } - unreachable() + errno = linux.execveat(exe_fd, "", &cargs[0], env, {.AT_EMPTY_PATH}) + + // NOTE: we can't tell the parent about this failure because we already wrote the success byte. + // So if this happens the user will just see the process failed when they call process_wait. + + assert(errno != nil) + intrinsics.trap() } process.pid = int(pid) diff --git a/core/os/os2/process_posix.odin b/core/os/os2/process_posix.odin index fc6e135d8..c9b67f199 100644 --- a/core/os/os2/process_posix.odin +++ b/core/os/os2/process_posix.odin @@ -3,9 +3,13 @@ package os2 import "base:runtime" -import "core:time" -import "core:sys/posix" +import "core:time" +import "core:strings" +import "core:path/filepath" + +import kq "core:sys/kqueue" +import "core:sys/posix" _exit :: proc "contextless" (code: int) -> ! { posix.exit(i32(code)) @@ -51,141 +55,286 @@ _process_start :: proc(desc: Process_Desc) -> (process: Process, err: Error) { return } + TEMP_ALLOCATOR_GUARD() + + // search PATH if just a plain name is provided. + exe_builder := strings.builder_make(temp_allocator()) + exe_name := desc.command[0] + if strings.index_byte(exe_name, '/') < 0 { + path_env := get_env("PATH", temp_allocator()) + path_dirs := filepath.split_list(path_env, temp_allocator()) + + found: bool + for dir in path_dirs { + strings.builder_reset(&exe_builder) + strings.write_string(&exe_builder, dir) + strings.write_byte(&exe_builder, '/') + strings.write_string(&exe_builder, exe_name) + + if exe_fd := posix.open(strings.to_cstring(&exe_builder), {.CLOEXEC, .EXEC}); exe_fd == -1 { + continue + } else { + posix.close(exe_fd) + found = true + break + } + } + if !found { + // check in cwd to match windows behavior + strings.builder_reset(&exe_builder) + strings.write_string(&exe_builder, desc.working_dir) + if len(desc.working_dir) > 0 && desc.working_dir[len(desc.working_dir)-1] != '/' { + strings.write_byte(&exe_builder, '/') + } + strings.write_string(&exe_builder, "./") + strings.write_string(&exe_builder, exe_name) + + // "hello/./world" is fine right? + + if exe_fd := posix.open(strings.to_cstring(&exe_builder), {.CLOEXEC, .EXEC}); exe_fd == -1 { + err = .Not_Exist + return + } else { + posix.close(exe_fd) + } + } + } else { + strings.builder_reset(&exe_builder) + strings.write_string(&exe_builder, exe_name) + + if exe_fd := posix.open(strings.to_cstring(&exe_builder), {.CLOEXEC, .EXEC}); exe_fd == -1 { + err = .Not_Exist + return + } else { + posix.close(exe_fd) + } + } + cwd: cstring; if desc.working_dir != "" { cwd = temp_cstring(desc.working_dir) } - cmd := make([]cstring, len(desc.command)+1, temp_allocator()) + cmd := make([]cstring, len(desc.command) + 1, temp_allocator()) for part, i in desc.command { cmd[i] = temp_cstring(part) } + env: [^]cstring + if desc.env == nil { + // take this process's current environment + env = posix.environ + } else { + cenv := make([]cstring, len(desc.env) + 1, temp_allocator()) + for env, i in desc.env { + cenv[i] = temp_cstring(env) + } + env = raw_data(cenv) + } + + READ :: 0 + WRITE :: 1 + + pipe: [2]posix.FD + if posix.pipe(&pipe) != .OK { + err = _get_platform_error() + return + } + defer posix.close(pipe[WRITE]) + defer posix.close(pipe[READ]) + + if posix.fcntl(pipe[READ], .SETFD, i32(posix.FD_CLOEXEC)) == -1 { + err = _get_platform_error() + return + } + if posix.fcntl(pipe[WRITE], .SETFD, i32(posix.FD_CLOEXEC)) == -1 { + err = _get_platform_error() + return + } + switch pid := posix.fork(); pid { case -1: err = _get_platform_error() return case 0: - // NOTE(laytan): would need to use execvp and look up the command in the PATH. - assert(len(desc.env) == 0, "unimplemented: process_start with env") + abort :: proc(parent_fd: posix.FD) -> ! { + #assert(len(posix.Errno) < max(u8)) + errno := u8(posix.errno()) + posix.write(parent_fd, &errno, 1) + runtime.trap() + } - null := posix.open("/dev/null", { .RDWR, .CLOEXEC }) - assert(null != -1) // TODO: Does this happen/need to be handled? + null := posix.open("/dev/null", {.RDWR}) + if null == -1 { abort(pipe[WRITE]) } stderr := (^File_Impl)(desc.stderr.impl).fd if desc.stderr != nil else null stdout := (^File_Impl)(desc.stdout.impl).fd if desc.stdout != nil else null stdin := (^File_Impl)(desc.stdin.impl).fd if desc.stdin != nil else null - posix.dup2(stderr, posix.STDERR_FILENO) - posix.dup2(stdout, posix.STDOUT_FILENO) - posix.dup2(stdin, posix.STDIN_FILENO ) - - // NOTE(laytan): is this how we should handle these? - // Maybe we can try to `stat` the cwd in the parent before forking? - // Does that mean no other errors could happen in chdir? - // How about execvp? + if posix.dup2(stderr, posix.STDERR_FILENO) == -1 { abort(pipe[WRITE]) } + if posix.dup2(stdout, posix.STDOUT_FILENO) == -1 { abort(pipe[WRITE]) } + if posix.dup2(stdin, posix.STDIN_FILENO ) == -1 { abort(pipe[WRITE]) } if cwd != nil { - if posix.chdir(cwd) != .OK { - posix.exit(i32(posix.errno())) // TODO: handle, or is it fine this way? - } + if posix.chdir(cwd) != .OK { abort(pipe[WRITE]) } } - posix.execvp(cmd[0], raw_data(cmd)) - posix.exit(i32(posix.errno())) // TODO: handle, or is it fine this way? + ok := u8(0) + posix.write(pipe[WRITE], &ok, 1) + + res := posix.execve(strings.to_cstring(&exe_builder), raw_data(cmd), env) + + // NOTE: we can't tell the parent about this failure because we already wrote the success byte. + // So if this happens the user will just see the process failed when they call process_wait. + + assert(res == -1) + runtime.trap() case: - fmt.println("returning") - process, _ = _process_open(int(pid), {}) + errno: posix.Errno + for { + errno_byte: u8 + switch posix.read(pipe[READ], &errno_byte, 1) { + case 1: + errno = posix.Errno(errno_byte) + case: + errno = posix.errno() + if errno == .EINTR { + continue + } else { + // If the read failed, something weird happened. Do not return the read + // error so the user knows to wait on it. + errno = nil + } + } + break + } + + if errno != nil { + // We can assume it trapped here. + + for { + info: posix.siginfo_t + wpid := posix.waitid(.P_PID, posix.id_t(process.pid), &info, {.EXITED}) + if wpid == -1 && posix.errno() == .EINTR { + continue + } + break + } + + err = errno + return + } + process.pid = int(pid) + process, _ = _process_open(int(pid), {}) return } } -import "core:fmt" -import "core:nbio/kqueue" - _process_wait :: proc(process: Process, timeout: time.Duration) -> (process_state: Process_State, err: Error) { process_state.pid = process.pid - if !process_posix_handle_still_valid(process) { - err = Platform_Error(posix.Errno.ESRCH) - return - } + _process_handle_still_valid(process) or_return - // prev := posix.signal(.SIGALRM, proc "c" (_: posix.Signal) { - // context = runtime.default_context() - // fmt.println("alarm") - // }) - // defer posix.signal(.SIGALRM, prev) + // timeout > 0 = use kqueue to wait (with a timeout) on process exit + // timeout == 0 = use waitid with WNOHANG so it returns immediately + // timeout > 0 = use waitid without WNOHANG so it waits indefinitely // - // posix.alarm(u32(time.duration_seconds(timeout))) - // defer posix.alarm(0) + // at the end use waitid to actually reap the process and get it's status - // TODO: if there's no timeout, don't set up a kqueue. + if timeout > 0 { + timeout := timeout - // TODO: if timeout is 0, don't set up a kqueue and use NO_HANG. + queue := kq.kqueue() or_return + defer posix.close(queue) - kq, qerr := kqueue.kqueue() - if qerr != nil { - err = Platform_Error(qerr) - return + changelist, eventlist: [1]kq.KEvent + + changelist[0] = { + ident = uintptr(process.pid), + filter = .Proc, + flags = { .Add }, + fflags = { + fproc = { .Exit }, + }, + } + + for { + start := time.tick_now() + n, kerr := kq.kevent(queue, changelist[:], eventlist[:], &{ + tv_sec = posix.time_t(timeout / time.Second), + tv_nsec = i64(timeout % time.Second), + }) + if kerr == .EINTR { + timeout -= time.tick_since(start) + continue + } else if kerr != nil { + err = kerr + return + } else if n == 0 { + err = .Timeout + _process_state_update_times(process, &process_state) + return + } else { + _process_state_update_times(process, &process_state) + break + } + } + } else { + flags := posix.Wait_Flags{.EXITED, .NOWAIT} + if timeout == 0 { + flags += {.NOHANG} + } + + info: posix.siginfo_t + for { + wpid := posix.waitid(.P_PID, posix.id_t(process.pid), &info, flags) + if wpid == -1 { + if errno := posix.errno(); errno == .EINTR { + continue + } else { + err = _get_platform_error() + return + } + } + break + } + + _process_state_update_times(process, &process_state) + + if info.si_signo == nil { + assert(timeout == 0) + err = .Timeout + return + } } - changelist, eventlist: [1]kqueue.KEvent - - changelist[0] = { - ident = uintptr(process.pid), - filter = .Proc, - flags = { .Add }, - fflags = { - fproc = 0x80000000, - }, + info: posix.siginfo_t + for { + wpid := posix.waitid(.P_PID, posix.id_t(process.pid), &info, {.EXITED}) + if wpid == -1 { + if errno := posix.errno(); errno == .EINTR { + continue + } else { + err = _get_platform_error() + return + } + } + break } - // NOTE: could this be interrupted which means it should be looped and subtracting the timeout on EINTR. - - n, eerr := kqueue.kevent(kq, changelist[:], eventlist[:], &{ - seconds = i64(timeout / time.Second), - nanoseconds = i64(timeout % time.Second), - }) - if eerr != nil { - err = Platform_Error(eerr) - return - } - - if n == 0 { - err = .Timeout - - // TODO: populate the time fields. - - return - } - - // NOTE(laytan): should this be looped untill WIFEXITED/WIFSIGNALED? - - status: i32 - wpid := posix.waitpid(posix.pid_t(process.pid), &status, {}) - if wpid == -1 { - err = _get_platform_error() - return - } - - process_state.exited = true - - // TODO: populate times - - switch { - case posix.WIFEXITED(status): - fmt.printfln("child exited, status=%v", posix.WEXITSTATUS(status)) - process_state.exit_code = int(posix.WEXITSTATUS(status)) - process_state.success = true - case posix.WIFSIGNALED(status): - fmt.printfln("child killed (signal %v)", posix.WTERMSIG(status)) - process_state.exit_code = int(posix.WTERMSIG(status)) + switch info.si_code.chld { + case: unreachable() + case .CONTINUED, .STOPPED: unreachable() + case .EXITED: + process_state.exited = true + process_state.exit_code = int(info.si_status) + process_state.success = process_state.exit_code == 0 + case .KILLED, .DUMPED, .TRAPPED: + process_state.exited = true + process_state.exit_code = int(info.si_status) process_state.success = false - case: - fmt.panicf("unexpected status (%x)", status) } return @@ -196,10 +345,7 @@ _process_close :: proc(process: Process) -> Error { } _process_kill :: proc(process: Process) -> (err: Error) { - if !process_posix_handle_still_valid(process) { - err = Platform_Error(posix.Errno.ESRCH) - return - } + _process_handle_still_valid(process) or_return if posix.kill(posix.pid_t(process.pid), .SIGKILL) != .OK { err = _get_platform_error() diff --git a/core/os/os2/process_posix_darwin.odin b/core/os/os2/process_posix_darwin.odin index 5558ef00a..ace7a4fa2 100644 --- a/core/os/os2/process_posix_darwin.odin +++ b/core/os/os2/process_posix_darwin.odin @@ -8,6 +8,7 @@ import "core:bytes" import "core:sys/darwin" import "core:sys/posix" import "core:sys/unix" +import "core:time" foreign import lib "system:System.framework" @@ -19,8 +20,6 @@ foreign lib { ) -> posix.result --- } -import "core:fmt" - _process_info_by_pid :: proc(pid: int, selection: Process_Info_Fields, allocator: runtime.Allocator) -> (info: Process_Info, err: Error) { get_pidinfo :: proc(pid: int, selection: Process_Info_Fields) -> (ppid: u32, prio: Maybe(i32), uid: posix.uid_t, ok: bool) { // Short info is enough and requires less permissions if the priority isn't requested. @@ -258,31 +257,56 @@ _process_list :: proc(allocator: runtime.Allocator) -> (list: []int, err: Error) } _process_open :: proc(pid: int, flags: Process_Open_Flags) -> (process: Process, err: Error) { - - // NOTE(laytan): pids can get reused, and afaik posix/macos doesn't have a unique identifier - // for a specific process execution, next best thing to me is checking the time the process - // started as some extra "uniqueness". We could also hash a bunch of the fields in this info. - - // This incidentally also checks if the pid is actually valid so that's nice. - - pinfo: darwin.proc_bsdinfo - ret := darwin.proc_pidinfo(posix.pid_t(pid), .BSDINFO, 0, &pinfo, size_of(pinfo)) - if ret <= 0 { + rusage: darwin.rusage_info_v0 + if ret := darwin.proc_pid_rusage(posix.pid_t(pid), .V0, &rusage); ret != 0 { err = _get_platform_error() return } - assert(ret == size_of(pinfo)) - process = { int(pid), uintptr(pinfo.pbi_start_tvusec) } + // XOR fold the UUID so it fits the handle, I think this is enough to verify pid uniqueness. + #assert(size_of(uintptr) == size_of(u64)) + a := intrinsics.unaligned_load((^u64)(&rusage.ri_uuid)) + b := intrinsics.unaligned_load((^u64)(&rusage.ri_uuid[8])) + process.handle = uintptr(a ~ b) + + process.pid = int(pid) return } -process_posix_handle_still_valid :: proc(p: Process) -> bool { - pinfo: darwin.proc_bsdinfo - ret := darwin.proc_pidinfo(posix.pid_t(p.pid), .BSDINFO, 0, &pinfo, size_of(pinfo)) - if ret <= 0 { - return false +_process_handle_still_valid :: proc(p: Process) -> Error { + rusage: darwin.rusage_info_v0 + if ret := darwin.proc_pid_rusage(posix.pid_t(p.pid), .V0, &rusage); ret != 0 { + return _get_platform_error() } - return uintptr(pinfo.pbi_start_tvusec) == p.handle + // XOR fold the UUID so it fits the handle, I think this is enough to verify pid uniqueness. + #assert(size_of(uintptr) == size_of(u64)) + a := intrinsics.unaligned_load((^u64)(&rusage.ri_uuid)) + b := intrinsics.unaligned_load((^u64)(&rusage.ri_uuid[8])) + handle := uintptr(a ~ b) + + if p.handle != handle { + return posix.Errno.ESRCH + } + + return nil +} + +_process_state_update_times :: proc(p: Process, state: ^Process_State) { + rusage: darwin.rusage_info_v0 + if ret := darwin.proc_pid_rusage(posix.pid_t(p.pid), .V0, &rusage); ret != 0 { + return + } + + // NOTE(laytan): I have no clue if this is correct, the output seems correct comparing it with `time`'s output. + HZ :: 20000000 + + state.user_time = ( + (time.Duration(rusage.ri_user_time) / HZ * time.Second) + + time.Duration(rusage.ri_user_time % HZ)) + state.system_time = ( + (time.Duration(rusage.ri_system_time) / HZ * time.Second) + + time.Duration(rusage.ri_system_time % HZ)) + + return } diff --git a/core/os/os2/process_posix_other.odin b/core/os/os2/process_posix_other.odin index ce84b1012..02f78b9ac 100644 --- a/core/os/os2/process_posix_other.odin +++ b/core/os/os2/process_posix_other.odin @@ -13,3 +13,16 @@ _process_list :: proc(allocator: runtime.Allocator) -> (list: []int, err: Error) err = .Unsupported return } + +_process_open :: proc(pid: int, flags: Process_Open_Flags) -> (process: Process, err: Error) { + err = .Unsupported + return +} + +_process_handle_still_valid :: proc(p: Process) -> Error { + return nil +} + +_process_state_update_times :: proc(p: Process, state: ^Process_State) { + return +} diff --git a/core/sys/darwin/proc.odin b/core/sys/darwin/proc.odin index 8f7eb1fcb..fa5391f6f 100644 --- a/core/sys/darwin/proc.odin +++ b/core/sys/darwin/proc.odin @@ -12,6 +12,7 @@ foreign lib { proc_pidinfo :: proc(pid: posix.pid_t, flavor: PID_Info_Flavor, arg: i64, buffer: rawptr, buffersize: i32) -> i32 --- proc_pidpath :: proc(pid: posix.pid_t, buffer: [^]byte, buffersize: u32) -> i32 --- proc_listallpids :: proc(buffer: [^]i32, buffersize: i32) -> i32 --- + proc_pid_rusage :: proc(pid: posix.pid_t, flavor: Pid_Rusage_Flavor, buffer: rawptr) -> i32 --- } MAXCOMLEN :: 16 @@ -166,3 +167,26 @@ PID_Info_Flavor :: enum i32 { } PIDPATHINFO_MAXSIZE :: 4*posix.PATH_MAX + +Pid_Rusage_Flavor :: enum i32 { + V0, + V1, + V2, + V3, + V4, + V5, +} + +rusage_info_v0 :: struct { + ri_uuid: [16]u8, + ri_user_time: u64, + ri_system_time: u64, + ri_pkg_idle_wkups: u64, + ri_interrupt_wkups: u64, + ri_pageins: u64, + ri_wired_size: u64, + ri_resident_size: u64, + ri_phys_footprint: u64, + ri_proc_start_abstime: u64, + ri_proc_exit_abstime: u64, +} diff --git a/core/sys/kqueue/kqueue.odin b/core/sys/kqueue/kqueue.odin new file mode 100644 index 000000000..27d1ecaae --- /dev/null +++ b/core/sys/kqueue/kqueue.odin @@ -0,0 +1,256 @@ +//+build darwin, netbsd, openbsd, freebsd +package kqueue + +when ODIN_OS == .Darwin { + foreign import lib "system:System.framework" +} else { + foreign import lib "system:c" +} + +import "base:intrinsics" + +import "core:c" +import "core:sys/posix" + +KQ :: posix.FD + +kqueue :: proc() -> (kq: KQ, err: posix.Errno) { + kq = _kqueue() + if kq == -1 { + err = posix.errno() + } + return +} + +kevent :: proc(kq: KQ, change_list: []KEvent, event_list: []KEvent, timeout: ^posix.timespec) -> (n_events: c.int, err: posix.Errno) { + n_events = _kevent( + kq, + raw_data(change_list), + c.int(len(change_list)), + raw_data(event_list), + c.int(len(event_list)), + timeout, + ) + if n_events == -1 { + err = posix.errno() + } + return +} + +Flag :: enum _Flags_Backing { + Add = log2(0x0001), // Add event to kq (implies .Enable). + Delete = log2(0x0002), // Delete event from kq. + Enable = log2(0x0004), // Enable event. + Disable = log2(0x0008), // Disable event (not reported). + One_Shot = log2(0x0010), // Only report one occurrence. + Clear = log2(0x0020), // Clear event state after reporting. + Receipt = log2(0x0040), // Force immediate event output. + Dispatch = log2(0x0080), // Disable event after reporting. + + Error = log2(0x4000), // Error, data contains errno. + EOF = log2(0x8000), // EOF detected. +} +Flags :: bit_set[Flag; _Flags_Backing] + +Filter :: enum _Filter_Backing { + Read = _FILTER_READ, // Check for read availability on the file descriptor. + Write = _FILTER_WRITE, // Check for write availability on the file descriptor. + AIO = _FILTER_AIO, // Attached to AIO requests. + VNode = _FILTER_VNODE, // Check for changes to the subject file. + Proc = _FILTER_PROC, // Check for changes to the subject process. + Signal = _FILTER_SIGNAL, // Check for signals delivered to the process. + Timer = _FILTER_TIMER, // Timers. +} + +RW_Flag :: enum u32 { + Low_Water_Mark = log2(0x00000001), +} +RW_Flags :: bit_set[RW_Flag; u32] + +VNode_Flag :: enum u32 { + Delete = log2(0x00000001), // Deleted. + Write = log2(0x00000002), // Contents changed. + Extend = log2(0x00000004), // Size increased. + Attrib = log2(0x00000008), // Attributes changed. + Link = log2(0x00000010), // Link count changed. + Rename = log2(0x00000020), // Renamed. + Revoke = log2(0x00000040), // Access was revoked. +} +VNode_Flags :: bit_set[VNode_Flag; u32] + +Proc_Flag :: enum u32 { + Exit = log2(0x80000000), // Process exited. + Fork = log2(0x40000000), // Process forked. + Exec = log2(0x20000000), // Process exec'd. + Signal = log2(0x08000000), // Shared with `Filter.Signal`. +} +Proc_Flags :: bit_set[Proc_Flag; u32] + +Timer_Flag :: enum u32 { + Seconds = log2(0x00000001), // Data is seconds. + USeconds = log2(0x00000002), // Data is microseconds. + NSeconds = log2(_NOTE_NSECONDS), // Data is nanoseconds. + Absolute = log2(_NOTE_ABSOLUTE), // Absolute timeout. +} +Timer_Flags :: bit_set[Timer_Flag; u32] + +when ODIN_OS == .Darwin { + + _Filter_Backing :: distinct i16 + _Flags_Backing :: distinct u16 + + _FILTER_READ :: -1 + _FILTER_WRITE :: -2 + _FILTER_AIO :: -3 + _FILTER_VNODE :: -4 + _FILTER_PROC :: -5 + _FILTER_SIGNAL :: -6 + _FILTER_TIMER :: -7 + + _NOTE_NSECONDS :: 0x00000004 + _NOTE_ABSOLUTE :: 0x00000008 + + KEvent :: struct #align(4) { + // Value used to identify this event. The exact interpretation is determined by the attached filter. + ident: uintptr, + // Filter for event. + filter: Filter, + // General flags. + flags: Flags, + // Filter specific flags. + fflags: struct #raw_union { + rw: RW_Flags, + vnode: VNode_Flags, + fproc: Proc_Flags, + // vm: VM_Flags, + timer: Timer_Flags, + }, + // Filter specific data. + data: c.long /* intptr_t */, + // Opaque user data passed through the kernel unchanged. + udata: rawptr, + } + +} else when ODIN_OS == .FreeBSD { + + _Filter_Backing :: distinct i16 + _Flags_Backing :: distinct u16 + + _FILTER_READ :: -1 + _FILTER_WRITE :: -2 + _FILTER_AIO :: -3 + _FILTER_VNODE :: -4 + _FILTER_PROC :: -5 + _FILTER_SIGNAL :: -6 + _FILTER_TIMER :: -7 + + _NOTE_NSECONDS :: 0x00000004 + _NOTE_ABSOLUTE :: 0x00000008 + + KEvent :: struct { + // Value used to identify this event. The exact interpretation is determined by the attached filter. + ident: uintptr, + // Filter for event. + filter: Filter, + // General flags. + flags: Flags, + // Filter specific flags. + fflags: struct #raw_union { + rw: RW_Flags, + vnode: VNode_Flags, + fproc: Proc_Flags, + // vm: VM_Flags, + timer: Timer_Flags, + }, + // Filter specific data. + data: i64, + // Opaque user data passed through the kernel unchanged. + udata: rawptr, + // Extensions. + ext: [4]u64, + } +} else when ODIN_OS == .NetBSD { + + _Filter_Backing :: distinct u32 + _Flags_Backing :: distinct u32 + + _FILTER_READ :: 0 + _FILTER_WRITE :: 1 + _FILTER_AIO :: 2 + _FILTER_VNODE :: 3 + _FILTER_PROC :: 4 + _FILTER_SIGNAL :: 5 + _FILTER_TIMER :: 6 + + _NOTE_NSECONDS :: 0x00000003 + _NOTE_ABSOLUTE :: 0x00000010 + + KEvent :: struct #align(4) { + // Value used to identify this event. The exact interpretation is determined by the attached filter. + ident: uintptr, + // Filter for event. + filter: Filter, + // General flags. + flags: Flags, + // Filter specific flags. + fflags: struct #raw_union { + rw: RW_Flags, + vnode: VNode_Flags, + fproc: Proc_Flags, + // vm: VM_Flags, + timer: Timer_Flags, + }, + // Filter specific data. + data: i64, + // Opaque user data passed through the kernel unchanged. + udata: rawptr, + // Extensions. + ext: [4]u64, + } +} else when ODIN_OS == .OpenBSD { + + _Filter_Backing :: distinct i16 + _Flags_Backing :: distinct u16 + + _FILTER_READ :: -1 + _FILTER_WRITE :: -2 + _FILTER_AIO :: -3 + _FILTER_VNODE :: -4 + _FILTER_PROC :: -5 + _FILTER_SIGNAL :: -6 + _FILTER_TIMER :: -7 + + _NOTE_NSECONDS :: 0x00000003 + _NOTE_ABSOLUTE :: 0x00000010 + + KEvent :: struct #align(4) { + // Value used to identify this event. The exact interpretation is determined by the attached filter. + ident: uintptr, + // Filter for event. + filter: Filter, + // General flags. + flags: Flags, + // Filter specific flags. + fflags: struct #raw_union { + rw: RW_Flags, + vnode: VNode_Flags, + fproc: Proc_Flags, + // vm: VM_Flags, + timer: Timer_Flags, + }, + // Filter specific data. + data: i64, + // Opaque user data passed through the kernel unchanged. + udata: rawptr, + } +} + +@(private) +log2 :: intrinsics.constant_log2 + +foreign lib { + @(link_name="kqueue") + _kqueue :: proc() -> KQ --- + @(link_name="kevent") + _kevent :: proc(kq: KQ, change_list: [^]KEvent, n_changes: c.int, event_list: [^]KEvent, n_events: c.int, timeout: ^posix.timespec) -> c.int --- +} From caef37bc18477b1bb945f28be2598dd5fabe2473 Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Sat, 17 Aug 2024 02:09:52 +0200 Subject: [PATCH 39/43] orca: implement core:time and core:log --- core/log/file_console_logger.odin | 1 + core/sys/orca/macros.odin | 41 ------------------------------- core/sys/orca/odin.odin | 22 +++++++++++++++++ core/time/time_orca.odin | 29 ++++++++++++++++++++++ core/time/time_other.odin | 1 + 5 files changed, 53 insertions(+), 41 deletions(-) create mode 100644 core/sys/orca/odin.odin create mode 100644 core/time/time_orca.odin diff --git a/core/log/file_console_logger.odin b/core/log/file_console_logger.odin index fb968ccb6..f05f7a258 100644 --- a/core/log/file_console_logger.odin +++ b/core/log/file_console_logger.odin @@ -1,4 +1,5 @@ //+build !freestanding +//+build !orca package log import "core:encoding/ansi" diff --git a/core/sys/orca/macros.odin b/core/sys/orca/macros.odin index 9e7dbed85..d6a1a0f82 100644 --- a/core/sys/orca/macros.odin +++ b/core/sys/orca/macros.odin @@ -2,8 +2,6 @@ package orca -import "core:fmt" - //////////////////////////////////////////////////////////////////////////////// // Helpers for logging, asserting and aborting. //////////////////////////////////////////////////////////////////////////////// @@ -18,16 +16,6 @@ log_error :: proc "contextless" (msg: cstring, loc := #caller_location) { ) } -log_errorf :: proc(format: string, args: ..any, loc := #caller_location) { - log_ext( - .ERROR, - cstring(raw_data(loc.procedure)), - cstring(raw_data(loc.file_path)), - loc.line, - fmt.ctprintf(format, ..args), - ) -} - log_warning :: proc "contextless" (msg: cstring, loc := #caller_location) { log_ext( .WARNING, @@ -38,16 +26,6 @@ log_warning :: proc "contextless" (msg: cstring, loc := #caller_location) { ) } -log_warningf :: proc(format: string, args: ..any, loc := #caller_location) { - log_ext( - .WARNING, - cstring(raw_data(loc.procedure)), - cstring(raw_data(loc.file_path)), - loc.line, - fmt.ctprintf(format, ..args), - ) -} - log_info :: proc "contextless" (msg: cstring, loc := #caller_location) { log_ext( .INFO, @@ -58,16 +36,6 @@ log_info :: proc "contextless" (msg: cstring, loc := #caller_location) { ) } -log_infof :: proc(format: string, args: ..any, loc := #caller_location) { - log_ext( - .INFO, - cstring(raw_data(loc.procedure)), - cstring(raw_data(loc.file_path)), - loc.line, - fmt.ctprintf(format, ..args), - ) -} - abort :: proc "contextless" (msg: cstring, loc := #caller_location) { abort_ext( cstring(raw_data(loc.procedure)), @@ -77,15 +45,6 @@ abort :: proc "contextless" (msg: cstring, loc := #caller_location) { ) } -abortf :: proc(format: string, args: ..any, loc := #caller_location) { - abort_ext( - cstring(raw_data(loc.procedure)), - cstring(raw_data(loc.file_path)), - loc.line, - fmt.ctprintf(format, ..args), - ) -} - //////////////////////////////////////////////////////////////////////////////// // Types and helpers for doubly-linked lists. //////////////////////////////////////////////////////////////////////////////// diff --git a/core/sys/orca/odin.odin b/core/sys/orca/odin.odin new file mode 100644 index 000000000..5c3e3e4d9 --- /dev/null +++ b/core/sys/orca/odin.odin @@ -0,0 +1,22 @@ +// File contains Odin specific helpers. + +package orca + +import "base:runtime" + +create_odin_logger :: proc(lowest := runtime.Logger_Level.Debug, ident := "") -> runtime.Logger { + return runtime.Logger{odin_logger_proc, nil, lowest, {}} +} + +odin_logger_proc :: proc(logger_data: rawptr, level: runtime.Logger_Level, text: string, options: runtime.Logger_Options, location := #caller_location) { + cbuf := make([]byte, len(text)+1, context.temp_allocator) + copy(cbuf, text) + ctext := cstring(raw_data(cbuf)) + + switch level { + case .Debug, .Info: log_info(ctext, location) + case .Warning: log_warning(ctext, location) + case: fallthrough + case .Error, .Fatal: log_error(ctext, location) + } +} diff --git a/core/time/time_orca.odin b/core/time/time_orca.odin new file mode 100644 index 000000000..b2598fd6e --- /dev/null +++ b/core/time/time_orca.odin @@ -0,0 +1,29 @@ +//+private +//+build orca +package time + +import "base:intrinsics" + +import "core:sys/orca" + +_IS_SUPPORTED :: true + +_now :: proc "contextless" () -> Time { + CLK_JAN_1970 :: 2208988800 + secs := orca.clock_time(.DATE) + return Time{i64((secs - CLK_JAN_1970) * 1e9)} +} + +_sleep :: proc "contextless" (d: Duration) { + // NOTE: no way to sleep afaict. + if d > 0 { + orca.log_warning("core:time 'sleep' is unimplemented for orca") + } +} + +_tick_now :: proc "contextless" () -> Tick { + secs := orca.clock_time(.MONOTONIC) + return Tick{i64(secs * 1e9)} +} + +_yield :: proc "contextless" () {} diff --git a/core/time/time_other.odin b/core/time/time_other.odin index dd3d39644..164d23f25 100644 --- a/core/time/time_other.odin +++ b/core/time/time_other.odin @@ -8,6 +8,7 @@ //+build !darwin //+build !wasi //+build !windows +//+build !orca package time _IS_SUPPORTED :: false From c2a7c29ce89e907893c9baf6b855331ad5213a7e Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Fri, 23 Aug 2024 20:24:06 +0200 Subject: [PATCH 40/43] os2: fix using uuid as process handle for darwin, once it goes zombie it changes --- core/os/os2/process_posix_darwin.odin | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/core/os/os2/process_posix_darwin.odin b/core/os/os2/process_posix_darwin.odin index ace7a4fa2..648c4d389 100644 --- a/core/os/os2/process_posix_darwin.odin +++ b/core/os/os2/process_posix_darwin.odin @@ -263,13 +263,10 @@ _process_open :: proc(pid: int, flags: Process_Open_Flags) -> (process: Process, return } - // XOR fold the UUID so it fits the handle, I think this is enough to verify pid uniqueness. - #assert(size_of(uintptr) == size_of(u64)) - a := intrinsics.unaligned_load((^u64)(&rusage.ri_uuid)) - b := intrinsics.unaligned_load((^u64)(&rusage.ri_uuid[8])) - process.handle = uintptr(a ~ b) - - process.pid = int(pid) + // Using the start time as the handle, there is no pidfd or anything on Darwin. + // There is a uuid, but once a process becomes a zombie it changes... + process.handle = uintptr(rusage.ri_proc_start_abstime) + process.pid = int(pid) return } @@ -279,12 +276,7 @@ _process_handle_still_valid :: proc(p: Process) -> Error { return _get_platform_error() } - // XOR fold the UUID so it fits the handle, I think this is enough to verify pid uniqueness. - #assert(size_of(uintptr) == size_of(u64)) - a := intrinsics.unaligned_load((^u64)(&rusage.ri_uuid)) - b := intrinsics.unaligned_load((^u64)(&rusage.ri_uuid[8])) - handle := uintptr(a ~ b) - + handle := uintptr(rusage.ri_proc_start_abstime) if p.handle != handle { return posix.Errno.ESRCH } From 963e8544f4c558dbe78b47d5b4ba3773ec54f11c Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Fri, 23 Aug 2024 20:24:40 +0200 Subject: [PATCH 41/43] os2: CLOEXEC the fds from `pipe` for posix implementation --- core/os/os2/pipe_posix.odin | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/os/os2/pipe_posix.odin b/core/os/os2/pipe_posix.odin index 62d6fc9fb..13c1f8aec 100644 --- a/core/os/os2/pipe_posix.odin +++ b/core/os/os2/pipe_posix.odin @@ -12,6 +12,15 @@ _pipe :: proc() -> (r, w: ^File, err: Error) { return } + if posix.fcntl(fds[0], .SETFD, i32(posix.FD_CLOEXEC)) == -1 { + err = _get_platform_error() + return + } + if posix.fcntl(fds[1], .SETFD, i32(posix.FD_CLOEXEC)) == -1 { + err = _get_platform_error() + return + } + r = __new_file(fds[0]) ri := (^File_Impl)(r.impl) From ce53805d94eefbecac49d36407b0cc707bc988ff Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Fri, 23 Aug 2024 20:25:19 +0200 Subject: [PATCH 42/43] os2: fix read_entire_file wrong slice end variable --- core/os/os2/file_util.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/os/os2/file_util.odin b/core/os/os2/file_util.odin index 90ce96223..2dc457f05 100644 --- a/core/os/os2/file_util.odin +++ b/core/os/os2/file_util.odin @@ -154,7 +154,7 @@ read_entire_file_from_file :: proc(f: ^File, allocator: runtime.Allocator) -> (d n: int n, err = read(f, buffer[:]) total += n - append_elems(&out_buffer, ..buffer[:total]) + append_elems(&out_buffer, ..buffer[:n]) if err != nil { if err == .EOF || err == .Broken_Pipe { err = nil From d4102817665e044b092f3c136802373fd830eb4d Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Fri, 23 Aug 2024 20:25:47 +0200 Subject: [PATCH 43/43] os2: do read_entire_file in parts if the file size is 0 --- core/os/os2/file_util.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/os/os2/file_util.odin b/core/os/os2/file_util.odin index 2dc457f05..e328f9a02 100644 --- a/core/os/os2/file_util.odin +++ b/core/os/os2/file_util.odin @@ -130,7 +130,7 @@ read_entire_file_from_file :: proc(f: ^File, allocator: runtime.Allocator) -> (d return } - if has_size { + if has_size && size > 0 { total: int data = make([]byte, size, allocator) or_return for total < len(data) {