mirror of
https://github.com/Ed94/Odin.git
synced 2026-07-28 18:30:06 +00:00
Add core:text/regex
This commit is contained in:
Feoramund
@@ -0,0 +1,175 @@
|
||||
/*
|
||||
package regex_vm implements a threaded virtual machine for interpreting
|
||||
regular expressions, based on the designs described by Russ Cox and attributed
|
||||
to both Ken Thompson and Rob Pike.
|
||||
|
||||
The virtual machine executes all threads in lock step, i.e. the string pointer
|
||||
does not advance until all threads have finished processing the current rune.
|
||||
The algorithm does not look backwards.
|
||||
|
||||
Threads merge when splitting or jumping to positions already visited by another
|
||||
thread, based on the observation that each thread having visited one PC
|
||||
(Program Counter) state will execute identically to the previous thread.
|
||||
|
||||
Each thread keeps a save state of its capture groups, and thread priority is
|
||||
used to allow higher precedence operations to complete first with correct save
|
||||
states, such as greedy versus non-greedy repetition.
|
||||
|
||||
For more information, see: https://swtch.com/~rsc/regexp/regexp2.html
|
||||
|
||||
|
||||
**Implementation Details:**
|
||||
|
||||
- Each opcode is 8 bits in size, and most instructions have no operands.
|
||||
|
||||
- All operands larger than `u8` are read in system endian order.
|
||||
|
||||
- Jump and Split instructions operate on absolute positions in `u16` operands.
|
||||
|
||||
- Classes such as `[0-9]` are stored in a RegEx-specific slice of structs which
|
||||
are then dereferenced by a `u8` index from the `Rune_Class` instructions.
|
||||
|
||||
- Each Byte and Rune opcode have their operands stored inline after the opcode,
|
||||
sized `u8` and `i32` respectively.
|
||||
|
||||
- A bitmap is used to determine which PC positions are occupied by a thread to
|
||||
perform merging. The bitmap is cleared with every new frame.
|
||||
|
||||
- The VM supports two modes: ASCII and Unicode, decided by a compile-time
|
||||
boolean constant argument provided to `run`. The procedure differs only in
|
||||
string decoding. This was done for the sake of performance.
|
||||
|
||||
- No allocations are ever freed; the VM expects an arena or temporary allocator
|
||||
to be used in the context preceding it.
|
||||
|
||||
|
||||
**Opcode Reference:**
|
||||
|
||||
(0x00) Match
|
||||
|
||||
The terminal opcode which ends a thread. This always comes at the end of
|
||||
the program.
|
||||
|
||||
(0x01) Match_And_Exit
|
||||
|
||||
A modified version of Match which stops the virtual machine entirely. It is
|
||||
only compiled for `No_Capture` expressions, as those expressions do not
|
||||
need to determine which thread may have saved the most appropriate capture
|
||||
groups.
|
||||
|
||||
(0x02) Byte
|
||||
|
||||
Consumes one byte from the text using its operand, which is also a byte.
|
||||
|
||||
(0x03) Rune
|
||||
|
||||
Consumes one Unicode codepoint from the text using its operand, which is
|
||||
four bytes long in a system-dependent endian order.
|
||||
|
||||
(0x04) Rune_Class
|
||||
|
||||
Consumes one character (which may be an ASCII byte or Unicode codepoint,
|
||||
wholly dependent on which mode the virtual machine is running in) from the
|
||||
text.
|
||||
|
||||
The actual data storing what runes and ranges of runes apply to the class
|
||||
are stored alongside the program in the Regular_Expression structure and
|
||||
the operand for this opcode is a single byte which indexes into a
|
||||
collection of these data structures.
|
||||
|
||||
(0x05) Rune_Class_Negated
|
||||
|
||||
A modified version of Rune_Class that functions the same, save for how it
|
||||
returns the opposite of what Rune_Class matches.
|
||||
|
||||
(0x06) Wildcard
|
||||
|
||||
Consumes one byte or one Unicode codepoint, depending on the VM mode.
|
||||
|
||||
(0x07) Jump
|
||||
|
||||
Sets the Program Counter of a VM thread to the operand, which is a u16.
|
||||
This opcode is used to implement Alternation (coming at the end of the left
|
||||
choice) and Repeat_Zero (to cause the thread to loop backwards).
|
||||
|
||||
(0x08) Split
|
||||
|
||||
Spawns a new thread for the X operand and causes the current thread to jump
|
||||
to the Y operand. This opcode is used to implement Alternation, all the
|
||||
Repeat variations, and the Optional nodes.
|
||||
|
||||
Splitting threads is how the virtual machine is able to execute optional
|
||||
control flow paths, letting it evaluate different possible ways to match
|
||||
text.
|
||||
|
||||
(0x09) Save
|
||||
|
||||
Saves the current string index to a slot on the thread dictated by the
|
||||
operand. These values will be used later to reconstruct capture groups.
|
||||
|
||||
(0x0A) Assert_Start
|
||||
|
||||
Asserts that the thread is at the beginning of a string.
|
||||
|
||||
(0x0B) Assert_End
|
||||
|
||||
Asserts that the thread is at the end of a string.
|
||||
|
||||
(0x0C) Assert_Word_Boundary
|
||||
|
||||
Asserts that the thread is on a word boundary, which can be the start or
|
||||
end of the text. This examines both the current rune and the next rune.
|
||||
|
||||
(0x0D) Assert_Non_Word_Boundary
|
||||
|
||||
A modified version of Assert_Word_Boundary that returns the opposite value.
|
||||
|
||||
(0x0E) Multiline_Open
|
||||
|
||||
This opcode is compiled in only when the `Multiline` flag is present, and
|
||||
it replaces both `^` and `$` text anchors.
|
||||
|
||||
It asserts that either the current thread is on one of the string
|
||||
boundaries, or it consumes a `\n` or `\r` character.
|
||||
|
||||
If a `\r` character is consumed, the PC will be advanced to the sibling
|
||||
`Multiline_Close` opcode to optionally consume a `\n` character on the next
|
||||
frame.
|
||||
|
||||
(0x0F) Multiline_Close
|
||||
|
||||
This opcode is always present after `Multiline_Open`.
|
||||
|
||||
It handles consuming the second half of a complete newline, if necessary.
|
||||
For example, Windows newlines are represented by the characters `\r\n`,
|
||||
whereas UNIX newlines are `\n` and Macintosh newlines are `\r`.
|
||||
|
||||
(0x10) Wait_For_Byte
|
||||
(0x11) Wait_For_Rune
|
||||
(0x12) Wait_For_Rune_Class
|
||||
(0x13) Wait_For_Rune_Class_Negated
|
||||
|
||||
These opcodes are an optimization around restarting threads on failed
|
||||
matches when the beginning to a pattern is predictable and the Global flag
|
||||
is set.
|
||||
|
||||
They will cause the VM to wait for the next rune to match before splitting,
|
||||
as would happen in the un-optimized version.
|
||||
|
||||
(0x14) Match_All_And_Escape
|
||||
|
||||
This opcode is an optimized version of `.*$` or `.+$` that causes the
|
||||
active thread to immediately work on escaping the program by following all
|
||||
Jumps out to the end.
|
||||
|
||||
While running through the rest of the program, the thread will trigger on
|
||||
every Save instruction it passes to store the length of the string.
|
||||
|
||||
This way, any time a program hits one of these `.*$` constructs, the
|
||||
virtual machine can exit early, vastly improving processing times.
|
||||
|
||||
Be aware, this opcode is not compiled in if the `Multiline` flag is on, as
|
||||
the meaning of `$` changes with that flag.
|
||||
|
||||
*/
|
||||
package regex_vm
|
||||
@@ -0,0 +1,73 @@
|
||||
package regex_vm
|
||||
|
||||
Opcode_Iterator :: struct {
|
||||
code: Program,
|
||||
pc: int,
|
||||
}
|
||||
|
||||
iterate_opcodes :: proc(iter: ^Opcode_Iterator) -> (opcode: Opcode, pc: int, ok: bool) {
|
||||
if iter.pc >= len(iter.code) {
|
||||
return
|
||||
}
|
||||
|
||||
opcode = iter.code[iter.pc]
|
||||
pc = iter.pc
|
||||
ok = true
|
||||
|
||||
switch opcode {
|
||||
case .Match: iter.pc += size_of(Opcode)
|
||||
case .Match_And_Exit: iter.pc += size_of(Opcode)
|
||||
case .Byte: iter.pc += size_of(Opcode) + size_of(u8)
|
||||
case .Rune: iter.pc += size_of(Opcode) + size_of(rune)
|
||||
case .Rune_Class: iter.pc += size_of(Opcode) + size_of(u8)
|
||||
case .Rune_Class_Negated: iter.pc += size_of(Opcode) + size_of(u8)
|
||||
case .Wildcard: iter.pc += size_of(Opcode)
|
||||
case .Jump: iter.pc += size_of(Opcode) + size_of(u16)
|
||||
case .Split: iter.pc += size_of(Opcode) + 2 * size_of(u16)
|
||||
case .Save: iter.pc += size_of(Opcode) + size_of(u8)
|
||||
case .Assert_Start: iter.pc += size_of(Opcode)
|
||||
case .Assert_End: iter.pc += size_of(Opcode)
|
||||
case .Assert_Word_Boundary: iter.pc += size_of(Opcode)
|
||||
case .Assert_Non_Word_Boundary: iter.pc += size_of(Opcode)
|
||||
case .Multiline_Open: iter.pc += size_of(Opcode)
|
||||
case .Multiline_Close: iter.pc += size_of(Opcode)
|
||||
case .Wait_For_Byte: iter.pc += size_of(Opcode) + size_of(u8)
|
||||
case .Wait_For_Rune: iter.pc += size_of(Opcode) + size_of(rune)
|
||||
case .Wait_For_Rune_Class: iter.pc += size_of(Opcode) + size_of(u8)
|
||||
case .Wait_For_Rune_Class_Negated: iter.pc += size_of(Opcode) + size_of(u8)
|
||||
case .Match_All_And_Escape: iter.pc += size_of(Opcode)
|
||||
case:
|
||||
panic("Invalid opcode found in RegEx program.")
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
opcode_to_name :: proc(opcode: Opcode) -> (str: string) {
|
||||
switch opcode {
|
||||
case .Match: str = "Match"
|
||||
case .Match_And_Exit: str = "Match_And_Exit"
|
||||
case .Byte: str = "Byte"
|
||||
case .Rune: str = "Rune"
|
||||
case .Rune_Class: str = "Rune_Class"
|
||||
case .Rune_Class_Negated: str = "Rune_Class_Negated"
|
||||
case .Wildcard: str = "Wildcard"
|
||||
case .Jump: str = "Jump"
|
||||
case .Split: str = "Split"
|
||||
case .Save: str = "Save"
|
||||
case .Assert_Start: str = "Assert_Start"
|
||||
case .Assert_End: str = "Assert_End"
|
||||
case .Assert_Word_Boundary: str = "Assert_Word_Boundary"
|
||||
case .Assert_Non_Word_Boundary: str = "Assert_Non_Word_Boundary"
|
||||
case .Multiline_Open: str = "Multiline_Open"
|
||||
case .Multiline_Close: str = "Multiline_Close"
|
||||
case .Wait_For_Byte: str = "Wait_For_Byte"
|
||||
case .Wait_For_Rune: str = "Wait_For_Rune"
|
||||
case .Wait_For_Rune_Class: str = "Wait_For_Rune_Class"
|
||||
case .Wait_For_Rune_Class_Negated: str = "Wait_For_Rune_Class_Negated"
|
||||
case .Match_All_And_Escape: str = "Match_All_And_Escape"
|
||||
case: str = "<UNKNOWN>"
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,638 @@
|
||||
package regex_vm
|
||||
|
||||
@require import "core:io"
|
||||
import "core:text/regex/common"
|
||||
import "core:text/regex/parser"
|
||||
import "core:unicode/utf8"
|
||||
|
||||
Rune_Class_Range :: parser.Rune_Class_Range
|
||||
|
||||
// NOTE: This structure differs intentionally from the one in `regex/parser`,
|
||||
// as this data doesn't need to be a dynamic array once it hits the VM.
|
||||
Rune_Class_Data :: struct {
|
||||
runes: []rune,
|
||||
ranges: []Rune_Class_Range,
|
||||
}
|
||||
|
||||
Opcode :: enum u8 {
|
||||
// | [ operands ]
|
||||
Match = 0x00, // |
|
||||
Match_And_Exit = 0x01, // |
|
||||
Byte = 0x02, // | u8
|
||||
Rune = 0x03, // | i32
|
||||
Rune_Class = 0x04, // | u8
|
||||
Rune_Class_Negated = 0x05, // | u8
|
||||
Wildcard = 0x06, // |
|
||||
Jump = 0x07, // | u16
|
||||
Split = 0x08, // | u16, u16
|
||||
Save = 0x09, // | u8
|
||||
Assert_Start = 0x0A, // |
|
||||
Assert_End = 0x0B, // |
|
||||
Assert_Word_Boundary = 0x0C, // |
|
||||
Assert_Non_Word_Boundary = 0x0D, // |
|
||||
Multiline_Open = 0x0E, // |
|
||||
Multiline_Close = 0x0F, // |
|
||||
Wait_For_Byte = 0x10, // | u8
|
||||
Wait_For_Rune = 0x11, // | i32
|
||||
Wait_For_Rune_Class = 0x12, // | u8
|
||||
Wait_For_Rune_Class_Negated = 0x13, // | u8
|
||||
Match_All_And_Escape = 0x14, // |
|
||||
}
|
||||
|
||||
Thread :: struct {
|
||||
pc: int,
|
||||
saved: ^[2 * common.MAX_CAPTURE_GROUPS]int,
|
||||
}
|
||||
|
||||
Program :: []Opcode
|
||||
|
||||
Machine :: struct {
|
||||
// Program state
|
||||
memory: string,
|
||||
class_data: []Rune_Class_Data,
|
||||
code: Program,
|
||||
|
||||
// Thread state
|
||||
top_thread: int,
|
||||
threads: [^]Thread,
|
||||
next_threads: [^]Thread,
|
||||
|
||||
// The busy map is used to merge threads based on their program counters.
|
||||
busy_map: []u64,
|
||||
|
||||
// Global state
|
||||
string_pointer: int,
|
||||
|
||||
current_rune: rune,
|
||||
current_rune_size: int,
|
||||
next_rune: rune,
|
||||
next_rune_size: int,
|
||||
}
|
||||
|
||||
|
||||
// @MetaCharacter
|
||||
// NOTE: This must be kept in sync with the compiler & tokenizer.
|
||||
is_word_class :: #force_inline proc "contextless" (r: rune) -> bool {
|
||||
switch r {
|
||||
case '0'..='9', 'A'..='Z', '_', 'a'..='z':
|
||||
return true
|
||||
case:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
set_busy_map :: #force_inline proc "contextless" (vm: ^Machine, pc: int) -> bool #no_bounds_check {
|
||||
slot := cast(u64)pc >> 6
|
||||
bit: u64 = 1 << (cast(u64)pc & 0x3F)
|
||||
if vm.busy_map[slot] & bit > 0 {
|
||||
return false
|
||||
}
|
||||
vm.busy_map[slot] |= bit
|
||||
return true
|
||||
}
|
||||
|
||||
check_busy_map :: #force_inline proc "contextless" (vm: ^Machine, pc: int) -> bool #no_bounds_check {
|
||||
slot := cast(u64)pc >> 6
|
||||
bit: u64 = 1 << (cast(u64)pc & 0x3F)
|
||||
return vm.busy_map[slot] & bit > 0
|
||||
}
|
||||
|
||||
add_thread :: proc(vm: ^Machine, saved: ^[2 * common.MAX_CAPTURE_GROUPS]int, pc: int) #no_bounds_check {
|
||||
if check_busy_map(vm, pc) {
|
||||
return
|
||||
}
|
||||
|
||||
saved := saved
|
||||
pc := pc
|
||||
|
||||
resolution_loop: for {
|
||||
if !set_busy_map(vm, pc) {
|
||||
return
|
||||
}
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "Thread [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "] thinking about ")
|
||||
io.write_string(common.debug_stream, opcode_to_name(vm.code[pc]))
|
||||
io.write_rune(common.debug_stream, '\n')
|
||||
}
|
||||
|
||||
#partial switch vm.code[pc] {
|
||||
case .Jump:
|
||||
pc = cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode)])^
|
||||
continue
|
||||
|
||||
case .Split:
|
||||
jmp_x := cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode)])^
|
||||
jmp_y := cast(int)(cast(^u16)&vm.code[pc + size_of(Opcode) + size_of(u16)])^
|
||||
|
||||
add_thread(vm, saved, jmp_x)
|
||||
pc = jmp_y
|
||||
continue
|
||||
|
||||
case .Save:
|
||||
new_saved := new([2 * common.MAX_CAPTURE_GROUPS]int)
|
||||
new_saved ^= saved^
|
||||
saved = new_saved
|
||||
|
||||
index := vm.code[pc + size_of(Opcode)]
|
||||
sp := vm.string_pointer+vm.current_rune_size
|
||||
saved[index] = sp
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "Thread [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "] saving state: (slot ")
|
||||
io.write_int(common.debug_stream, cast(int)index)
|
||||
io.write_string(common.debug_stream, " = ")
|
||||
io.write_int(common.debug_stream, sp)
|
||||
io.write_string(common.debug_stream, ")\n")
|
||||
}
|
||||
|
||||
pc += size_of(Opcode) + size_of(u8)
|
||||
continue
|
||||
|
||||
case .Assert_Start:
|
||||
sp := vm.string_pointer+vm.current_rune_size
|
||||
if sp == 0 {
|
||||
pc += size_of(Opcode)
|
||||
continue
|
||||
}
|
||||
case .Assert_End:
|
||||
sp := vm.string_pointer+vm.current_rune_size
|
||||
if sp == len(vm.memory) {
|
||||
pc += size_of(Opcode)
|
||||
continue
|
||||
}
|
||||
case .Multiline_Open:
|
||||
sp := vm.string_pointer+vm.current_rune_size
|
||||
if sp == 0 || sp == len(vm.memory) {
|
||||
if vm.next_rune == '\r' || vm.next_rune == '\n' {
|
||||
// The VM is currently on a newline at the string boundary,
|
||||
// so consume the newline next frame.
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved }
|
||||
vm.top_thread += 1
|
||||
} else {
|
||||
// Skip the `Multiline_Close` opcode.
|
||||
pc += 2 * size_of(Opcode)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
// Not on a string boundary.
|
||||
// Try to consume a newline next frame in the other opcode loop.
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved }
|
||||
vm.top_thread += 1
|
||||
}
|
||||
case .Assert_Word_Boundary:
|
||||
sp := vm.string_pointer+vm.current_rune_size
|
||||
if sp == 0 || sp == len(vm.memory) {
|
||||
pc += size_of(Opcode)
|
||||
continue
|
||||
} else {
|
||||
last_rune_is_wc := is_word_class(vm.current_rune)
|
||||
this_rune_is_wc := is_word_class(vm.next_rune)
|
||||
|
||||
if last_rune_is_wc && !this_rune_is_wc || !last_rune_is_wc && this_rune_is_wc {
|
||||
pc += size_of(Opcode)
|
||||
continue
|
||||
}
|
||||
}
|
||||
case .Assert_Non_Word_Boundary:
|
||||
sp := vm.string_pointer+vm.current_rune_size
|
||||
if sp != 0 && sp != len(vm.memory) {
|
||||
last_rune_is_wc := is_word_class(vm.current_rune)
|
||||
this_rune_is_wc := is_word_class(vm.next_rune)
|
||||
|
||||
if last_rune_is_wc && this_rune_is_wc || !last_rune_is_wc && !this_rune_is_wc {
|
||||
pc += size_of(Opcode)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
case .Wait_For_Byte:
|
||||
operand := cast(rune)vm.code[pc + size_of(Opcode)]
|
||||
if vm.next_rune == operand {
|
||||
add_thread(vm, saved, pc + size_of(Opcode) + size_of(u8))
|
||||
}
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved }
|
||||
vm.top_thread += 1
|
||||
|
||||
case .Wait_For_Rune:
|
||||
operand := (cast(^rune)&vm.code[pc + size_of(Opcode)])^
|
||||
if vm.next_rune == operand {
|
||||
add_thread(vm, saved, pc + size_of(Opcode) + size_of(rune))
|
||||
}
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved }
|
||||
vm.top_thread += 1
|
||||
|
||||
case .Wait_For_Rune_Class:
|
||||
operand := cast(u8)vm.code[pc + size_of(Opcode)]
|
||||
class_data := vm.class_data[operand]
|
||||
next_rune := vm.next_rune
|
||||
|
||||
check: {
|
||||
for r in class_data.runes {
|
||||
if next_rune == r {
|
||||
add_thread(vm, saved, pc + size_of(Opcode) + size_of(u8))
|
||||
break check
|
||||
}
|
||||
}
|
||||
for range in class_data.ranges {
|
||||
if range.lower <= next_rune && next_rune <= range.upper {
|
||||
add_thread(vm, saved, pc + size_of(Opcode) + size_of(u8))
|
||||
break check
|
||||
}
|
||||
}
|
||||
}
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved }
|
||||
vm.top_thread += 1
|
||||
|
||||
case .Wait_For_Rune_Class_Negated:
|
||||
operand := cast(u8)vm.code[pc + size_of(Opcode)]
|
||||
class_data := vm.class_data[operand]
|
||||
next_rune := vm.next_rune
|
||||
|
||||
check_negated: {
|
||||
for r in class_data.runes {
|
||||
if next_rune == r {
|
||||
break check_negated
|
||||
}
|
||||
}
|
||||
for range in class_data.ranges {
|
||||
if range.lower <= next_rune && next_rune <= range.upper {
|
||||
break check_negated
|
||||
}
|
||||
}
|
||||
add_thread(vm, saved, pc + size_of(Opcode) + size_of(u8))
|
||||
}
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved }
|
||||
vm.top_thread += 1
|
||||
|
||||
case:
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = pc, saved = saved }
|
||||
vm.top_thread += 1
|
||||
}
|
||||
|
||||
break resolution_loop
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
run :: proc(vm: ^Machine, $UNICODE_MODE: bool) -> (saved: ^[2 * common.MAX_CAPTURE_GROUPS]int, ok: bool) #no_bounds_check {
|
||||
when UNICODE_MODE {
|
||||
vm.next_rune, vm.next_rune_size = utf8.decode_rune_in_string(vm.memory)
|
||||
} else {
|
||||
if len(vm.memory) > 0 {
|
||||
vm.next_rune = cast(rune)vm.memory[0]
|
||||
vm.next_rune_size = 1
|
||||
}
|
||||
}
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "### Adding initial thread.\n")
|
||||
}
|
||||
|
||||
{
|
||||
starter_saved := new([2 * common.MAX_CAPTURE_GROUPS]int)
|
||||
starter_saved ^= -1
|
||||
|
||||
add_thread(vm, starter_saved, 0)
|
||||
}
|
||||
|
||||
// `add_thread` adds to `next_threads` by default, but we need to put this
|
||||
// thread in the current thread buffer.
|
||||
vm.threads, vm.next_threads = vm.next_threads, vm.threads
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "### VM starting.\n")
|
||||
defer io.write_string(common.debug_stream, "### VM finished.\n")
|
||||
}
|
||||
|
||||
for {
|
||||
for i := 0; i < len(vm.busy_map); i += 1 {
|
||||
vm.busy_map[i] = 0
|
||||
}
|
||||
|
||||
assert(vm.string_pointer <= len(vm.memory), "VM string pointer went out of bounds.")
|
||||
|
||||
current_rune := vm.next_rune
|
||||
vm.current_rune = current_rune
|
||||
vm.current_rune_size = vm.next_rune_size
|
||||
when UNICODE_MODE {
|
||||
vm.next_rune, vm.next_rune_size = utf8.decode_rune_in_string(vm.memory[vm.string_pointer+vm.current_rune_size:])
|
||||
} else {
|
||||
if vm.string_pointer+size_of(u8) < len(vm.memory) {
|
||||
vm.next_rune = cast(rune)vm.memory[vm.string_pointer+size_of(u8)]
|
||||
vm.next_rune_size = size_of(u8)
|
||||
} else {
|
||||
vm.next_rune = 0
|
||||
vm.next_rune_size = 0
|
||||
}
|
||||
}
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, ">>> Dispatching rune: ")
|
||||
io.write_encoded_rune(common.debug_stream, current_rune)
|
||||
io.write_byte(common.debug_stream, '\n')
|
||||
}
|
||||
|
||||
thread_count := vm.top_thread
|
||||
vm.top_thread = 0
|
||||
thread_loop: for i := 0; i < thread_count; i += 1 {
|
||||
t := vm.threads[i]
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "Thread [PC:")
|
||||
common.write_padded_hex(common.debug_stream, t.pc, 4)
|
||||
io.write_string(common.debug_stream, "] stepping on ")
|
||||
io.write_string(common.debug_stream, opcode_to_name(vm.code[t.pc]))
|
||||
io.write_byte(common.debug_stream, '\n')
|
||||
}
|
||||
|
||||
#partial opcode: switch vm.code[t.pc] {
|
||||
case .Match:
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "Thread matched!\n")
|
||||
}
|
||||
saved = t.saved
|
||||
ok = true
|
||||
break thread_loop
|
||||
|
||||
case .Match_And_Exit:
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "Thread matched! (Exiting)\n")
|
||||
}
|
||||
return nil, true
|
||||
|
||||
case .Byte:
|
||||
operand := cast(rune)vm.code[t.pc + size_of(Opcode)]
|
||||
if current_rune == operand {
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8))
|
||||
}
|
||||
|
||||
case .Rune:
|
||||
operand := (cast(^rune)&vm.code[t.pc + size_of(Opcode)])^
|
||||
if current_rune == operand {
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(rune))
|
||||
}
|
||||
|
||||
case .Rune_Class:
|
||||
operand := cast(u8)vm.code[t.pc + size_of(Opcode)]
|
||||
class_data := vm.class_data[operand]
|
||||
|
||||
for r in class_data.runes {
|
||||
if current_rune == r {
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8))
|
||||
break opcode
|
||||
}
|
||||
}
|
||||
for range in class_data.ranges {
|
||||
if range.lower <= current_rune && current_rune <= range.upper {
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8))
|
||||
break opcode
|
||||
}
|
||||
}
|
||||
|
||||
case .Rune_Class_Negated:
|
||||
operand := cast(u8)vm.code[t.pc + size_of(Opcode)]
|
||||
class_data := vm.class_data[operand]
|
||||
for r in class_data.runes {
|
||||
if current_rune == r {
|
||||
break opcode
|
||||
}
|
||||
}
|
||||
for range in class_data.ranges {
|
||||
if range.lower <= current_rune && current_rune <= range.upper {
|
||||
break opcode
|
||||
}
|
||||
}
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8))
|
||||
|
||||
case .Wildcard:
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode))
|
||||
|
||||
case .Multiline_Open:
|
||||
if current_rune == '\n' {
|
||||
// UNIX newline.
|
||||
add_thread(vm, t.saved, t.pc + 2 * size_of(Opcode))
|
||||
} else if current_rune == '\r' {
|
||||
if vm.next_rune == '\n' {
|
||||
// Windows newline. (1/2)
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode))
|
||||
} else {
|
||||
// Mac newline.
|
||||
add_thread(vm, t.saved, t.pc + 2 * size_of(Opcode))
|
||||
}
|
||||
}
|
||||
case .Multiline_Close:
|
||||
if current_rune == '\n' {
|
||||
// Windows newline. (2/2)
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode))
|
||||
}
|
||||
|
||||
case .Wait_For_Byte:
|
||||
operand := cast(rune)vm.code[t.pc + size_of(Opcode)]
|
||||
if vm.next_rune == operand {
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8))
|
||||
}
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, t.pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = t.pc, saved = t.saved }
|
||||
vm.top_thread += 1
|
||||
|
||||
case .Wait_For_Rune:
|
||||
operand := (cast(^rune)&vm.code[t.pc + size_of(Opcode)])^
|
||||
if vm.next_rune == operand {
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(rune))
|
||||
}
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, t.pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = t.pc, saved = t.saved }
|
||||
vm.top_thread += 1
|
||||
|
||||
case .Wait_For_Rune_Class:
|
||||
operand := cast(u8)vm.code[t.pc + size_of(Opcode)]
|
||||
class_data := vm.class_data[operand]
|
||||
next_rune := vm.next_rune
|
||||
|
||||
check: {
|
||||
for r in class_data.runes {
|
||||
if next_rune == r {
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8))
|
||||
break check
|
||||
}
|
||||
}
|
||||
for range in class_data.ranges {
|
||||
if range.lower <= next_rune && next_rune <= range.upper {
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8))
|
||||
break check
|
||||
}
|
||||
}
|
||||
}
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, t.pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = t.pc, saved = t.saved }
|
||||
vm.top_thread += 1
|
||||
|
||||
case .Wait_For_Rune_Class_Negated:
|
||||
operand := cast(u8)vm.code[t.pc + size_of(Opcode)]
|
||||
class_data := vm.class_data[operand]
|
||||
next_rune := vm.next_rune
|
||||
|
||||
check_negated: {
|
||||
for r in class_data.runes {
|
||||
if next_rune == r {
|
||||
break check_negated
|
||||
}
|
||||
}
|
||||
for range in class_data.ranges {
|
||||
if range.lower <= next_rune && next_rune <= range.upper {
|
||||
break check_negated
|
||||
}
|
||||
}
|
||||
add_thread(vm, t.saved, t.pc + size_of(Opcode) + size_of(u8))
|
||||
}
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "*** New thread added [PC:")
|
||||
common.write_padded_hex(common.debug_stream, t.pc, 4)
|
||||
io.write_string(common.debug_stream, "]\n")
|
||||
}
|
||||
vm.next_threads[vm.top_thread] = Thread{ pc = t.pc, saved = t.saved }
|
||||
vm.top_thread += 1
|
||||
|
||||
case .Match_All_And_Escape:
|
||||
t.pc += size_of(Opcode)
|
||||
// The point of this loop is to walk out of wherever this
|
||||
// opcode lives to the end of the program, while saving the
|
||||
// index to the length of the string at each pass on the way.
|
||||
escape_loop: for {
|
||||
#partial switch vm.code[t.pc] {
|
||||
case .Match, .Match_And_Exit:
|
||||
break escape_loop
|
||||
|
||||
case .Jump:
|
||||
t.pc = cast(int)(cast(^u16)&vm.code[t.pc + size_of(Opcode)])^
|
||||
|
||||
case .Save:
|
||||
index := vm.code[t.pc + size_of(Opcode)]
|
||||
t.saved[index] = len(vm.memory)
|
||||
t.pc += size_of(Opcode) + size_of(u8)
|
||||
|
||||
case .Match_All_And_Escape:
|
||||
// Layering these is fine.
|
||||
t.pc += size_of(Opcode)
|
||||
|
||||
// If the loop has to process any opcode not listed above,
|
||||
// it means someone did something odd like `a(.*$)b`, in
|
||||
// which case, just fail. Technically, the expression makes
|
||||
// no sense.
|
||||
case:
|
||||
break opcode
|
||||
}
|
||||
}
|
||||
|
||||
saved = t.saved
|
||||
ok = true
|
||||
return
|
||||
|
||||
case:
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "Opcode: ")
|
||||
io.write_int(common.debug_stream, cast(int)vm.code[t.pc])
|
||||
io.write_string(common.debug_stream, "\n")
|
||||
}
|
||||
panic("Invalid opcode in RegEx thread loop.")
|
||||
}
|
||||
}
|
||||
|
||||
vm.threads, vm.next_threads = vm.next_threads, vm.threads
|
||||
|
||||
when common.ODIN_DEBUG_REGEX {
|
||||
io.write_string(common.debug_stream, "<<< Frame ended. (Threads: ")
|
||||
io.write_int(common.debug_stream, vm.top_thread)
|
||||
io.write_string(common.debug_stream, ")\n")
|
||||
}
|
||||
|
||||
if vm.string_pointer == len(vm.memory) || vm.top_thread == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
vm.string_pointer += vm.current_rune_size
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
opcode_count :: proc(code: Program) -> (opcodes: int) {
|
||||
iter := Opcode_Iterator{ code, 0 }
|
||||
for _ in iterate_opcodes(&iter) {
|
||||
opcodes += 1
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
create :: proc(code: Program, str: string) -> (vm: Machine) {
|
||||
assert(len(code) > 0, "RegEx VM has no instructions.")
|
||||
|
||||
vm.memory = str
|
||||
vm.code = code
|
||||
|
||||
sizing := len(code) >> 6 + (1 if len(code) & 0x3F > 0 else 0)
|
||||
assert(sizing > 0)
|
||||
vm.busy_map = make([]u64, sizing)
|
||||
|
||||
max_possible_threads := max(1, opcode_count(vm.code) - 1)
|
||||
|
||||
vm.threads = make([^]Thread, max_possible_threads)
|
||||
vm.next_threads = make([^]Thread, max_possible_threads)
|
||||
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user