Compare commits

..

36 Commits

Author SHA1 Message Date
9bc4b9e8de EOI: Lecture 14 2022-07-24 23:00:55 -04:00
88644a95df EOI: Lecture 13 complete 2022-07-24 10:33:26 -04:00
a6c99fcfa6 EOI : lecture 12 compelte 2022-07-24 09:50:38 -04:00
bd6e39ecdd EOI: Lecture 10 & 11 complete. 2022-07-24 07:54:17 -04:00
ba53395fc7 EOI: Lecture 8 complete 2022-07-24 04:36:18 -04:00
e120749a7e EOI: Lecture 7 complete 2022-07-24 02:56:37 -04:00
de420a8111 Lecture 6 complete. 2022-07-20 14:57:26 -04:00
f85c9615e4 EoI : Lectures 1-5 complete 2022-07-20 09:27:19 -04:00
Ed
9eba81139d setup for future courses 2022-07-17 21:46:00 -04:00
a53e735b74 Fixed comments. 2022-07-17 15:28:57 -04:00
856a1faa9c Added support for comments. 2022-07-17 11:27:10 -04:00
31f1ae9b8f Fixes, also added support for spaces. (RegM) 2022-07-17 11:04:02 -04:00
2041732e28 Fleshed out the UI for RegM, fixes to SRegEx 2022-07-17 10:18:24 -04:00
77e9e0431e Fixed some formatting in notes. 2022-07-17 07:39:06 -04:00
17c3b8fe36 SRegEx works!!!!
Its not a full flegged transpiler but it works at least on RDP's lexer. I can expand on demand.
2022-07-17 07:32:57 -04:00
5ae405e284 Worked on SRegex transpiler to RegEx, 2022-07-17 03:09:42 -04:00
7197ef4262 reformated ascii state digrams to use spaces only. 2022-07-16 20:01:13 -04:00
548c2a3032 testing to see if editorconfig influences markdown 2022-07-16 19:58:42 -04:00
0dbc2c04ba RegEx : Complted lectures 1-7. NOT TESTED. 2022-07-16 19:55:57 -04:00
d48610d5b8 ui changes. 2022-07-14 17:45:24 -04:00
d4ee6574b2 Renamed BAPFS -> RDP, RDP completed. 2022-07-14 17:12:25 -04:00
0acd6ecbaf BAPFS Lecture 18
Instead of making a cli I made updates the gui to open any file.
2022-07-14 01:55:31 -04:00
8ad16eb886 BAPFS Lecture 17 2022-07-14 01:25:31 -04:00
83b0ee2974 BAPFS Lecture 16 2022-07-14 00:35:32 -04:00
37e4b8874e BAPFS Lecture 15 2022-07-13 22:09:54 -04:00
8117abedc9 Dev update 2 2022-07-12 18:15:51 -04:00
2b982bda84 Dev env update 2022-07-12 04:23:21 -04:00
5e3fd7a13c BAPFS Lecture 14 2022-07-12 03:48:08 -04:00
6832deb5ad BAPFS Lecture 13 2022-07-12 02:47:32 -04:00
948974857a Corrections to Lecture 12, fixing some regex 2022-07-12 01:38:54 -04:00
7544c4bedc BAPFS Lecture 12
I reorganized the functions by their precedence.
2022-07-12 01:25:47 -04:00
d34ccb04d2 BAPFS Lecture 11 2022-07-11 01:05:33 -04:00
dbd84c8921 BAPFS Lecture 10 2022-07-10 23:59:43 -04:00
0ec2581a46 BAPFS Lecture 9 2022-07-10 21:53:55 -04:00
601cc21bab BAPFS Lecture 8 2022-07-10 21:12:41 -04:00
19cc68a5eb BAPFS Lecture 7. 2022-07-10 19:27:28 -04:00
116 changed files with 16930 additions and 236 deletions

View File

@ -1,3 +1,7 @@
[*.zig] [*.gd]
indent_style = space indent_style = tab
indent_size = 4 indent_size = 4
[*.md]
indent_style = tab
indent_size = 4

18
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,18 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "GDScript Godot",
"type": "godot",
"request": "launch",
"project": "${workspaceFolder}/App",
"port": 6007,
"address": "127.0.0.1",
"launch_game_instance": true,
"launch_scene": false
}
]
}

View File

@ -1,3 +1,4 @@
{ {
"editor.formatOnType": true "editor.formatOnType": true,
"godot_tools.editor_path": "p:\\Study\\LangStudies\\Engine\\gd\\bin\\godot.windows.opt.tools.64.exe"
} }

Binary file not shown.

After

Width:  |  Height:  |  Size: 193 KiB

View File

@ -0,0 +1,35 @@
[remap]
importer="texture"
type="StreamTexture"
path="res://.import/EoI_Class_Cover.png-180761d77f8e0fd8967a2d9e7fc7462a.stex"
metadata={
"vram_texture": false
}
[deps]
source_file="res://Assets/Branding/EoI_Class_Cover.png"
dest_files=[ "res://.import/EoI_Class_Cover.png-180761d77f8e0fd8967a2d9e7fc7462a.stex" ]
[params]
compress/mode=0
compress/lossy_quality=0.7
compress/hdr_mode=0
compress/bptc_ldr=0
compress/normal_map=0
flags/repeat=0
flags/filter=true
flags/mipmaps=false
flags/anisotropic=false
flags/srgb=2
process/fix_alpha_border=true
process/premult_alpha=false
process/HDR_as_SRGB=false
process/invert_color=false
process/normal_map_invert_y=false
stream=false
size_limit=0
detect_3d=true
svg/scale=1.0

View File

Before

Width:  |  Height:  |  Size: 495 KiB

After

Width:  |  Height:  |  Size: 495 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 467 KiB

View File

@ -0,0 +1,35 @@
[remap]
importer="texture"
type="StreamTexture"
path="res://.import/RegM_Class_cover_small.png-9128ac026427f18f59811eadf663fe9b.stex"
metadata={
"vram_texture": false
}
[deps]
source_file="res://Assets/Branding/RegM_Class_cover_small.png"
dest_files=[ "res://.import/RegM_Class_cover_small.png-9128ac026427f18f59811eadf663fe9b.stex" ]
[params]
compress/mode=0
compress/lossy_quality=0.7
compress/hdr_mode=0
compress/bptc_ldr=0
compress/normal_map=0
flags/repeat=0
flags/filter=true
flags/mipmaps=false
flags/anisotropic=false
flags/srgb=2
process/fix_alpha_border=true
process/premult_alpha=false
process/HDR_as_SRGB=false
process/invert_color=false
process/normal_map_invert_y=false
stream=false
size_limit=0
detect_3d=true
svg/scale=1.0

View File

@ -1,4 +1,4 @@
[gd_resource type="StyleBoxFlat" format=2] [gd_resource type="StyleBoxFlat" format=2]
[resource] [resource]
bg_color = Color( 0.0941176, 0.0666667, 0.137255, 1 ) bg_color = Color( 0.12549, 0.0823529, 0.0862745, 1 )

View File

@ -0,0 +1,4 @@
[gd_resource type="StyleBoxFlat" format=2]
[resource]
bg_color = Color( 0.12549, 0.109804, 0.164706, 1 )

View File

@ -0,0 +1,4 @@
[gd_resource type="StyleBoxFlat" format=2]
[resource]
bg_color = Color( 0.101961, 0.101961, 0.101961, 1 )

View File

@ -0,0 +1,4 @@
[gd_resource type="StyleBoxFlat" format=2]
[resource]
bg_color = Color( 0.164706, 0.109804, 0.117647, 1 )

View File

@ -1,9 +1,11 @@
[gd_resource type="Theme" load_steps=3 format=2] [gd_resource type="Theme" load_steps=4 format=2]
[ext_resource path="res://Assets/Styles/Editor.SytleBoxFlat.tres" type="StyleBox" id=1] [ext_resource path="res://Assets/Styles/Editor.Purple.SytleBoxFlat.tres" type="StyleBox" id=1]
[ext_resource path="res://Assets/Fonts/DF_RecMonoSemiCasul.tres" type="DynamicFont" id=2] [ext_resource path="res://Assets/Fonts/DF_RecMonoSemiCasul.tres" type="DynamicFont" id=2]
[ext_resource path="res://Assets/Styles/Editor.SytleBoxFlat.Inactive.tres" type="StyleBox" id=3]
[resource] [resource]
Button/styles/disabled = ExtResource( 3 )
TextEdit/colors/font_color = Color( 1, 1, 1, 1 ) TextEdit/colors/font_color = Color( 1, 1, 1, 1 )
TextEdit/colors/font_color_readonly = Color( 1, 1, 1, 1 ) TextEdit/colors/font_color_readonly = Color( 1, 1, 1, 1 )
TextEdit/fonts/font = ExtResource( 2 ) TextEdit/fonts/font = ExtResource( 2 )

View File

@ -0,0 +1,15 @@
[gd_resource type="Theme" load_steps=5 format=2]
[ext_resource path="res://Assets/Styles/Editor.Wine.SytleBoxFlat.tres" type="StyleBox" id=1]
[ext_resource path="res://Assets/Fonts/DF_RecMonoSemiCasul.tres" type="DynamicFont" id=2]
[ext_resource path="res://Assets/Styles/Editor.SytleBoxFlat.Inactive.tres" type="StyleBox" id=3]
[ext_resource path="res://Assets/Styles/Editor.DarkWine.SytleBoxFlat.tres" type="StyleBox" id=4]
[resource]
Button/styles/disabled = ExtResource( 3 )
Panel/styles/panel = ExtResource( 1 )
TextEdit/colors/font_color = Color( 1, 1, 1, 1 )
TextEdit/colors/font_color_readonly = Color( 1, 1, 1, 1 )
TextEdit/fonts/font = ExtResource( 2 )
TextEdit/styles/normal = ExtResource( 4 )
TextEdit/styles/read_only = ExtResource( 4 )

View File

View File

115
App/EoI/EoI_Viewer.tscn Normal file
View File

@ -0,0 +1,115 @@
[gd_scene load_steps=5 format=2]
[ext_resource path="res://Assets/Styles/Purple.EditorTheme.tres" type="Theme" id=1]
[ext_resource path="res://Assets/Branding/EoI_Class_Cover.png" type="Texture" id=2]
[ext_resource path="res://EoI/Scripts/EoI_Viewer.gd" type="Script" id=4]
[ext_resource path="res://Assets/Styles/Wine.EditorTheme.tres" type="Theme" id=5]
[node name="EoI_Panel" type="Panel"]
anchor_right = 1.0
anchor_bottom = 1.0
theme = ExtResource( 5 )
script = ExtResource( 4 )
[node name="CourseBrand" type="TextureRect" parent="."]
anchor_right = 1.0
anchor_bottom = 1.0
rect_scale = Vector2( 0.2, 0.2 )
texture = ExtResource( 2 )
expand = true
stretch_mode = 6
[node name="Letter_FDialog" type="FileDialog" parent="."]
anchor_left = 0.35
anchor_top = 0.15
anchor_right = 0.45
anchor_bottom = 0.25
margin_right = 356.0
margin_bottom = 373.0
theme = ExtResource( 1 )
popup_exclusive = true
window_title = "Open a File"
mode = 0
access = 2
[node name="VBox" type="VBoxContainer" parent="."]
anchor_top = 0.196
anchor_right = 0.2
anchor_bottom = 1.0
margin_top = 2.39999
margin_right = -1.8
margin_bottom = -2.0
[node name="Eva_Interpret_Btn" type="Button" parent="VBox"]
margin_right = 203.0
margin_bottom = 27.0
rect_pivot_offset = Vector2( -123, -302 )
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
theme = ExtResource( 1 )
text = "Eva: Interpret"
[node name="Eva_ResetEnv_Btn" type="Button" parent="VBox"]
margin_top = 31.0
margin_right = 203.0
margin_bottom = 59.0
rect_pivot_offset = Vector2( -123, -302 )
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
theme = ExtResource( 1 )
text = "Eva: Reset Enviornment"
[node name="ClearOutput_Btn" type="Button" parent="VBox"]
margin_top = 63.0
margin_right = 203.0
margin_bottom = 91.0
rect_pivot_offset = Vector2( -123, -302 )
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
theme = ExtResource( 1 )
text = "Clear Output"
[node name="Separator" type="HSeparator" parent="VBox"]
modulate = Color( 0.145098, 0.145098, 0.164706, 0 )
margin_top = 95.0
margin_right = 203.0
margin_bottom = 445.0
size_flags_vertical = 15
theme = ExtResource( 5 )
[node name="Back_Btn" type="Button" parent="VBox"]
margin_top = 449.0
margin_right = 203.0
margin_bottom = 478.0
rect_pivot_offset = Vector2( -123, -302 )
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
theme = ExtResource( 1 )
text = "Course Directory"
[node name="Editor_TEdit" type="TextEdit" parent="."]
anchor_left = 0.2
anchor_right = 0.625
anchor_bottom = 0.8
theme = ExtResource( 5 )
show_line_numbers = true
draw_tabs = true
highlight_all_occurrences = true
smooth_scrolling = true
minimap_draw = true
[node name="Output_TEdit" type="TextEdit" parent="."]
anchor_left = 0.2
anchor_top = 0.8
anchor_right = 0.625
anchor_bottom = 1.0
margin_left = 0.199997
theme = ExtResource( 5 )
readonly = true
[node name="Debug_TEdit" type="TextEdit" parent="."]
anchor_left = 0.625
anchor_right = 1.0
anchor_bottom = 1.0
theme = ExtResource( 5 )
readonly = true

View File

@ -0,0 +1,110 @@
# Pipeline
Interpretation Involves **RUNTIME SEMANTICS**
***THE MEANING OF THE PROGRAM MODEL***
Compliation delegates semantics of runtime behavior to
***a TARGET language***
Interpetation deals with the semantics itself.
Types of Interpreter **implementation**:
* AST-based (recursive)
* Bytecode (Virtual Machines)
Types of Compiler **implementation**:
* Ahead-of-time (AOT)
* Just-in-time (JIT)
## Interpeter-AST:
**Translation Stage:**
1. Program Model
2. Lexer: Processing into token elements for a parser.
* Output : Tokens
3. Parser: Syntactic Analysis
* Output : Abstract Syntax Tree (AST)
**Runtime Stage:**
4. Interpeter
5. Runtime Behavior
## Interpreter-Bytecode:
**Translation Stage:**
1. Program Model
2. Lexer: Processing into token elements for a parser.
* Output : Tokens
3. Parser: Syntactic Analysis
* Output : Abstract Syntax Tree (AST)
4. Bytecode Emitter
5. Bytecode instructions primed.
**Runtime Stage:**
6. Interpreter
7. Runtime Behavior
**Types of Virtual Machine behavior:**
* Stack based
* Stack for operands and operators
* Result is always on top of stack
* Register based
* Virtual registers
* Result in accumulation register
* Map to real via register allocation
## Compiler Ahead-of-Time:
1. Program Model
2. Lexer: Processing into token elements for a parser.
* Output : Tokens
3. Parser: Syntactic Analysis
* Output : Abstract Syntax Tree (AST)
4. Code Generator
5. Intermediate representation primed
6. Target machine instruction set code generation
7. Target machine is intended interpretation platform.
8. Runtime Behavior.
## Compiler with LLVM platform:
1. Program Model
2. Lexer: Processing into token elements for a parser.
* Output : Tokens
3. Parser: Syntactic Analysis
* Output : Abstract Syntax Tree (AST)
4. LLVM IR generator
5. LLVM Native code generator
6. Target machine is intended interpretation platform
7. Runtime Behavior.
Lexer, and parser are considered **FRONT-END**.
Code Generation or byte-code gen or interpreter ast impelementation gen
for target instruction platform is considered **BACK-END**.
## Jit Compiler:
1. Program Model
2. Lexer: Processing into token elements for a parser.
* Output : Tokens
3. Parser: Syntactic Analysis
* Output : Abstract Syntax Tree (AST)
4. Bytecode Emitter
5. Bytecode fed to interpeter
6. Interetor may code gen immediately to target hardware platform or interpret ast directly.
7. Runtime Behavior.
## Transpiler:
1. Program Model in input langauge
2. Lexer: Processing into token elements for a parser.
* Output : Tokens
3. Parser: Syntactic Analysis
* Output : Abstract Syntax Tree (AST)
4. AST Transformation to target AST
5. Code generation
6. Program Model in output langauge

View File

@ -0,0 +1,45 @@
# Eva Programming Langauge
Dynamic programming langauge.
Simple syntax, functional heart, OOP support.
## Eva Expressions:
```
(<type> <op1> <op2> ... <opN>)
```
Example:
```
(+ 5 10)
(set x 15)
(if (> x 10)
(print "ok")
(print "error")
)
```
```
(def foo (bar)
(+ bar 10)
)
```
```
(lambda (x) (* x x) 10)
```
## Design Goals
* Simple syntax: S-Expression
* Everything is an expression
* No explicit return, last evalulated expression is the result
* First class functions
* Static scope: all functions are closures
* Lambda functions
* Funcitonal programming
* Imperative programming
* Namespaces and modules
* OOP: Class or prototype based.

View File

@ -0,0 +1,13 @@
# Environment
A repository of variables and functions defined in a scope.
## Structure
Record: Actual storage, table, etc.
Parent: Optional refrence to a parent environment.
## Interface
Define: Variable
Assign: Value to a varaible
Lookup: Variable

View File

@ -0,0 +1,55 @@
extends Node
# Eva -------------------------------------------------------
const SLexer = preload("Lexer.gd")
var Lexer : SLexer
const SParser = preload("Parser.gd")
var Parser : SParser
const SEva = preload("Eva.gd")
var Eva : SEva
# UX --------------------------------------------------------
onready var Editor = get_node("Editor_TEdit")
onready var Output = get_node("Output_TEdit")
onready var Debug = get_node("Debug_TEdit")
onready var Eva_Btn = get_node("VBox/Eva_Interpret_Btn")
onready var Eva_Reset_Btn = get_node("VBox/Eva_ResetEnv_Btn")
onready var Clear_Btn = get_node("VBox/ClearOutput_Btn")
onready var Back_Btn = get_node("VBox/Back_Btn")
func evaBtn_pressed():
Lexer = SLexer.new(Editor.text, Output)
Parser = SParser.new(Lexer, Output)
var ast = Parser.parse()
var result = Eva.eval(ast)
if result != null:
Output.text += "\nResult: " + result
Debug.text = JSON.print( Eva.get_EnvSnapshot(), "\t" )
func evaResetBtn_pressed():
Eva = SEva.new(null, Output)
Debug.text = JSON.print( Eva.get_EnvSnapshot(), "\t" )
func clearBtn_pressed():
Output.text = ""
func backBtn_pressed():
queue_free()
func _ready():
Eva = SEva.new(null, Output)
Debug.text = JSON.print( Eva.get_EnvSnapshot(), "\t" )
Eva_Btn.connect("pressed", self, "evaBtn_pressed")
Eva_Reset_Btn.connect("pressed", self, "evaResetBtn_pressed")
Clear_Btn.connect("pressed", self, "clearBtn_pressed")
Back_Btn.connect("pressed", self, "backBtn_pressed")

303
App/EoI/Scripts/Eva.gd Normal file
View File

@ -0,0 +1,303 @@
extends Object
# ---------------------------------------------------------- UTILITIES
var EvalOut
func check( condition : bool, message : String):
assert(condition, message)
if ! condition:
EvalOut.text = "Eva - Error: " + message
func throw( message ):
assert(false, message)
EvalOut.text = "Eva - Error: " + message
# ---------------------------------------------------------- UTILITIES END
class_name Eva
# ---------------------------------------------------------- GLOBALS
const Parser = preload("Parser.gd")
const NType = Parser.NType
const EvaEnv = preload("EvaEnv.gd")
var Env : EvaEnv
var Parent
# ---------------------------------------------------------- GLOBALS END
func get_class():
return "Eva"
func _init(parent, evalOut):
EvalOut = evalOut
Env = EvaEnv.new(EvalOut)
Parent = parent
if Parent == null:
Env.setup_Globals()
func eval( ast ):
match ast.type():
NType.program :
var index = 1;
while index < ast.num_args():
eval( ast.arg(index) )
index += 1
var result = eval( ast.arg(index) )
if result != null:
if typeof(result) == TYPE_OBJECT && result.get_class() == "ASTNode":
return JSON.print(result.to_SExpression())
return String( result )
else:
return null
NType.block :
return eval_Block( ast )
NType.conditional :
var condition = eval( ast.arg(1) )
if condition:
# consequent
return eval( ast.arg(2) )
# Alternate
if ast.num_args() > 2:
return eval( ast.arg(3))
NType.expr_Switch:
var index = 1
while ast.arg(index).is_op_Relation():
if eval( ast.arg(index) ):
return eval( ast.arg(index + 1) )
index += 2
return eval( ast.arg(index) )
NType.expr_While :
var result
while eval( ast.arg(1) ):
result = eval( ast.arg(2) )
return result
NType.expr_For:
var forEva = get_script().new( self, EvalOut )
forEva.eval( ast.arg(1) )
var index = 3; var result
while forEva.eval( ast.arg(2) ) :
result = forEva.eval( ast.arg(index) )
index += 1
if index > ast.num_args() :
index = 3
return result
NType.fn_User :
var symbol = ast.arg(1)
var fnDef = \
[
ast.arg(2), # Parameters
ast.arg(3), # Body
self # Closure (Environment capture)
]
Env.define(symbol, fnDef)
return Env.lookup(symbol)
NType.fn_Lambda:
var fnDef = \
[
ast.arg(1), # Parameters
ast.arg(2), # Body
self # Closure (Environment capture)
]
return fnDef
NType.fn_IIL:
var params = ast.arg(1).arg(1)
var body = ast.arg(1).arg(2)
var fnEva = get_script().new( self, EvalOut )
if params.type() != NType.empty:
var index = 1
while index <= params.num_args():
var paramVal = eval( ast.arg(index + 1) )
fnEva.Env.define(params.arg(index), paramVal )
index += 1
var result
var index = 1;
while index <= body.num_args() :
result = fnEva.eval( body.arg( index ) )
index += 1
return result
NType.identifier :
return eval_Lookup( ast )
NType.op_Assign :
return eval_Assign( ast )
NType.op_Fn:
return eval_Func( ast )
NType.op_Add:
var result = 0.0; var index = 1
while index <= ast.num_args():
result += eval( ast.arg(index) )
index += 1
return result
NType.op_Sub:
if ast.num_args() < 2:
return -eval( ast.arg(1) )
var result = eval( ast.arg(1) ); var index = 2
while index <= ast.num_args():
result -= eval( ast.arg(index) )
index += 1
return result
NType.op_Mult:
var result = 1.0; var index = 1
while index <= ast.num_args():
result *= eval( ast.arg(index) )
index += 1
return result
NType.op_Div:
var result = eval( ast.arg(1) ); var index = 2
while index <= ast.num_args():
result /= eval( ast.arg(index) )
index += 1
return result
NType.op_Increment:
return eval( ast.arg(1) ) + 1
NType.op_Decrement:
return eval( ast.arg(1) ) - 1
NType.op_Greater:
return eval( ast.arg(1) ) > eval( ast.arg(2) )
NType.op_Lesser:
return eval( ast.arg(1) ) < eval( ast.arg(2) )
NType.op_GreaterEqual:
return eval( ast.arg(1) ) >= eval( ast.arg(2) )
NType.op_LesserEqual:
return eval( ast.arg(1) ) <= eval( ast.arg(2) )
NType.op_Equal:
return eval( ast.arg(1) ) == eval( ast.arg(2) )
NType.op_NotEqual:
return eval( ast.arg(1) ) != eval( ast.arg(2) )
NType.fn_Print :
return eval_Print( ast )
NType.variable :
var symbol = ast.arg(1)
var value
if ast.num_args() == 2:
value = eval( ast.arg(2) )
Env.define(symbol, value)
return Env.lookup(symbol)
if ast.is_Number() :
return float( ast.arg(1) )
elif ast.is_String() :
return ast.string()
var msgT = "eval - Unimplemented: {ast}"
var msg = msgT.format({"ast" : JSON.print(ast.to_SExpression(), "\t") })
throw(msg)
func eval_Block( ast ):
var eva_Block = get_script().new( self, EvalOut )
var result
var index = 1;
while index <= ast.num_args() :
result = eva_Block.eval( ast.arg(index) )
index += 1
return result
func eval_Lookup( ast ) :
var identifier = ast.arg(1)
if Parent != null && !Env.has( identifier):
return Parent.eval_Lookup( ast )
return Env.lookup( identifier )
func eval_Assign( ast, oriEva = null ) :
var symbol = ast.arg(1)
if Parent != null && !Env.has( symbol):
return Parent.eval_Assign( ast, self )
var value
if oriEva != null :
value = oriEva.eval( ast.arg(2) )
else :
value = eval( ast.arg(2) )
return Env.set( symbol, value )
func eval_Func( ast ):
var fn = eval_Lookup( ast )
var params = fn[0]
var body = fn[1]
var fnEva = get_script().new( fn[2], EvalOut )
if params.type() != NType.empty:
var index = 1
while index <= params.num_args():
var paramVal = eval( ast.arg(index + 1) )
fnEva.Env.define(params.arg(index), paramVal )
index += 1
var result
var index = 1;
while index <= body.num_args() :
result = fnEva.eval( body.arg( index ) )
index += 1
return result
func eval_Print( ast ):
EvalOut.text += "\n" + String( eval( ast.arg(1) ) )
return null
func get_EnvSnapshot():
var \
snapshot = EvaEnv.new(EvalOut)
snapshot.Records = Env.Records.duplicate(true)
if Parent != null:
snapshot[Parent] = Parent.get_EnvSnapshot()
return snapshot.to_Dictionary()

119
App/EoI/Scripts/EvaEnv.gd Normal file
View File

@ -0,0 +1,119 @@
extends Object
# ---------------------------------------------------------- UTILITIES
var ErrorOut
func check( condition : bool, message : String):
assert(condition, message)
if ! condition:
ErrorOut.text = "Eva - Error: " + message
func throw( message ):
assert(false, message)
ErrorOut.text = "Eva - Error: " + message
# ---------------------------------------------------------- UTILITIES END
class_name EvaEnv
var Records : Dictionary
func define(symbol : String, value) :
Records[symbol] = value
func has(symbol : String) :
return Records.has(symbol)
func lookup(symbol : String) :
check(Records.has(symbol), String("Symbol not found in environment records"))
return Records[symbol]
func set(symbol : String, value) :
check(Records.has(symbol), String("Symbol not found in environment records"))
Records[symbol] = value
return Records[symbol]
func setup_Globals():
Records["null"] = null
Records["true"] = true
Records["false"] = false
func _init(errorOut):
ErrorOut = errorOut
# Serialization ----------------------------------------------------
var SEva
func array_Serialize(array, fn_objSerializer) :
var result = []
for entry in array :
if typeof(entry) == TYPE_ARRAY :
result.append( array_Serialize( entry, fn_objSerializer ))
elif typeof(entry) == TYPE_OBJECT :
if entry.get_class() == "Eva":
result.append(entry)
else:
fn_objSerializer.set_instance(entry)
result.append( fn_objSerializer.call_func() )
else :
result.append( entry )
return result
func to_SExpression():
var expression = []
for key in Records.keys() :
var entry = [key]
var Value = Records[key]
if typeof( Value ) == TYPE_ARRAY :
var \
to_SExpression_Fn = FuncRef.new()
to_SExpression_Fn.set_function("to_SExpression")
var array = array_Serialize( Value, to_SExpression_Fn )
entry.append(array)
elif typeof( Value ) == TYPE_OBJECT :
entry.append( Value.to_SExpression() )
else :
entry.append(Value)
expression.append(entry)
return expression
func to_Dictionary():
var result = {}
for key in Records.keys() :
var Value = Records[key]
if typeof(Value) == TYPE_ARRAY :
var \
to_SExpression_Fn = FuncRef.new()
to_SExpression_Fn.set_function("to_SExpression")
var array = array_Serialize( Value, to_SExpression_Fn )
result[key] = array
elif typeof(Value) == TYPE_OBJECT :
result[key] = Value.to_SExpression()
else :
result[key] = Value
return result
# Serialization END -------------------------------------------------

183
App/EoI/Scripts/Lexer.gd Normal file
View File

@ -0,0 +1,183 @@
extends Object
# ---------------------------------------------------------- UTILITIES
var ErrorOut
func check( condition : bool, message : String):
assert(condition, message)
if ! condition:
ErrorOut.text = "Eva - Error: " + message
func throw( message ):
assert(false, message)
ErrorOut.text = "Eva - Error: " + message
# ---------------------------------------------------------- UTILITIES END
class_name Lexer
var SRegEx = preload("res://RegM/Scripts/SRegex.gd").new()
const TType : Dictionary = \
{
fmt_S = "Formatting",
cmt_SL = "Comment Single-Line",
cmt_ML = "Comment Multi-Line",
def_Start = "Expression Start",
def_End = "Expression End",
def_Block = "Expression Block Start",
def_Cond = "Expression Conditional",
def_Switch = "Expresssion Switch",
def_While = "Expression While",
def_For = "Expression For",
def_Var = "Variable Declaration",
def_Func = "Function Declaration",
def_Lambda = "Lambda Declaration",
literal_Number = "Literal: Number",
literal_String = "Literal: String",
op_Assgin = "Assignment",
op_Numeric = "Numeric Operation",
op_Relational = "Relational Operation",
op_Equality = "Equality Operation",
fn_Print = "Print",
identifier = "Identifier"
}
const Spec : Dictionary = \
{
TType.cmt_SL : "start // inline.repeat(0-)",
TType.cmt_ML : "start /* set(whitespace !whitespace).repeat(0-).lazy */",
TType.fmt_S : "start whitespace.repeat(1-).lazy",
TType.def_Start : "start \\(",
TType.def_End : "start \\)",
TType.def_Block : "start \"begin\"",
TType.def_Cond : "start \"if\"",
TType.def_Switch : "start \"switch\"",
TType.def_While : "start \"while\"",
TType.def_For : "start \"for\"",
TType.def_Var : "start \"var\"",
TType.def_Func : "start \"def\"",
TType.def_Lambda : "start \"lambda\"",
TType.literal_Number : \
"""start
set(+ \\-).repeat(0-1)
( set(0-9).repeat(1-) \\. ).repeat(0-1)
set(0-9).repeat(1-)
""",
TType.literal_String : "start \\\" !set( \\\" ).repeat(0-) \\\" ",
TType.op_Assgin : "start \"set\"",
TType.op_Numeric : "start set(+ \\- * /) set(+ \\-).repeat(0-1)",
TType.op_Relational : "start set(> <) =.repeat(0-1)",
TType.op_Equality : "start \\!.repeat(0-1) =",
TType.fn_Print : "start \"print\"",
TType.identifier :
"""start
(
set(A-z).repeat(1-)
set(\\- _).repeat(0-1)
)
.repeat(0-1)
"""
}
class Token:
var Type : String
var Value : String
func is_Literal():
return Type == TType.literal_Number || Type == TType.literal_String;
var SourceText : String
var Cursor : int
var SpecRegex : Dictionary
var Tokens : Array
var TokenIndex : int = 0
func compile_regex():
for type in TType.values() :
var regex = RegEx.new()
var result = SRegEx.compile(Spec[type])
regex.compile( result )
SpecRegex[type] = regex
func next_Token():
var nextToken = null
if Tokens.size() > TokenIndex :
nextToken = Tokens[TokenIndex]
TokenIndex += 1
return nextToken
func reached_EndOfText():
return Cursor >= SourceText.length()
func tokenize():
Tokens.clear()
while reached_EndOfText() == false :
var srcLeft = SourceText.substr(Cursor)
var token = Token.new()
var error = true
for type in TType.values() :
var result = SpecRegex[type].search( srcLeft )
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TType.cmt_SL || type == TType.cmt_ML :
Cursor += result.get_string().length()
error = false
break
# Skip Whitespace
if type == TType.fmt_S :
var addVal = result.get_string().length()
Cursor += addVal
error = false
break
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
Tokens.append( token )
error = false
break;
if error :
var assertStrTmplt = "Lexer - tokenize: Source text not understood by tokenizer at Cursor pos: {value} -: {txt}"
var assertStr = assertStrTmplt.format({"value" : Cursor, "txt" : srcLeft})
throw(assertStr)
return
func _init(programSrcText, errorOut) :
ErrorOut = errorOut
SourceText = programSrcText
Cursor = 0
TokenIndex = 0
if SpecRegex.size() == 0 :
compile_regex()
tokenize()

466
App/EoI/Scripts/Parser.gd Normal file
View File

@ -0,0 +1,466 @@
extends Object
# ---------------------------------------------------------- UTILITIES
var ErrorOut
func check( condition : bool, message : String):
assert(condition, message)
if ! condition:
ErrorOut.text = "Eva - Error: " + message
func throw( message ):
assert(false, message)
ErrorOut.text = "Eva - Error: " + message
# ---------------------------------------------------------- UTILITIES END
class_name Parser
# ---------------------------------------------------------- AST Node
const NType = \
{
program = "Program",
empty = "Empty",
block = "Scope Block",
conditional = "Conditional",
expr_Switch = "Expression Switch",
expr_While = "Expression While",
expr_For = "Expression For",
literal_Number = "Literal: Number",
literal_String = "Literal: String",
op_Assign = "Assignment",
op_Fn = "Function Call",
op_Add = "+",
op_Sub = "-",
op_Mult = "*",
op_Div = "/",
op_Increment = "++",
op_Decrement = "--",
op_Greater = ">",
op_GreaterEqual = ">=",
op_Lesser = "<",
op_LesserEqual = "<=",
op_Equal = "=",
op_NotEqual = "!=",
fn_Print = "Print",
fn_User = "User Function",
fn_Lambda = "Lambda Function",
fn_IIL = "Lambda Function Immediate Invocation",
fn_Params = "Function Parameters",
fn_Body = "Function Body",
identifier = "Identifier",
variable = "Variable"
}
class ASTNode:
var Data : Array
func get_class() :
return "ASTNode"
func add_Expr( expr ):
Data.append(expr)
func add_TokenValue( token ):
Data.append( token.Value )
func set_Type( nType ):
Data.append(nType)
func arg( id ):
return Data[id]
func num_args():
return Data.size() - 1
func type():
return Data[0]
func is_op_Relation():
match type():
NType.op_Greater: return true
NType.op_Lesser: return true
NType.op_GreaterEqual: return true
NType.op_LesserEqual: return true
NType.op_Equal: return true
NType.op_NotEqual: return true
_: return false
func is_op_Numeric():
match type():
NType.op_Add: return true
NType.op_Sub: return true
NType.op_Mult: return true
NType.op_Div: return true
NType.op_Increment: return true
NType.op_Decrement: return true
_: return false
func is_Number():
return type() == NType.literal_Number
func is_String():
return type() == NType.literal_String
func string():
return arg(1).substr(1, arg(1).length() -2)
# Serialization ----------------------------------------------------
func array_Serialize(array, fn_objSerializer) :
var result = []
for entry in array :
if typeof(entry) == TYPE_ARRAY :
result.append( array_Serialize( entry, fn_objSerializer ))
elif typeof(entry) == TYPE_OBJECT :
if entry.get_class() == "Eva":
result.append(entry)
else:
fn_objSerializer.set_instance(entry)
result.append( fn_objSerializer.call_func() )
else :
result.append( entry )
return result
func to_SExpression():
var \
to_SExpression_Fn = FuncRef.new()
to_SExpression_Fn.set_function("to_SExpression")
return array_Serialize( self.Data, to_SExpression_Fn )
# Serialization END -------------------------------------------------
# ---------------------------------------------------------- AST Node END
const SLexer = preload("Lexer.gd")
const TType = SLexer.TType
var Lexer : SLexer
var NextToken : SLexer.Token
# Gets the next token only if the current token is the specified intended token (tokenType)
func eat(tokenType):
var currToken = NextToken
check(currToken != null, "Parser - eat: NextToken was null")
var assertStrTmplt = "Parser - eat: Unexpected token: {value}, expected: {type}"
var assertStr = assertStrTmplt.format({"value" : currToken.Value, "type" : tokenType})
check(currToken.Type == tokenType, assertStr)
NextToken = Lexer.next_Token()
return currToken
func parse():
var \
node = ASTNode.new()
node.set_Type(NType.program)
while NextToken != null :
if NextToken.Type == TType.def_Start:
node.add_Expr( parse_Expression() )
elif NextToken.Type == TType.identifier:
node.add_Expr( parse_Identifier() )
elif NextToken.is_Literal():
node.add_Expr( parse_Literal() )
return node
func parse_Expression():
eat(TType.def_Start)
var node : ASTNode
match NextToken.Type :
TType.def_Block:
node = parse_Simple(TType.def_Block, NType.block)
TType.def_Cond:
node = parse_Simple(TType.def_cond, NType.conditional)
TType.def_Switch:
node = parse_Switch()
TType.def_While:
node = parse_Simple(TType.def_While, NType.expr_While)
TType.def_For:
node = parse_Simple(TType.def_For, NType.expr_For)
TType.def_Var:
node = parse_Variable()
TType.def_Func:
node = parse_fn_User()
TType.def_Lambda:
node = parse_fn_Lambda()
TType.fn_Print:
node = parse_Simple(TType.fn_Print, NType.fn_Print)
TType.op_Assgin:
node = parse_op_Assign()
TType.op_Numeric:
node = parse_op_Numeric()
TType.op_Relational:
node = parse_op_Relational()
TType.op_Equality:
node = ASTNode.new()
match NextToken.Value:
NType.op_Equal:
node.set_Type(NType.op_Equal)
NType.op_NotEqual:
node.set_Type(NType.op_NotEqual)
eat(TType.op_Equality)
TType.identifier:
node = parse_op_Fn()
TType.def_Start:
node = parse_fn_IIL()
while NextToken.Type != TType.def_End:
if NextToken.Type == TType.def_Start:
node.add_Expr( parse_Expression() )
elif NextToken.Type == TType.identifier:
node.add_Expr( parse_Identifier() )
else :
node.add_Expr( parse_Literal() )
eat(TType.def_End)
if node == null:
node = ASTNode.new()
node.set_Type(NType.empty)
return node
func parse_Simple(tType, nType):
var \
node = ASTNode.new()
node.set_Type(nType)
eat(tType)
return node
func parse_Switch():
var \
node = ASTNode.new()
node.set_Type(NType.expr_Switch)
eat(TType.def_Switch)
return node
func parse_Variable():
var \
node = ASTNode.new()
node.set_Type(NType.variable)
eat(TType.def_Var)
check( NextToken.Type == TType.identifier,
String("Parser - parse_Variable: NextToken should have been identifier. TokenData - Type: {type} Value: {value}") \
.format({"type" : NextToken.Type, "value" : NextToken.Value })
)
node.add_TokenValue( NextToken )
eat(TType.identifier)
return node
func parse_fn_User():
var \
node = ASTNode.new()
node.set_Type(NType.fn_User)
eat(TType.def_Func)
check( NextToken.Type == TType.identifier,
String("Parser - parse_op_Assign: NextToken should have been identifier, Type: {type} Value: {value}") \
.format({"type" : NextToken.Type, "value" : NextToken.Value })
)
node.add_TokenValue( NextToken )
eat(TType.identifier)
# Parameters
var \
pNode = ASTNode.new()
pNode.set_Type(NType.fn_Params)
eat(TType.def_Start)
while NextToken.Type != TType.def_End:
check( NextToken.Type == TType.identifier,
String("Parser - parse_op_Assign: NextToken should have been identifier, Type: {type} Value: {value}") \
.format({"type" : NextToken.Type, "value" : NextToken.Value })
)
pNode.add_TokenValue(NextToken)
eat(TType.identifier)
eat(TType.def_End)
var \
bNode = ASTNode.new()
bNode.set_Type(NType.fn_Body)
while NextToken.Type != TType.def_End:
bNode.add_Expr( parse_Expression() )
node.add_Expr( pNode )
node.add_Expr( bNode )
return node
func parse_fn_Lambda():
var \
node = ASTNode.new()
node.set_Type(NType.fn_Lambda)
eat(TType.def_Lambda)
# Parameters
var \
pNode = ASTNode.new()
pNode.set_Type(NType.fn_Params)
eat(TType.def_Start)
while NextToken.Type != TType.def_End:
check( NextToken.Type == TType.identifier,
String("Parser - parse_op_Assign: NextToken should have been identifier, Type: {type} Value: {value}") \
.format({"type" : NextToken.Type, "value" : NextToken.Value })
)
pNode.add_TokenValue(NextToken)
eat(TType.identifier)
eat(TType.def_End)
var \
bNode = ASTNode.new()
bNode.set_Type(NType.fn_Body)
while NextToken.Type != TType.def_End:
bNode.add_Expr( parse_Expression() )
node.add_Expr( pNode )
node.add_Expr( bNode )
return node
func parse_fn_IIL():
var \
node = ASTNode.new()
node.set_Type(NType.fn_IIL)
# Lambda
node.add_Expr( parse_Expression() )
return node
func parse_Identifier():
var \
node = ASTNode.new()
node.set_Type(NType.identifier)
node.add_TokenValue(NextToken)
eat(TType.identifier)
return node
func parse_op_Assign():
var \
node = ASTNode.new()
node.set_Type(NType.op_Assign)
eat(TType.op_Assgin)
check( NextToken.Type == TType.identifier,
String("Parser - parse_op_Assign: NextToken should have been identifier, Type: {type} Value: {value}") \
.format({"type" : NextToken.Type, "value" : NextToken.Value })
)
node.add_TokenValue( NextToken )
eat(TType.identifier)
if NextToken.is_Literal() :
node.add_Expr( parse_Literal() )
elif NextToken.Type == TType.def_Start :
node.add_Expr( parse_Expression() )
return node
func parse_op_Numeric():
var node = ASTNode.new()
match NextToken.Value:
NType.op_Add:
node.set_Type(NType.op_Add)
NType.op_Sub:
node.set_Type(NType.op_Sub)
NType.op_Mult:
node.set_Type(NType.op_Mult)
NType.op_Div:
node.set_Type(NType.op_Div)
NType.op_Increment:
node.set_Type(NType.op_Increment)
NType.op_Decrement:
node.set_Type(NType.op_Decrement)
eat(TType.op_Numeric)
return node
func parse_op_Relational():
var node = ASTNode.new()
match NextToken.Value:
NType.op_Greater:
node.set_Type(NType.op_Greater)
NType.op_Lesser:
node.set_Type(NType.op_Lesser)
NType.op_GreaterEqual:
node.set_Type(NType.op_GreaterEqual)
NType.op_LesserEqual:
node.set_Type(NType.op_LesserEqual)
eat(TType.op_Relational)
return node
func parse_op_Fn():
var \
node = ASTNode.new()
node.set_Type(NType.op_Fn)
node.add_TokenValue( NextToken )
eat(TType.identifier)
return node
func parse_Literal():
var node = ASTNode.new()
match NextToken.Type:
TType.literal_Number:
node.set_Type(NType.literal_Number)
node.add_TokenValue(NextToken)
eat(TType.literal_Number)
TType.literal_String:
node.set_Type(NType.literal_String)
node.add_TokenValue(NextToken)
eat(TType.literal_String)
return node
func _init(lexer, errorOut) :
ErrorOut = errorOut
Lexer = lexer
NextToken = Lexer.next_Token()

5
App/EoP/EoP_Viewer.tscn Normal file
View File

@ -0,0 +1,5 @@
[gd_scene format=2]
[node name="Control" type="Control"]
anchor_right = 1.0
anchor_bottom = 1.0

View File

View File

View File

@ -0,0 +1,702 @@
extends Node
# This closesly follows the source provided in the lectures.
# Later on after the lectures are complete or when I deem
# Necessary there will be heavy refactors.
const TokenType = \
{
Program = "Program",
# Comments
CommentLine = "CommentLine",
CommentMultiLine = "CommentMultiLine",
# Formatting
Whitespace = "Whitespace",
# Expressions
ExpressionPStart = "ExpresssionParenthesisStart",
ExpressionPEnd = "ExpressionParenthesisEnd",
# Logical
RelationalOp = "RelationalOperator",
# Arithmetic
ComplexAssignment = "ComplexAssignment",
Assignment = "Assignment",
AdditiveOp = "AdditiveOperator",
MultiplicativeOp = "MultiplicativeOperator",
# Conditional
Conditional_if = "if Conditional",
Conditional_else = "else Conditional",
# Statements
StatementEnd = "StatementEnd",
StmtBlockStart = "BlockStatementStart",
StmtBlockEnd = "BlockStatementEnd",
CommaDelimiter = "CommaDelimiter",
# Literals
Number = "Number",
String = "String",
# Symbols
VarDeclare = "Variable Declaration",
Identifier = "Identifier"
}
const TokenSpec = \
{
# Comments
TokenType.CommentLine : "^\\/\\/.*",
TokenType.CommentMultiLine : "^\\/\\*[\\s\\S]*?\\*\\/",
# Formatting
TokenType.Whitespace : "^\\s+",
# Expressions
TokenType.ExpressionPStart : "^\\(",
TokenType.ExpressionPEnd : "^\\)",
# Logical
TokenType.RelationalOp : "^[>\\<]=?",
# Arithmetic
TokenType.ComplexAssignment : "^[*\\/\\+\\-]=",
TokenType.Assignment : "^=",
TokenType.AdditiveOp : "^[+\\-]",
TokenType.MultiplicativeOp : "^[*\\/]",
# Literal
TokenType.Number : "\\d+",
TokenType.String : "^\"[^\"]*\"",
TokenType.Conditional_if : "^\\bif\\b",
TokenType.Conditional_else : "^\\belse\\b",
# Statements
TokenType.StatementEnd : "^;",
TokenType.StmtBlockStart : "^{",
TokenType.StmtBlockEnd : "^}",
TokenType.CommaDelimiter : "^,",
# Symbols
TokenType.VarDeclare : "^\\blet\\b",
TokenType.Identifier : "^\\w+"
}
class Token:
var Type : String
var Value : String
func to_Dictionary():
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Tokenizer:
var SrcTxt : String
var Cursor : int;
# Sets up the tokenizer with the program source text.
func init(programSrcText):
SrcTxt = programSrcText
Cursor = 0
# Provides the next token in the source text.
func next_Token():
if reached_EndOfTxt() == true :
return null
var srcLeft = SrcTxt.substr(Cursor)
var regex = RegEx.new()
var token = Token.new()
for type in TokenSpec :
regex.compile(TokenSpec[type])
var result = regex.search(srcLeft)
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TokenType.CommentLine || type == TokenType.CommentMultiLine :
Cursor += result.get_string().length()
return next_Token()
# Skip Whitespace
if type == TokenType.Whitespace :
var addVal = result.get_string().length()
Cursor += addVal
return next_Token()
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
return token
var assertStrTmplt = "next_token: Source text not understood by tokenizer at Cursor pos: {value}"
var assertStr = assertStrTmplt.format({"value" : Cursor})
assert(true != true, assertStr)
return null
func reached_EndOfTxt():
return Cursor >= ( SrcTxt.length() )
var GTokenizer = Tokenizer.new()
const AST_Format = \
{
Dictionary = "Dictionary",
SExpression = "S-Expression"
}
const SyntaxNodeType = \
{
NumericLiteral = "NumericLiteral",
StringLiteral = "StringLiteral",
ExpressionStatement = "ExpressionStatement",
BlockStatement = "BlockStatement",
EmptyStatement = "EmptyStatement",
BinaryExpression = "BinaryExpression",
Identifier = "Identifier",
AssignmentExpression = "AssignmentExpression",
VariableStatement = "VariableStatement",
VariableDeclaration = "VariableDeclaration",
ConditionalStatement = "ConditionalStatement"
}
class SyntaxNode:
var Type : String
var Value # Not specifing a type implicity declares a Variant type.
func to_SExpression():
var expression = [ Type ]
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_SExpression() )
else :
array.append( entry )
expression.append(array)
return expression
if typeof(Value) == TYPE_OBJECT :
var result = [ Type, Value.to_SExpression() ]
return result
expression.append(Value)
return expression
func to_Dictionary():
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_Dictionary() )
else :
array.append( entry )
var result = \
{
Type = self.Type,
Value = array
}
return result
if typeof(Value) == TYPE_OBJECT :
var result = \
{
Type = self.Type,
Value = self.Value.to_Dictionary()
}
return result
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Parser:
var TokenizerRef : Tokenizer
var NextToken : Token
func is_Literal():
return NextToken.Type == TokenType.Number || NextToken.Type == TokenType.String
func eat(tokenType):
var currToken = self.NextToken
assert(currToken != null, "eat: NextToken was null")
var assertStrTmplt = "eat: Unexpected token: {value}, expected: {type}"
var assertStr = assertStrTmplt.format({"value" : currToken.Value, "type" : tokenType})
assert(currToken.Type == tokenType, assertStr)
NextToken = TokenizerRef.next_Token()
return currToken
# NumericLiteral
# : Number
# ;
func parse_NumericLiteral():
var Token = eat(TokenType.Number)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NumericLiteral
node.Value = int( Token.Value )
return node
# StringLiteral
# : String
# ;
func parse_StringLiteral():
var Token = eat(TokenType.String)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.StringLiteral
node.Value = Token.Value.substr( 1, Token.Value.length() - 2 )
return node
# Literal
# : NumericLiteral
# : StringLiteral
# ;
func parse_Literal():
match NextToken.Type :
TokenType.Number:
return parse_NumericLiteral()
TokenType.String:
return parse_StringLiteral()
assert(false, "parse_Literal: Was not able to detect valid literal type from NextToken")
# ParenthesizedExpression
# : ( Expression )
# ;
func parse_ParenthesizedExpression():
eat(TokenType.ExpressionPStart)
var expression = parse_Expression()
eat(TokenType.ExpressionPEnd)
return expression
# Relational Operators: >, >=, <, <=
#
# Relational Expression
# : AdditiveExpression
# | AdditiveExpression RelationalOp RelationalExpression
# ;
func parse_RelationalExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_AdditiveExpression")
return parse_BinaryExpression(parseFn, TokenType.RelationalOp)
# MultiplicativeExpression
# : PrimaryExpression
# : MultiplicativeExpression MultiplicativeOp PrimaryExpression -> PrimaryExpression MultiplicativeOp ... Literal
# ;
func parse_MultiplicativeExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_PrimaryExpression")
return parse_BinaryExpression(parseFn, TokenType.MultiplicativeOp)
# AdditiveExpression
# : MultiplicativeExpression
# | AdditiveExpression AdditiveOp MultiplicativeExpression -> MultiplicativeExpression AdditiveOp ... Literal
# ;
func parse_AdditiveExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_MultiplicativeExpression")
return parse_BinaryExpression(parseFn, TokenType.AdditiveOp)
# BinaryExpression
# : MultiplicativeExpression
# | AdditiveExpression
# ;
func parse_BinaryExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken:
var operator = eat(operatorToken)
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.BinaryExpression
nestedNode.Value = []
nestedNode.Value.append(operator.Value)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode;
return left
# Identifier
# : IdentifierSymbol
# ;
func parse_Identifier():
var name = eat(TokenType.Identifier).Value
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.Identifier
node.Value = name
return node
# ResolvedSymbol
# : Identiifer
# ;
func parse_ResolvedSymbol():
var resolvedSymbol = parse_Identifier()
if resolvedSymbol.Type == SyntaxNodeType.Identifier :
return resolvedSymbol
var assertStrTmplt = "parse_ResolvedSymbol: Unexpected symbol: {value}"
var assertStr = assertStrTmplt.format({"value" : resolvedSymbol.Type})
assert(true != true, assertStr)
# PrimaryExpression
# : Literal
# | ParenthesizedExpression
# | ResolvedSymbol
# ;
func parse_PrimaryExpression():
if is_Literal():
return parse_Literal()
match NextToken.Type:
TokenType.ExpressionPStart:
return parse_ParenthesizedExpression()
return parse_ResolvedSymbol()
# AssignmentExpression
# : RelationalExpression
# | ResolvedSymbol AssignmentOperator AssignmetnExpression
# ;
func parse_AssignmentExpression():
var left = parse_RelationalExpression()
if NextToken.Type != TokenType.Assignment && NextToken.Type != TokenType.ComplexAssignment :
return left
var assignmentOp;
if NextToken.Type == TokenType.Assignment :
assignmentOp = eat(TokenType.Assignment)
elif NextToken.Type == TokenType.ComplexAssignment :
assignmentOp = eat(TokenType.ComplexAssignment)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.AssignmentExpression
node.Value = \
[
assignmentOp.Value,
left,
parse_AssignmentExpression()
]
return node
# Expression
# : AssignmentExpression
# ;
func parse_Expression():
return parse_AssignmentExpression()
# EmptyStatement
# ;
func parse_EmptyStatement():
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.EmptyStatement
return node
# If Statement
# : if ( Expression ) Statement
# | if ( Expression ) Statement else Statement
# ;
func parse_If_Statement():
eat(TokenType.Conditional_if)
eat(TokenType.ExpressionPStart)
var condition = parse_Expression()
eat(TokenType.ExpressionPEnd)
var consequent = parse_Statement()
var alternative = null
if NextToken != null && NextToken.Type == TokenType.Conditional_else :
eat(TokenType.Conditional_else)
alternative = parse_Statement()
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ConditionalStatement
node.Value = [ condition, consequent, alternative ]
return node
# VariableInitializer
# : Assignment AssignmentExpression
# ;
func parse_VariableInitializer():
eat(TokenType.Assignment)
return parse_AssignmentExpression()
# VariableDeclaration
# : Identifier OptVariableInitalizer
# ;
func parse_VariableDeclaration():
var identifier = parse_Identifier()
var initalizer
if NextToken.Type != TokenType.StatementEnd && NextToken.Type != TokenType.CommaDelimiter :
initalizer = parse_VariableInitializer()
else :
initalizer = null
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableDeclaration
node.Value = [ identifier, initalizer ]
return node
# VariableDeclarationList
# : VariableDeclaration
# | VariableDelcarationList , VariableDeclaration -> VariableDelcaration , ...
func parse_VariableDeclarationList():
var \
declarations = []
declarations.append(parse_VariableDeclaration())
while NextToken.Type == TokenType.CommaDelimiter :
eat(TokenType.CommaDelimiter)
declarations.append(parse_VariableDeclaration())
return declarations
# VariableStatement
# : VarDeclare VariableDeclarationList StatementEnd
# ;
func parse_VariableStatement():
eat(TokenType.VarDeclare)
var declarations = parse_VariableDeclarationList()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableStatement
node.Value = declarations
return node
# BlockStatement
# : { OptStatementList }
# ;
func parse_BlockStatement():
eat(TokenType.StmtBlockStart)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BlockStatement
if NextToken.Type != TokenType.StmtBlockEnd :
node.Value = parse_StatementList(TokenType.StmtBlockEnd)
else :
node.Value = []
eat(TokenType.StmtBlockEnd)
return node
# ExpressionStatement
# : Expression
# ;
func parse_ExpressionStatement():
var expression = parse_Expression()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ExpressionStatement
node.Value = expression
return expression
# Statement
# : ExpressionStatement
# | BlockStatement
# | EmptyStatement
# | VariableStatement
# | If_Statement
# ;
func parse_Statement():
if NextToken == null :
return null
match NextToken.Type :
TokenType.Conditional_if :
return parse_If_Statement()
TokenType.StatementEnd :
return parse_EmptyStatement()
TokenType.StmtBlockStart :
return parse_BlockStatement()
TokenType.VarDeclare :
return parse_VariableStatement()
return parse_ExpressionStatement()
# StatementList
# : Statement
# | StatementList Statement -> Statement ...
# ;
func parse_StatementList(endToken):
var statementList = [ parse_Statement() ]
while NextToken != null && NextToken.Type != endToken :
statementList.append( parse_Statement() )
return statementList
# Program
# : StatementList
# : Literal
# ;
func parse_Program():
var \
node = SyntaxNode.new()
node.Type = TokenType.Program
node.Value = parse_StatementList(null)
return node
# Parses the text program description into an AST.
func parse(TokenizerRef):
self.TokenizerRef = TokenizerRef
NextToken = TokenizerRef.next_Token()
return parse_Program()
var GParser = Parser.new()
onready var TextOut = GScene.get_node("TextOutput")
func tout(text):
TextOut.insert_text_at_cursor(text)
const Tests = \
{
MultiStatement = \
{
Name = "Multi-Statement",
File = "1.Multi-Statement.uf"
},
BlockStatement = \
{
Name = "Block Statement",
File = "2.BlockStatement.uf"
},
BinaryExpression = \
{
Name = "Binary Expression",
File = "3.BinaryExpression.uf"
},
Assignment = \
{
Name = "Assignment",
File = "4.Assignment.uf"
},
VaraibleDeclaration = \
{
Name = "Variable Declaration",
File = "5.VariableDeclaration.uf"
},
Conditionals = \
{
Name = "Conditionals",
File = "6.Conditionals.uf"
},
Relations = \
{
Name = "Relations",
File = "7.Relations.uf"
}
}
func test(entry):
var introMessage = "Testing: {Name}\n"
var introMessageFormatted = introMessage.format({"Name" : entry.Name})
tout(introMessageFormatted)
var path
if Engine.editor_hint :
path = "res://../Tests/{TestName}"
else :
path = "res://../Builds/Tests/{TestName}"
var pathFormatted = path.format({"TestName" : entry.File})
var \
file = File.new()
file.open(pathFormatted, File.READ)
var programDescription = file.get_as_text()
file.close()
GTokenizer.init(programDescription)
var ast = GParser.parse(GTokenizer)
var json = JSON.print(ast.to_SExpression(), '\t')
tout(json + "\n")
tout("Passed!\n")
# Main Entry point.
func _ready():
for Key in Tests :
test(Tests[Key])

View File

@ -0,0 +1,829 @@
extends Node
# This closesly follows the source provided in the lectures.
# Later on after the lectures are complete or when I deem
# Necessary there will be heavy refactors.
const TokenType = \
{
Program = "Program",
# Comments
CommentLine = "CommentLine",
CommentMultiLine = "CommentMultiLine",
# Formatting
Whitespace = "Whitespace",
# Expressions
ExpressionPStart = "ExpresssionParenthesisStart",
ExpressionPEnd = "ExpressionParenthesisEnd",
# Logical
RelationalOp = "RelationalOperator",
EqualityOp = "EqualityOperator",
Logical_And = "Logical_And_Op",
Logical_Or = "Logical_Or_Op",
# Arithmetic
ComplexAssignment = "ComplexAssignment",
Assignment = "Assignment",
AdditiveOp = "AdditiveOperator",
MultiplicativeOp = "MultiplicativeOperator",
# Conditional
Conditional_if = "if Conditional",
Conditional_else = "else Conditional",
# Statements
StatementEnd = "StatementEnd",
StmtBlockStart = "BlockStatementStart",
StmtBlockEnd = "BlockStatementEnd",
CommaDelimiter = "CommaDelimiter",
# Literals
Number = "Number",
String = "String",
# Symbols
Bool_true = "Boolean True",
Bool_false = "Boolean False",
VarDeclare = "Variable Declaration",
Identifier = "Identifier",
NullValue = "Null Value"
}
const TokenSpec = \
{
# Comments
TokenType.CommentLine : "^\\/\\/.*",
TokenType.CommentMultiLine : "^\\/\\*[\\s\\S]*?\\*\\/",
# Formatting
TokenType.Whitespace : "^\\s+",
# Expressions
TokenType.ExpressionPStart : "^\\(",
TokenType.ExpressionPEnd : "^\\)",
# Logical
TokenType.RelationalOp : "^[>\\<]=?",
TokenType.EqualityOp : "^[=!]=",
TokenType.Logical_And : "^&&",
TokenType.Logical_Or : "^\\|\\|",
# Arithmetic
TokenType.ComplexAssignment : "^[*\\/\\+\\-]=",
TokenType.Assignment : "^=",
TokenType.AdditiveOp : "^[+\\-]",
TokenType.MultiplicativeOp : "^[*\\/]",
# Literal
TokenType.Number : "\\d+",
TokenType.String : "^\"[^\"]*\"",
TokenType.Conditional_if : "^\\bif\\b",
TokenType.Conditional_else : "^\\belse\\b",
# Statements
TokenType.StatementEnd : "^;",
TokenType.StmtBlockStart : "^{",
TokenType.StmtBlockEnd : "^}",
TokenType.CommaDelimiter : "^,",
# Symbols
TokenType.Bool_true : "^\\btrue\\b",
TokenType.Bool_false : "^\\bfalse\\b",
TokenType.VarDeclare : "^\\blet\\b",
TokenType.Identifier : "^\\w+",
TokenType.NullValue : "^\\bnull\\b"
}
class Token:
var Type : String
var Value : String
func to_Dictionary():
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Tokenizer:
var SrcTxt : String
var Cursor : int;
# Sets up the tokenizer with the program source text.
func init(programSrcText):
SrcTxt = programSrcText
Cursor = 0
# Provides the next token in the source text.
func next_Token():
if reached_EndOfTxt() == true :
return null
var srcLeft = SrcTxt.substr(Cursor)
var regex = RegEx.new()
var token = Token.new()
for type in TokenSpec :
regex.compile(TokenSpec[type])
var result = regex.search(srcLeft)
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TokenType.CommentLine || type == TokenType.CommentMultiLine :
Cursor += result.get_string().length()
return next_Token()
# Skip Whitespace
if type == TokenType.Whitespace :
var addVal = result.get_string().length()
Cursor += addVal
return next_Token()
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
return token
var assertStrTmplt = "next_token: Source text not understood by tokenizer at Cursor pos: {value}"
var assertStr = assertStrTmplt.format({"value" : Cursor})
assert(true != true, assertStr)
return null
func reached_EndOfTxt():
return Cursor >= ( SrcTxt.length() )
var GTokenizer = Tokenizer.new()
const AST_Format = \
{
Dictionary = "Dictionary",
SExpression = "S-Expression"
}
const SyntaxNodeType = \
{
NumericLiteral = "NumericLiteral",
StringLiteral = "StringLiteral",
ExpressionStatement = "ExpressionStatement",
BlockStatement = "BlockStatement",
EmptyStatement = "EmptyStatement",
BinaryExpression = "BinaryExpression",
Identifier = "Identifier",
AssignmentExpression = "AssignmentExpression",
VariableStatement = "VariableStatement",
VariableDeclaration = "VariableDeclaration",
ConditionalStatement = "ConditionalStatement",
BooleanLiteral = "BooleanLiteral",
NullLiteral = "NullLiteral",
LogicalExpression = "LogicalExpression"
}
class SyntaxNode:
var Type : String
var Value # Not specifing a type implicity declares a Variant type.
func to_SExpression():
var expression = [ Type ]
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_SExpression() )
else :
array.append( entry )
expression.append(array)
return expression
if typeof(Value) == TYPE_OBJECT :
var result = [ Type, Value.to_SExpression() ]
return result
expression.append(Value)
return expression
func to_Dictionary():
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_Dictionary() )
else :
array.append( entry )
var result = \
{
Type = self.Type,
Value = array
}
return result
if typeof(Value) == TYPE_OBJECT :
var result = \
{
Type = self.Type,
Value = self.Value.to_Dictionary()
}
return result
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Parser:
var TokenizerRef : Tokenizer
var NextToken : Token
func is_Literal():
return \
NextToken.Type == TokenType.Number \
|| NextToken.Type == TokenType.String \
|| NextToken.Type == TokenType.Bool_true \
|| NextToken.Type == TokenType.Bool_false \
|| NextToken.Type == TokenType.NullValue
func eat(tokenType):
var currToken = self.NextToken
assert(currToken != null, "eat: NextToken was null")
var assertStrTmplt = "eat: Unexpected token: {value}, expected: {type}"
var assertStr = assertStrTmplt.format({"value" : currToken.Value, "type" : tokenType})
assert(currToken.Type == tokenType, assertStr)
NextToken = TokenizerRef.next_Token()
return currToken
# NumericLiteral
# : Number
# ;
func parse_NumericLiteral():
var Token = eat(TokenType.Number)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NumericLiteral
node.Value = int( Token.Value )
return node
# StringLiteral
# : String
# ;
func parse_StringLiteral():
var Token = eat(TokenType.String)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.StringLiteral
node.Value = Token.Value.substr( 1, Token.Value.length() - 2 )
return node
# BooleanLiteral
# : true
# | false
# ;
func parse_BooleanLiteral(token):
eat(token)
var value
if (TokenType.Bool_true == token) :
value = true
elif (TokenType.Bool_false == token) :
value = false
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BooleanLiteral
node.Value = value
return node
# NullLiteral
# : null
# ;
func parse_NullLiteral():
eat(TokenType.NullLiteral)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NullLiteral
node.Value = null
return node
# Literal
# : NumericLiteral
# | StringLiteral
# | BooleanLiteral
# | NullLiteral
# ;
func parse_Literal():
match NextToken.Type :
TokenType.Number:
return parse_NumericLiteral()
TokenType.String:
return parse_StringLiteral()
TokenType.Bool_true:
return parse_BooleanLiteral(TokenType.Bool_true)
TokenType.Bool_false:
return parse_BooleanLiteral(TokenType.Bool_false)
TokenType.NullValue:
return parse_NullLiteral()
assert(false, "parse_Literal: Was not able to detect valid literal type from NextToken")
# ParenthesizedExpression
# : ( Expression )
# ;
func parse_ParenthesizedExpression():
eat(TokenType.ExpressionPStart)
var expression = parse_Expression()
eat(TokenType.ExpressionPEnd)
return expression
# Relational Operators: >, >=, <, <=
#
# Relational Expression
# : AdditiveExpression
# | AdditiveExpression RelationalOp RelationalExpression
# ;
func parse_RelationalExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_AdditiveExpression")
return parse_BinaryExpression(parseFn, TokenType.RelationalOp)
# Equality Operators: ==, !=
#
# EqualityExpression
# : RelationalExpression EqualityOp RelationalExpression
# | RelationalExpression
# ;
func parse_EqualityExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_RelationalExpression")
return parse_BinaryExpression(parseFn, TokenType.EqualityOp)
# Logical Or Expression
# : LogicalAndExpression Logical_Or LogicalOrExpression
# | LogicalOrExpression
# ;
func parse_LogicalOrExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("pasre_LogicalAndExpression")
return parse_LogicalExpression(parseFn, TokenType.Logical_Or)
# Logical And Expression
# : EqualityExpression Logical_And LogicalAndExpression
# | EqualityExpression
# ;
func pasre_LogicalAndExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_EqualityExpression")
return parse_LogicalExpression(parseFn, TokenType.Logical_And)
func parse_LogicalExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken :
var operator = eat(operatorToken).Value
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.LogicalExpression
nestedNode.Value = []
nestedNode.Value.append(operator)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode
return left
# MultiplicativeExpression
# : PrimaryExpression
# : MultiplicativeExpression MultiplicativeOp PrimaryExpression -> PrimaryExpression MultiplicativeOp ... Literal
# ;
func parse_MultiplicativeExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_PrimaryExpression")
return parse_BinaryExpression(parseFn, TokenType.MultiplicativeOp)
# AdditiveExpression
# : MultiplicativeExpression
# | AdditiveExpression AdditiveOp MultiplicativeExpression -> MultiplicativeExpression AdditiveOp ... Literal
# ;
func parse_AdditiveExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_MultiplicativeExpression")
return parse_BinaryExpression(parseFn, TokenType.AdditiveOp)
# BinaryExpression
# : MultiplicativeExpression
# | AdditiveExpression
# ;
func parse_BinaryExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken:
var operator = eat(operatorToken)
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.BinaryExpression
nestedNode.Value = []
nestedNode.Value.append(operator.Value)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode;
return left
# Identifier
# : IdentifierSymbol
# ;
func parse_Identifier():
var name = eat(TokenType.Identifier).Value
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.Identifier
node.Value = name
return node
# ResolvedSymbol
# : Identiifer
# ;
func parse_ResolvedSymbol():
var resolvedSymbol = parse_Identifier()
if resolvedSymbol.Type == SyntaxNodeType.Identifier :
return resolvedSymbol
var assertStrTmplt = "parse_ResolvedSymbol: Unexpected symbol: {value}"
var assertStr = assertStrTmplt.format({"value" : resolvedSymbol.Type})
assert(true != true, assertStr)
# PrimaryExpression
# : Literal
# | ParenthesizedExpression
# | ResolvedSymbol
# ;
func parse_PrimaryExpression():
if is_Literal():
return parse_Literal()
match NextToken.Type:
TokenType.ExpressionPStart:
return parse_ParenthesizedExpression()
return parse_ResolvedSymbol()
# AssignmentExpression
# : RelationalExpression
# | ResolvedSymbol AssignmentOperator AssignmetnExpression
# ;
func parse_AssignmentExpression():
var left = parse_LogicalOrExpression()
if NextToken.Type != TokenType.Assignment && NextToken.Type != TokenType.ComplexAssignment :
return left
var assignmentOp;
if NextToken.Type == TokenType.Assignment :
assignmentOp = eat(TokenType.Assignment)
elif NextToken.Type == TokenType.ComplexAssignment :
assignmentOp = eat(TokenType.ComplexAssignment)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.AssignmentExpression
node.Value = \
[
assignmentOp.Value,
left,
parse_AssignmentExpression()
]
return node
# Expression
# : AssignmentExpression
# ;
func parse_Expression():
return parse_AssignmentExpression()
# EmptyStatement
# ;
func parse_EmptyStatement():
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.EmptyStatement
return node
# If Statement
# : if ( Expression ) Statement
# | if ( Expression ) Statement else Statement
# ;
func parse_If_Statement():
eat(TokenType.Conditional_if)
eat(TokenType.ExpressionPStart)
var condition = parse_Expression()
eat(TokenType.ExpressionPEnd)
var consequent = parse_Statement()
var alternative = null
if NextToken != null && NextToken.Type == TokenType.Conditional_else :
eat(TokenType.Conditional_else)
alternative = parse_Statement()
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ConditionalStatement
node.Value = [ condition, consequent, alternative ]
return node
# VariableInitializer
# : Assignment AssignmentExpression
# ;
func parse_VariableInitializer():
eat(TokenType.Assignment)
return parse_AssignmentExpression()
# VariableDeclaration
# : Identifier OptVariableInitalizer
# ;
func parse_VariableDeclaration():
var identifier = parse_Identifier()
var initalizer
if NextToken.Type != TokenType.StatementEnd && NextToken.Type != TokenType.CommaDelimiter :
initalizer = parse_VariableInitializer()
else :
initalizer = null
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableDeclaration
node.Value = [ identifier, initalizer ]
return node
# VariableDeclarationList
# : VariableDeclaration
# | VariableDelcarationList , VariableDeclaration -> VariableDelcaration , ...
func parse_VariableDeclarationList():
var \
declarations = []
declarations.append(parse_VariableDeclaration())
while NextToken.Type == TokenType.CommaDelimiter :
eat(TokenType.CommaDelimiter)
declarations.append(parse_VariableDeclaration())
return declarations
# VariableStatement
# : VarDeclare VariableDeclarationList StatementEnd
# ;
func parse_VariableStatement():
eat(TokenType.VarDeclare)
var declarations = parse_VariableDeclarationList()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableStatement
node.Value = declarations
return node
# BlockStatement
# : { OptStatementList }
# ;
func parse_BlockStatement():
eat(TokenType.StmtBlockStart)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BlockStatement
if NextToken.Type != TokenType.StmtBlockEnd :
node.Value = parse_StatementList(TokenType.StmtBlockEnd)
else :
node.Value = []
eat(TokenType.StmtBlockEnd)
return node
# ExpressionStatement
# : Expression
# ;
func parse_ExpressionStatement():
var expression = parse_Expression()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ExpressionStatement
node.Value = expression
return expression
# Statement
# : ExpressionStatement
# | BlockStatement
# | EmptyStatement
# | VariableStatement
# | If_Statement
# ;
func parse_Statement():
if NextToken == null :
return null
match NextToken.Type :
TokenType.Conditional_if :
return parse_If_Statement()
TokenType.StatementEnd :
return parse_EmptyStatement()
TokenType.StmtBlockStart :
return parse_BlockStatement()
TokenType.VarDeclare :
return parse_VariableStatement()
return parse_ExpressionStatement()
# StatementList
# : Statement
# | StatementList Statement -> Statement ...
# ;
func parse_StatementList(endToken):
var statementList = [ parse_Statement() ]
while NextToken != null && NextToken.Type != endToken :
statementList.append( parse_Statement() )
return statementList
# Program
# : StatementList
# : Literal
# ;
func parse_Program():
var \
node = SyntaxNode.new()
node.Type = TokenType.Program
node.Value = parse_StatementList(null)
return node
# Parses the text program description into an AST.
func parse(TokenizerRef):
self.TokenizerRef = TokenizerRef
NextToken = TokenizerRef.next_Token()
return parse_Program()
var GParser = Parser.new()
onready var TextOut = GScene.get_node("TextOutput")
func tout(text):
TextOut.insert_text_at_cursor(text)
const Tests = \
{
MultiStatement = \
{
Name = "Multi-Statement",
File = "1.Multi-Statement.uf"
},
BlockStatement = \
{
Name = "Block Statement",
File = "2.BlockStatement.uf"
},
BinaryExpression = \
{
Name = "Binary Expression",
File = "3.BinaryExpression.uf"
},
Assignment = \
{
Name = "Assignment",
File = "4.Assignment.uf"
},
VaraibleDeclaration = \
{
Name = "Variable Declaration",
File = "5.VariableDeclaration.uf"
},
Conditionals = \
{
Name = "Conditionals",
File = "6.Conditionals.uf"
},
Relations = \
{
Name = "Relations",
File = "7.Relations.uf"
},
Equality = \
{
Name = "Equality",
File = "8.Equality.uf"
},
Logical = \
{
Name = "Logical",
File = "9.Logical.uf"
}
}
func test(entry):
var introMessage = "Testing: {Name}\n"
var introMessageFormatted = introMessage.format({"Name" : entry.Name})
tout(introMessageFormatted)
var path
if Engine.editor_hint :
path = "res://../Tests/{TestName}"
else :
path = "res://../Builds/Tests/{TestName}"
var pathFormatted = path.format({"TestName" : entry.File})
var \
file = File.new()
file.open(pathFormatted, File.READ)
var programDescription = file.get_as_text()
file.close()
GTokenizer.init(programDescription)
var ast = GParser.parse(GTokenizer)
var json = JSON.print(ast.to_Dictionary(), '\t')
tout(json + "\n")
tout("Passed!\n")
# Main Entry point.
func _ready():
for Key in Tests :
test(Tests[Key])

View File

@ -0,0 +1,916 @@
extends Node
# This closesly follows the source provided in the lectures.
# Later on after the lectures are complete or when I deem
# Necessary there will be heavy refactors.
const TokenType = \
{
Program = "Program",
# Comments
CommentLine = "CommentLine",
CommentMultiLine = "CommentMultiLine",
# Formatting
Whitespace = "Whitespace",
# Expressions
ExpressionPStart = "ExpresssionParenthesisStart",
ExpressionPEnd = "ExpressionParenthesisEnd",
# Logical
RelationalOp = "RelationalOperator",
EqualityOp = "EqualityOperator",
Logical_And = "Logical_And_Op",
Logical_Or = "Logical_Or_Op",
Logical_Not = "Logical_Not_Op",
# Arithmetic
ComplexAssignment = "ComplexAssignment",
Assignment = "Assignment",
AdditiveOp = "AdditiveOperator",
MultiplicativeOp = "MultiplicativeOperator",
# Conditional
Conditional_if = "if Conditional",
Conditional_else = "else Conditional",
# Statements
StatementEnd = "StatementEnd",
StmtBlockStart = "BlockStatementStart",
StmtBlockEnd = "BlockStatementEnd",
CommaDelimiter = "CommaDelimiter",
# Literals
Number = "Number",
String = "String",
# Symbols
Bool_true = "Boolean True",
Bool_false = "Boolean False",
VarDeclare = "Variable Declaration",
Identifier = "Identifier",
NullValue = "Null Value"
}
const TokenSpec = \
{
# Comments
TokenType.CommentLine : "^\\/\\/.*",
TokenType.CommentMultiLine : "^\\/\\*[\\s\\S]*?\\*\\/",
# Formatting
TokenType.Whitespace : "^\\s+",
# Expressions
TokenType.ExpressionPStart : "^\\(",
TokenType.ExpressionPEnd : "^\\)",
# Logical
TokenType.RelationalOp : "^[>\\<]=?",
TokenType.EqualityOp : "^[=!]=",
TokenType.Logical_And : "^&&",
TokenType.Logical_Or : "^\\|\\|",
TokenType.Logical_Not : "^!",
# Arithmetic
TokenType.ComplexAssignment : "^[*\\/\\+\\-]=",
TokenType.Assignment : "^=",
TokenType.AdditiveOp : "^[+\\-]",
TokenType.MultiplicativeOp : "^[*\\/]",
# Literal
TokenType.Number : "\\d+",
TokenType.String : "^\"[^\"]*\"",
TokenType.Conditional_if : "^\\bif\\b",
TokenType.Conditional_else : "^\\belse\\b",
# Statements
TokenType.StatementEnd : "^;",
TokenType.StmtBlockStart : "^{",
TokenType.StmtBlockEnd : "^}",
TokenType.CommaDelimiter : "^,",
# Symbols
TokenType.Bool_true : "^\\btrue\\b",
TokenType.Bool_false : "^\\bfalse\\b",
TokenType.VarDeclare : "^\\blet\\b",
TokenType.Identifier : "^\\w+",
TokenType.NullValue : "^\\bnull\\b"
}
class Token:
var Type : String
var Value : String
func to_Dictionary():
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Tokenizer:
var SrcTxt : String
var Cursor : int;
# Sets up the tokenizer with the program source text.
func init(programSrcText):
SrcTxt = programSrcText
Cursor = 0
# Provides the next token in the source text.
func next_Token():
if reached_EndOfTxt() == true :
return null
var srcLeft = SrcTxt.substr(Cursor)
var regex = RegEx.new()
var token = Token.new()
for type in TokenSpec :
regex.compile(TokenSpec[type])
var result = regex.search(srcLeft)
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TokenType.CommentLine || type == TokenType.CommentMultiLine :
Cursor += result.get_string().length()
return next_Token()
# Skip Whitespace
if type == TokenType.Whitespace :
var addVal = result.get_string().length()
Cursor += addVal
return next_Token()
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
return token
var assertStrTmplt = "next_token: Source text not understood by tokenizer at Cursor pos: {value}"
var assertStr = assertStrTmplt.format({"value" : Cursor})
assert(true != true, assertStr)
return null
func reached_EndOfTxt():
return Cursor >= ( SrcTxt.length() )
var GTokenizer = Tokenizer.new()
const AST_Format = \
{
Dictionary = "Dictionary",
SExpression = "S-Expression"
}
const SyntaxNodeType = \
{
NumericLiteral = "NumericLiteral",
StringLiteral = "StringLiteral",
ExpressionStatement = "ExpressionStatement",
BlockStatement = "BlockStatement",
EmptyStatement = "EmptyStatement",
BinaryExpression = "BinaryExpression",
Identifier = "Identifier",
AssignmentExpression = "AssignmentExpression",
VariableStatement = "VariableStatement",
VariableDeclaration = "VariableDeclaration",
ConditionalStatement = "ConditionalStatement",
BooleanLiteral = "BooleanLiteral",
NullLiteral = "NullLiteral",
LogicalExpression = "LogicalExpression",
UnaryExpression = "UnaryExpression"
}
class SyntaxNode:
var Type : String
var Value # Not specifing a type implicity declares a Variant type.
func to_SExpression():
var expression = [ Type ]
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_SExpression() )
else :
array.append( entry )
expression.append(array)
return expression
if typeof(Value) == TYPE_OBJECT :
var result = [ Type, Value.to_SExpression() ]
return result
expression.append(Value)
return expression
func to_Dictionary():
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_Dictionary() )
else :
array.append( entry )
var result = \
{
Type = self.Type,
Value = array
}
return result
if typeof(Value) == TYPE_OBJECT :
var result = \
{
Type = self.Type,
Value = self.Value.to_Dictionary()
}
return result
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Parser:
var TokenizerRef : Tokenizer
var NextToken : Token
# --------------------------------------------------------------------- HELPERS
# Gets the next token only if the current token is the specified intended token (tokenType)
func eat(tokenType):
var currToken = self.NextToken
assert(currToken != null, "eat: NextToken was null")
var assertStrTmplt = "eat: Unexpected token: {value}, expected: {type}"
var assertStr = assertStrTmplt.format({"value" : currToken.Value, "type" : tokenType})
assert(currToken.Type == tokenType, assertStr)
NextToken = TokenizerRef.next_Token()
return currToken
func is_Literal():
return \
NextToken.Type == TokenType.Number \
|| NextToken.Type == TokenType.String \
|| NextToken.Type == TokenType.Bool_true \
|| NextToken.Type == TokenType.Bool_false \
|| NextToken.Type == TokenType.NullValue
# BinaryExpression
# : MultiplicativeExpression
# | AdditiveExpression
# ;
func parse_BinaryExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken:
var operator = eat(operatorToken)
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.BinaryExpression
nestedNode.Value = []
nestedNode.Value.append(operator.Value)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode;
return left
# LogicalExpression
# : LogicalAndExpression
# | LogicalOrExpression
# ;
func parse_LogicalExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken :
var operator = eat(operatorToken).Value
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.LogicalExpression
nestedNode.Value = []
nestedNode.Value.append(operator)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode
return left
# ------------------------------------------------------------------ END HELPERS
# Parses the text program description into an AST.
func parse(TokenizerRef):
self.TokenizerRef = TokenizerRef
NextToken = TokenizerRef.next_Token()
return parse_Program()
# > parse
# Program
# : StatementList
# : Literal
# ;
func parse_Program():
var \
node = SyntaxNode.new()
node.Type = TokenType.Program
node.Value = parse_StatementList(null)
return node
# > Program
# > BlockStatement
# StatementList
# : Statement
# | StatementList Statement -> Statement ...
# ;
func parse_StatementList(endToken):
var statementList = [ parse_Statement() ]
while NextToken != null && NextToken.Type != endToken :
statementList.append( parse_Statement() )
return statementList
# > StatementList
# > If_Statement
# >
# Statement
# : ExpressionStatement
# | BlockStatement
# | EmptyStatement
# | VariableStatement
# | If_Statement
# ;
func parse_Statement():
if NextToken == null :
return null
match NextToken.Type :
TokenType.Conditional_if :
return parse_If_Statement()
TokenType.StatementEnd :
return parse_EmptyStatement()
TokenType.StmtBlockStart :
return parse_BlockStatement()
TokenType.VarDeclare :
return parse_VariableStatement()
return parse_ExpressionStatement()
# If Statement
# : if ( Expression ) Statement
# | if ( Expression ) Statement else Statement
# ;
func parse_If_Statement():
eat(TokenType.Conditional_if)
eat(TokenType.ExpressionPStart)
var condition = parse_Expression()
eat(TokenType.ExpressionPEnd)
var consequent = parse_Statement()
var alternative = null
if NextToken != null && NextToken.Type == TokenType.Conditional_else :
eat(TokenType.Conditional_else)
alternative = parse_Statement()
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ConditionalStatement
node.Value = [ condition, consequent, alternative ]
return node
# > Statement
# EmptyStatement
# ;
func parse_EmptyStatement():
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.EmptyStatement
return node
# > Statement
# BlockStatement
# : { OptStatementList }
# ;
func parse_BlockStatement():
eat(TokenType.StmtBlockStart)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BlockStatement
if NextToken.Type != TokenType.StmtBlockEnd :
node.Value = parse_StatementList(TokenType.StmtBlockEnd)
else :
node.Value = []
eat(TokenType.StmtBlockEnd)
return node
# > Statement
# VariableStatement
# : VarDeclare VariableDeclarationList StatementEnd
# ;
func parse_VariableStatement():
eat(TokenType.VarDeclare)
var declarations = parse_VariableDeclarationList()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableStatement
node.Value = declarations
return node
# > Statement
# ExpressionStatement
# : Expression
# ;
func parse_ExpressionStatement():
var expression = parse_Expression()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ExpressionStatement
node.Value = expression
return expression
# > ExpressionStatement
# > If_Statement
# > PrimaryExpression
# Expression
# : AssignmentExpression
# ;
func parse_Expression():
return parse_AssignmentExpression()
# > VariableStatement
# VariableDeclarationList
# : VariableDeclaration
# | VariableDelcarationList , VariableDeclaration -> VariableDelcaration , ...
func parse_VariableDeclarationList():
var \
declarations = []
declarations.append(parse_VariableDeclaration())
while NextToken.Type == TokenType.CommaDelimiter :
eat(TokenType.CommaDelimiter)
declarations.append(parse_VariableDeclaration())
return declarations
# > VariableDeclarationList
# VariableDeclaration
# : Identifier OptVariableInitalizer
# ;
func parse_VariableDeclaration():
var identifier = parse_Identifier()
var initalizer
if NextToken.Type != TokenType.StatementEnd && NextToken.Type != TokenType.CommaDelimiter :
initalizer = parse_VariableInitializer()
else :
initalizer = null
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableDeclaration
node.Value = [ identifier, initalizer ]
return node
# > VariableDeclaration
# VariableInitializer
# : Assignment AssignmentExpression
# ;
func parse_VariableInitializer():
eat(TokenType.Assignment)
return parse_AssignmentExpression()
# > Expression
# > VariableInitializer
# > AssignmentExpression
# AssignmentExpression
# : RelationalExpression
# | ResolvedSymbol AssignmentOperator AssignmetnExpression
# ;
func parse_AssignmentExpression():
var left = parse_LogicalOrExpression()
if NextToken.Type != TokenType.Assignment && NextToken.Type != TokenType.ComplexAssignment :
return left
var assignmentOp;
if NextToken.Type == TokenType.Assignment :
assignmentOp = eat(TokenType.Assignment)
elif NextToken.Type == TokenType.ComplexAssignment :
assignmentOp = eat(TokenType.ComplexAssignment)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.AssignmentExpression
node.Value = \
[
assignmentOp.Value,
left,
parse_AssignmentExpression()
]
return node
# > VariableDeclaration
# > ParenthesizedExpression
# Identifier
# : IdentifierSymbol
# ;
func parse_Identifier():
var name = eat(TokenType.Identifier).Value
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.Identifier
node.Value = name
return node
# > AssignmentExpression
# Logical Or Expression
# : LogicalAndExpression Logical_Or LogicalOrExpression
# | LogicalOrExpression
# ;
func parse_LogicalOrExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("pasre_LogicalAndExpression")
return parse_LogicalExpression(parseFn, TokenType.Logical_Or)
# > LogicaOrExpression
# Logical And Expression
# : EqualityExpression Logical_And LogicalAndExpression
# | EqualityExpression
# ;
func pasre_LogicalAndExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_EqualityExpression")
return parse_LogicalExpression(parseFn, TokenType.Logical_And)
# Equality Operators: ==, !=
#
# > LogicalAndExpression
# EqualityExpression
# : RelationalExpression EqualityOp RelationalExpression
# | RelationalExpression
# ;
func parse_EqualityExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_RelationalExpression")
return parse_BinaryExpression(parseFn, TokenType.EqualityOp)
# Relational Operators: >, >=, <, <=
#
# > EqualityExpression
# Relational Expression
# : AdditiveExpression
# | AdditiveExpression RelationalOp RelationalExpression
# ;
func parse_RelationalExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_AdditiveExpression")
return parse_BinaryExpression(parseFn, TokenType.RelationalOp)
# > RelationalExpression
# AdditiveExpression
# : MultiplicativeExpression
# | AdditiveExpression AdditiveOp MultiplicativeExpression -> MultiplicativeExpression AdditiveOp ... Literal
# ;
func parse_AdditiveExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_MultiplicativeExpression")
return parse_BinaryExpression(parseFn, TokenType.AdditiveOp)
# > AdditiveExpression
# MultiplicativeExpression
# : UnaryExpressioon
# : MultiplicativeExpression MultiplicativeOp UnaryExpression -> UnaryExpression MultiplicativeOp ... Literal
# ;
func parse_MultiplicativeExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_UnaryExpression")
return parse_BinaryExpression(parseFn, TokenType.MultiplicativeOp)
# > MultiplicativeExpression
# > UnaryExpression
# UnaryExpression
# : ResolvedSymbol
# | AdditiveOp UnaryExpression
# | Logical_Not UnaryExpression
# ;
func parse_UnaryExpression():
var operator
match NextToken.Type:
TokenType.AdditiveOp:
operator = eat(TokenType.AdditiveOp).Value
TokenType.Logical_Not:
operator = eat(TokenType.Logical_Not).Value
if operator == null :
return parse_ResolvedSymbol()
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.UnaryExpression
node.Value = [ operator, parse_UnaryExpression() ]
return node;
# > UnaryExpression
# > PrimaryExpression
# ResolvedSymbol (LeftHandExpression)
# : PrimaryExpression
# ;
func parse_ResolvedSymbol():
return parse_PrimaryExpression()
# var resolvedSymbol = parse_Identifier()
# if resolvedSymbol.Type == SyntaxNodeType.Identifier :
# return resolvedSymbol
# var assertStrTmplt = "parse_ResolvedSymbol: Unexpected symbol: {value}"
# var assertStr = assertStrTmplt.format({"value" : resolvedSymbol.Type})
# assert(true != true, assertStr)
# > ResolvedSymbol
# PrimaryExpression
# : Literal
# | ParenthesizedExpression
# ;
func parse_PrimaryExpression():
if is_Literal():
return parse_Literal()
match NextToken.Type:
TokenType.ExpressionPStart:
return parse_ParenthesizedExpression()
TokenType.Identifier:
var identifier = parse_Identifier()
if identifier.Type == SyntaxNodeType.Identifier :
return identifier
var assertStrTmplt = "parse_PrimaryExpression: (Identifier) Unexpected symbol: {value}"
var assertStr = assertStrTmplt.format({"value" : identifier.Type})
assert(true != true, assertStr)
return parse_ResolvedSymbol()
# > PrimaryExpression
# Literal
# : NumericLiteral
# | StringLiteral
# | BooleanLiteral
# | NullLiteral
# ;
func parse_Literal():
match NextToken.Type :
TokenType.Number:
return parse_NumericLiteral()
TokenType.String:
return parse_StringLiteral()
TokenType.Bool_true:
return parse_BooleanLiteral(TokenType.Bool_true)
TokenType.Bool_false:
return parse_BooleanLiteral(TokenType.Bool_false)
TokenType.NullValue:
return parse_NullLiteral()
assert(false, "parse_Literal: Was not able to detect valid literal type from NextToken")
# > PrimaryExpression
# ParenthesizedExpression
# : ( Expression )
# ;
func parse_ParenthesizedExpression():
eat(TokenType.ExpressionPStart)
var expression = parse_Expression()
eat(TokenType.ExpressionPEnd)
return expression
# > Literal
# NumericLiteral
# : Number
# ;
func parse_NumericLiteral():
var Token = eat(TokenType.Number)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NumericLiteral
node.Value = int( Token.Value )
return node
# > Literal
# StringLiteral
# : String
# ;
func parse_StringLiteral():
var Token = eat(TokenType.String)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.StringLiteral
node.Value = Token.Value.substr( 1, Token.Value.length() - 2 )
return node
# > Literal
# BooleanLiteral
# : true
# | false
# ;
func parse_BooleanLiteral(token):
eat(token)
var value
if (TokenType.Bool_true == token) :
value = true
elif (TokenType.Bool_false == token) :
value = false
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BooleanLiteral
node.Value = value
return node
# > Literal
# NullLiteral
# : null
# ;
func parse_NullLiteral():
eat(TokenType.NullLiteral)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NullLiteral
node.Value = null
return node
var GParser = Parser.new()
onready var TextOut = GScene.get_node("TextOutput")
func tout(text):
TextOut.insert_text_at_cursor(text)
const Tests = \
{
MultiStatement = \
{
Name = "Multi-Statement",
File = "1.Multi-Statement.uf"
},
BlockStatement = \
{
Name = "Block Statement",
File = "2.BlockStatement.uf"
},
BinaryExpression = \
{
Name = "Binary Expression",
File = "3.BinaryExpression.uf"
},
Assignment = \
{
Name = "Assignment",
File = "4.Assignment.uf"
},
VaraibleDeclaration = \
{
Name = "Variable Declaration",
File = "5.VariableDeclaration.uf"
},
Conditionals = \
{
Name = "Conditionals",
File = "6.Conditionals.uf"
},
Relations = \
{
Name = "Relations",
File = "7.Relations.uf"
},
Equality = \
{
Name = "Equality",
File = "8.Equality.uf"
},
Logical = \
{
Name = "Logical",
File = "9.Logical.uf"
},
Unary = \
{
Name = "Unary",
File = "10.Unary.uf"
}
}
func test(entry):
var introMessage = "Testing: {Name}\n"
var introMessageFormatted = introMessage.format({"Name" : entry.Name})
tout(introMessageFormatted)
var path
if Engine.editor_hint :
path = "res://../Tests/{TestName}"
else :
path = "res://../Builds/Tests/{TestName}"
var pathFormatted = path.format({"TestName" : entry.File})
var \
file = File.new()
file.open(pathFormatted, File.READ)
var programDescription = file.get_as_text()
file.close()
GTokenizer.init(programDescription)
var ast = GParser.parse(GTokenizer)
var json = JSON.print(ast.to_Dictionary(), '\t')
tout(json + "\n")
tout("Passed!\n")
# Main Entry point.
func _ready():
for Key in Tests :
test(Tests[Key])

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -6,7 +6,7 @@ extends Node
const TokenType = \ const TokenType = \
{ {
Program = "Program", Program = "Program",
# Comments # Comments
CommentLine = "CommentLine", CommentLine = "CommentLine",
@ -16,13 +16,13 @@ const TokenType = \
Whitespace = "Whitespace", Whitespace = "Whitespace",
# Statements # Statements
StatementEnd = "StatementEnd", StatementEnd = "StatementEnd",
StmtBlockStart = "BlockStatementStart", StmtBlockStart = "BlockStatementStart",
StmtBlockEnd = "BlockStatementEnd", StmtBlockEnd = "BlockStatementEnd",
# Literals # Literals
Number = "Number", Number = "Number",
String = "String" String = "String"
} }
const TokenSpec = \ const TokenSpec = \
@ -81,7 +81,7 @@ class Tokenizer:
# Skip Whitespace # Skip Whitespace
if type == TokenType.Whitespace : if type == TokenType.Whitespace :
var addVal = result.get_string().length() var addVal = result.get_string().length()
self.Cursor += addVal self.Cursor += addVal
return next_Token() return next_Token()
@ -240,7 +240,7 @@ class Parser:
eat(TokenType.StatementEnd) eat(TokenType.StatementEnd)
var \ var \
node = SyntaxNode.new() node = SyntaxNode.new()
node.Type = SyntaxNodeType.EmptyStatement node.Type = SyntaxNodeType.EmptyStatement
return node return node

View File

@ -0,0 +1,485 @@
extends Node
# This closesly follows the source provided in the lectures.
# Later on after the lectures are complete or when I deem
# Necessary there will be heavy refactors.
const TokenType = \
{
Program = "Program",
# Comments
CommentLine = "CommentLine",
CommentMultiLine = "CommentMultiLine",
# Formatting
Whitespace = "Whitespace",
# Expressions
ExpressionPStart = "Expresssion Parenthesis Start",
ExpressionPEnd = "Expression Parenthesis End",
# Arithmetic
AdditiveOp = "AdditiveOperator",
MultiplicativeOp = "MultiplicativeOperator",
# Statements
StatementEnd = "StatementEnd",
StmtBlockStart = "BlockStatementStart",
StmtBlockEnd = "BlockStatementEnd",
# Literals
Number = "Number",
String = "String"
}
const TokenSpec = \
{
# Comments
TokenType.CommentLine : "^\/\/.*",
TokenType.CommentMultiLine : "^\/\\*[\\s\\S]*?\\*\/",
# Formatting
TokenType.Whitespace : "^\\s+",
# Expressions
TokenType.ExpressionPStart : "^\\(",
TokenType.ExpressionPEnd : "^\\)",
# Arithmetic
TokenType.AdditiveOp : "^[+\\-]",
TokenType.MultiplicativeOp : "^[*\\/]",
# Literal
TokenType.Number : "\\d+",
TokenType.String : "^\"[^\"]*\"",
# Statements
TokenType.StatementEnd : "^;",
TokenType.StmtBlockStart : "^{",
TokenType.StmtBlockEnd : "^}"
}
class Token:
var Type : String
var Value : String
func to_Dictionary():
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Tokenizer:
var SrcTxt : String
var Cursor : int;
# Sets up the tokenizer with the program source text.
func init(programSrcText):
SrcTxt = programSrcText
Cursor = 0
# Provides the next token in the source text.
func next_Token():
if reached_EndOfTxt() == true :
return null
var srcLeft = SrcTxt.substr(Cursor)
var regex = RegEx.new()
var token = Token.new()
for type in TokenSpec :
regex.compile(TokenSpec[type])
var result = regex.search(srcLeft)
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TokenType.CommentLine || type == TokenType.CommentMultiLine :
Cursor += result.get_string().length()
return next_Token()
# Skip Whitespace
if type == TokenType.Whitespace :
var addVal = result.get_string().length()
Cursor += addVal
return next_Token()
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
return token
var assertStrTmplt = "next_token: Source text not understood by tokenizer at Cursor pos: {value}"
var assertStr = assertStrTmplt.format({"value" : Cursor})
assert(true != true, assertStr)
return null
func reached_EndOfTxt():
return Cursor >= ( SrcTxt.length() )
var GTokenizer = Tokenizer.new()
const AST_Format = \
{
Dictionary = "Dictionary",
SExpression = "S-Expression"
}
const SyntaxNodeType = \
{
NumericLiteral = "NumericLiteral",
StringLiteral = "StringLiteral",
ExpressionStatement = "ExpressionStatement",
BlockStatement = "BlockStatement",
EmptyStatement = "EmptyStatement",
BinaryExpression = "BinaryExpression",
# MultiplicativeExpression = "MultiplicativeExpression"
}
class SyntaxNode:
var Type : String
var Value # Not specifing a type implicity declares a Variant type.
func to_SExpression():
var expression = [ Type ]
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_SExpression() )
else :
array.append( entry )
expression.append(array)
return expression
if typeof(Value) == TYPE_OBJECT :
var result = [ Type, Value.to_SExpression() ]
return result
expression.append(Value)
return expression
func to_Dictionary():
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_Dictionary() )
else :
array.append( entry )
var result = \
{
Type = self.Type,
Value = array
}
return result
if typeof(Value) == TYPE_OBJECT :
var result = \
{
Type = self.Type,
Value = self.Value.to_Dictionary()
}
return result
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Parser:
var TokenizerRef : Tokenizer
var NextToken : Token
func eat(tokenType):
var currToken = self.NextToken
assert(currToken != null, "eat: NextToken was null")
var assertStrTmplt = "eat: Unexpected token: {value}, expected: {type}"
var assertStr = assertStrTmplt.format({"value" : currToken.Value, "type" : tokenType})
assert(currToken.Type == tokenType, assertStr)
NextToken = TokenizerRef.next_Token()
return currToken
# NumericLiteral
# : Number
# ;
func parse_NumericLiteral():
var Token = eat(TokenType.Number)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NumericLiteral
node.Value = int( Token.Value )
return node
# StringLiteral
# : String
# ;
func parse_StringLiteral():
var Token = eat(TokenType.String)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.StringLiteral
node.Value = Token.Value.substr( 1, Token.Value.length() - 2 )
return node
# Literal
# : NumericLiteral
# : StringLiteral
# ;
func parse_Literal():
match NextToken.Type :
TokenType.Number:
return parse_NumericLiteral()
TokenType.String:
return parse_StringLiteral()
assert(false, "parse_Literal: Was not able to detect valid literal type from NextToken")
# ParenthesizedExpression
# : ( Expression )
# ;
func parse_ParenthesizedExpression():
eat(TokenType.ExpressionPStart)
var expression = parse_Expression()
eat(TokenType.ExpressionPEnd)
return expression
# PrimaryExpression
# : Literal
# | ParenthesizedExpression
# ;
func parse_PrimaryExpression():
match NextToken.Type:
TokenType.ExpressionPStart:
return parse_ParenthesizedExpression()
return parse_Literal()
# MultiplicativeExpression
# : PrimaryExpression
# : MultiplicativeExpression MultiplicativeOp PrimaryExpression -> PrimaryExpression MultiplicativeOp ... Literal
# ;
func parse_MultiplicativeExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_PrimaryExpression")
return parse_BinaryExpression(parseFn, TokenType.MultiplicativeOp)
# AdditiveExpression
# : MultiplicativeExpression
# | AdditiveExpression AdditiveOp MultiplicativeExpression -> MultiplicativeExpression AdditiveOp ... Literal
# ;
func parse_AdditiveExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_MultiplicativeExpression")
return parse_BinaryExpression(parseFn, TokenType.AdditiveOp)
# BinaryExpression
# : MultiplicativeExpression
# | AdditiveExpression
# ;
func parse_BinaryExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken:
var operator = eat(operatorToken)
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.BinaryExpression
nestedNode.Value = []
nestedNode.Value.append(operator.Value)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode;
return left
# Expression
# : Literal
# : AdditiveExpression
# ;
func parse_Expression():
return parse_AdditiveExpression()
# EmptyStatement
# ;
func parse_EmptyStatement():
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.EmptyStatement
return node
# BlockStatement
# : { OptStatementList }
# ;
func parse_BlockStatement():
eat(TokenType.StmtBlockStart)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BlockStatement
if NextToken.Type != TokenType.StmtBlockEnd :
node.Value = parse_StatementList(TokenType.StmtBlockEnd)
else :
node.Value = []
eat(TokenType.StmtBlockEnd)
return node
# ExpressionStatement
# : Expression
# ;
func parse_ExpressionStatement():
var expression = parse_Expression()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ExpressionStatement
node.Value = expression
return expression
# Statement
# : ExpressionStatement
# : BlockStatement
# : EmptyStatement
# ;
func parse_Statement():
match NextToken.Type :
TokenType.StatementEnd :
return parse_EmptyStatement()
TokenType.StmtBlockStart :
return parse_BlockStatement()
return parse_ExpressionStatement()
# StatementList
# : Statement
# | StatementList Statement -> Statement ...
# ;
func parse_StatementList(endToken):
var statementList = [ parse_Statement() ]
while NextToken != null && NextToken.Type != endToken :
statementList.append( parse_Statement() )
return statementList
# Program
# : StatementList
# : Literal
# ;
func parse_Program():
var \
node = SyntaxNode.new()
node.Type = TokenType.Program
node.Value = parse_StatementList(null)
return node
# Parses the text program description into an AST.
func parse(TokenizerRef):
self.TokenizerRef = TokenizerRef
NextToken = TokenizerRef.next_Token()
return parse_Program()
var GParser = Parser.new()
onready var TextOut = GScene.get_node("TextOutput")
func tout(text):
TextOut.insert_text_at_cursor(text)
const Tests = \
{
MultiStatement = \
{
Name = "Multi-Statement",
File = "1.Multi-Statement.uf"
},
BlockStatement = \
{
Name = "Block Statement",
File = "2.BlockStatement.uf"
},
BinaryExpression = \
{
Name = "Binary Expression",
File = "3.BinaryExpression.uf"
}
}
func test(entry):
var introMessage = "Testing: {Name}\n"
var introMessageFormatted = introMessage.format({"Name" : entry.Name})
tout(introMessageFormatted)
var path
if Engine.editor_hint :
path = "res://../Tests/{TestName}"
else :
path = "res://../Builds/Tests/{TestName}"
var pathFormatted = path.format({"TestName" : entry.File})
var \
file = File.new()
file.open(pathFormatted, File.READ)
var programDescription = file.get_as_text()
file.close()
GTokenizer.init(programDescription)
var ast = GParser.parse(GTokenizer)
var json = JSON.print(ast.to_SExpression(), '\t')
tout(json + "\n")
tout("Passed!\n")
# Main Entry point.
func _ready():
for Key in Tests :
test(Tests[Key])

View File

@ -0,0 +1,563 @@
extends Node
# This closesly follows the source provided in the lectures.
# Later on after the lectures are complete or when I deem
# Necessary there will be heavy refactors.
const TokenType = \
{
Program = "Program",
# Comments
CommentLine = "CommentLine",
CommentMultiLine = "CommentMultiLine",
# Formatting
Whitespace = "Whitespace",
# Expressions
ExpressionPStart = "Expresssion Parenthesis Start",
ExpressionPEnd = "Expression Parenthesis End",
# Arithmetic
ComplexAssignment = "ComplexAssignment",
Assignment = "Assignment",
AdditiveOp = "AdditiveOperator",
MultiplicativeOp = "MultiplicativeOperator",
# Statements
StatementEnd = "StatementEnd",
StmtBlockStart = "BlockStatementStart",
StmtBlockEnd = "BlockStatementEnd",
# Literals
Number = "Number",
String = "String",
# Symbols
Identifier = "Identifier"
}
const TokenSpec = \
{
# Comments
TokenType.CommentLine : "^\/\/.*",
TokenType.CommentMultiLine : "^\/\\*[\\s\\S]*?\\*\/",
# Formatting
TokenType.Whitespace : "^\\s+",
# Expressions
TokenType.ExpressionPStart : "^\\(",
TokenType.ExpressionPEnd : "^\\)",
# Arithmetic
TokenType.ComplexAssignment : "^[*\\/\\+\\-]=",
TokenType.Assignment : "^=",
TokenType.AdditiveOp : "^[+\\-]",
TokenType.MultiplicativeOp : "^[*\\/]",
# Literal
TokenType.Number : "\\d+",
TokenType.String : "^\"[^\"]*\"",
# Statements
TokenType.StatementEnd : "^;",
TokenType.StmtBlockStart : "^{",
TokenType.StmtBlockEnd : "^}",
# Symbols
TokenType.Identifier : "^\\w+"
}
class Token:
var Type : String
var Value : String
func to_Dictionary():
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Tokenizer:
var SrcTxt : String
var Cursor : int;
# Sets up the tokenizer with the program source text.
func init(programSrcText):
SrcTxt = programSrcText
Cursor = 0
# Provides the next token in the source text.
func next_Token():
if reached_EndOfTxt() == true :
return null
var srcLeft = SrcTxt.substr(Cursor)
var regex = RegEx.new()
var token = Token.new()
for type in TokenSpec :
regex.compile(TokenSpec[type])
var result = regex.search(srcLeft)
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TokenType.CommentLine || type == TokenType.CommentMultiLine :
Cursor += result.get_string().length()
return next_Token()
# Skip Whitespace
if type == TokenType.Whitespace :
var addVal = result.get_string().length()
Cursor += addVal
return next_Token()
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
return token
var assertStrTmplt = "next_token: Source text not understood by tokenizer at Cursor pos: {value}"
var assertStr = assertStrTmplt.format({"value" : Cursor})
assert(true != true, assertStr)
return null
func reached_EndOfTxt():
return Cursor >= ( SrcTxt.length() )
var GTokenizer = Tokenizer.new()
const AST_Format = \
{
Dictionary = "Dictionary",
SExpression = "S-Expression"
}
const SyntaxNodeType = \
{
NumericLiteral = "NumericLiteral",
StringLiteral = "StringLiteral",
ExpressionStatement = "ExpressionStatement",
BlockStatement = "BlockStatement",
EmptyStatement = "EmptyStatement",
BinaryExpression = "BinaryExpression",
Identifier = "Identifier",
AssignmentExpression = "AssignmentExpression"
}
class SyntaxNode:
var Type : String
var Value # Not specifing a type implicity declares a Variant type.
func to_SExpression():
var expression = [ Type ]
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_SExpression() )
else :
array.append( entry )
expression.append(array)
return expression
if typeof(Value) == TYPE_OBJECT :
var result = [ Type, Value.to_SExpression() ]
return result
expression.append(Value)
return expression
func to_Dictionary():
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_Dictionary() )
else :
array.append( entry )
var result = \
{
Type = self.Type,
Value = array
}
return result
if typeof(Value) == TYPE_OBJECT :
var result = \
{
Type = self.Type,
Value = self.Value.to_Dictionary()
}
return result
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Parser:
var TokenizerRef : Tokenizer
var NextToken : Token
func is_Literal():
return NextToken.Type == TokenType.Number || NextToken.Type == TokenType.String
func eat(tokenType):
var currToken = self.NextToken
assert(currToken != null, "eat: NextToken was null")
var assertStrTmplt = "eat: Unexpected token: {value}, expected: {type}"
var assertStr = assertStrTmplt.format({"value" : currToken.Value, "type" : tokenType})
assert(currToken.Type == tokenType, assertStr)
NextToken = TokenizerRef.next_Token()
return currToken
# Literal
# : NumericLiteral
# : StringLiteral
# ;
func parse_Literal():
match NextToken.Type :
TokenType.Number:
return parse_NumericLiteral()
TokenType.String:
return parse_StringLiteral()
assert(false, "parse_Literal: Was not able to detect valid literal type from NextToken")
# NumericLiteral
# : Number
# ;
func parse_NumericLiteral():
var Token = eat(TokenType.Number)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NumericLiteral
node.Value = int( Token.Value )
return node
# StringLiteral
# : String
# ;
func parse_StringLiteral():
var Token = eat(TokenType.String)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.StringLiteral
node.Value = Token.Value.substr( 1, Token.Value.length() - 2 )
return node
# ParenthesizedExpression
# : ( Expression )
# ;
func parse_ParenthesizedExpression():
eat(TokenType.ExpressionPStart)
var expression = parse_Expression()
eat(TokenType.ExpressionPEnd)
return expression
# MultiplicativeExpression
# : PrimaryExpression
# : MultiplicativeExpression MultiplicativeOp PrimaryExpression -> PrimaryExpression MultiplicativeOp ... Literal
# ;
func parse_MultiplicativeExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_PrimaryExpression")
return parse_BinaryExpression(parseFn, TokenType.MultiplicativeOp)
# AdditiveExpression
# : MultiplicativeExpression
# | AdditiveExpression AdditiveOp MultiplicativeExpression -> MultiplicativeExpression AdditiveOp ... Literal
# ;
func parse_AdditiveExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_MultiplicativeExpression")
return parse_BinaryExpression(parseFn, TokenType.AdditiveOp)
# BinaryExpression
# : MultiplicativeExpression
# | AdditiveExpression
# ;
func parse_BinaryExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken:
var operator = eat(operatorToken)
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.BinaryExpression
nestedNode.Value = []
nestedNode.Value.append(operator.Value)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode;
return left
# Identifier
# : IdentifierSymbol
# ;
func parse_Identifier():
var name = eat(TokenType.Identifier).Value
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.Identifier
node.Value = name
return node
# ResolvedSymbol
# : Identiifer
# ;
func parse_ResolvedSymbol():
var resolvedSymbol = parse_Identifier()
if resolvedSymbol.Type == SyntaxNodeType.Identifier :
return resolvedSymbol
var assertStrTmplt = "parse_ResolvedSymbol: Unexpected symbol: {value}"
var assertStr = assertStrTmplt.format({"value" : resolvedSymbol.Type})
assert(true != true, assertStr)
# PrimaryExpression
# : Literal
# | ParenthesizedExpression
# | ResolvedSymbol
# ;
func parse_PrimaryExpression():
if is_Literal():
return parse_Literal()
match NextToken.Type:
TokenType.ExpressionPStart:
return parse_ParenthesizedExpression()
return parse_ResolvedSymbol()
# AssignmentExpression
# : AdditiveExpression
# | ResolvedSymbol AssignmentOperator AssignmetnExpression
# ;
func parse_AssignmentExpression():
var left = parse_AdditiveExpression()
if NextToken.Type != TokenType.Assignment && NextToken.Type != TokenType.ComplexAssignment :
return left
var assignmentOp;
if NextToken.Type == TokenType.Assignment :
assignmentOp = eat(TokenType.Assignment)
elif NextToken.Type == TokenType.ComplexAssignment :
assignmentOp = eat(TokenType.ComplexAssignment)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.AssignmentExpression
node.Value = \
[
assignmentOp.Value,
left,
parse_AssignmentExpression()
]
return node
# Expression
# : AssignmentExpression
# ;
func parse_Expression():
return parse_AssignmentExpression()
# EmptyStatement
# ;
func parse_EmptyStatement():
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.EmptyStatement
return node
# BlockStatement
# : { OptStatementList }
# ;
func parse_BlockStatement():
eat(TokenType.StmtBlockStart)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BlockStatement
if NextToken.Type != TokenType.StmtBlockEnd :
node.Value = parse_StatementList(TokenType.StmtBlockEnd)
else :
node.Value = []
eat(TokenType.StmtBlockEnd)
return node
# ExpressionStatement
# : Expression
# ;
func parse_ExpressionStatement():
var expression = parse_Expression()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ExpressionStatement
node.Value = expression
return expression
# Statement
# : ExpressionStatement
# : BlockStatement
# : EmptyStatement
# ;
func parse_Statement():
match NextToken.Type :
TokenType.StatementEnd :
return parse_EmptyStatement()
TokenType.StmtBlockStart :
return parse_BlockStatement()
return parse_ExpressionStatement()
# StatementList
# : Statement
# | StatementList Statement -> Statement ...
# ;
func parse_StatementList(endToken):
var statementList = [ parse_Statement() ]
while NextToken != null && NextToken.Type != endToken :
statementList.append( parse_Statement() )
return statementList
# Program
# : StatementList
# : Literal
# ;
func parse_Program():
var \
node = SyntaxNode.new()
node.Type = TokenType.Program
node.Value = parse_StatementList(null)
return node
# Parses the text program description into an AST.
func parse(TokenizerRef):
self.TokenizerRef = TokenizerRef
NextToken = TokenizerRef.next_Token()
return parse_Program()
var GParser = Parser.new()
onready var TextOut = GScene.get_node("TextOutput")
func tout(text):
TextOut.insert_text_at_cursor(text)
const Tests = \
{
MultiStatement = \
{
Name = "Multi-Statement",
File = "1.Multi-Statement.uf"
},
BlockStatement = \
{
Name = "Block Statement",
File = "2.BlockStatement.uf"
},
BinaryExpression = \
{
Name = "Binary Expression",
File = "3.BinaryExpression.uf"
},
Assignment = \
{
Name = "Assignment",
File = "4.Assignment.uf"
}
}
func test(entry):
var introMessage = "Testing: {Name}\n"
var introMessageFormatted = introMessage.format({"Name" : entry.Name})
tout(introMessageFormatted)
var path
if Engine.editor_hint :
path = "res://../Tests/{TestName}"
else :
path = "res://../Builds/Tests/{TestName}"
var pathFormatted = path.format({"TestName" : entry.File})
var \
file = File.new()
file.open(pathFormatted, File.READ)
var programDescription = file.get_as_text()
file.close()
GTokenizer.init(programDescription)
var ast = GParser.parse(GTokenizer)
var json = JSON.print(ast.to_SExpression(), '\t')
tout(json + "\n")
tout("Passed!\n")
# Main Entry point.
func _ready():
for Key in Tests :
test(Tests[Key])

View File

@ -0,0 +1,634 @@
extends Node
# This closesly follows the source provided in the lectures.
# Later on after the lectures are complete or when I deem
# Necessary there will be heavy refactors.
const TokenType = \
{
Program = "Program",
# Comments
CommentLine = "CommentLine",
CommentMultiLine = "CommentMultiLine",
# Formatting
Whitespace = "Whitespace",
# Expressions
ExpressionPStart = "ExpresssionParenthesisStart",
ExpressionPEnd = "ExpressionParenthesisEnd",
# Arithmetic
ComplexAssignment = "ComplexAssignment",
Assignment = "Assignment",
AdditiveOp = "AdditiveOperator",
MultiplicativeOp = "MultiplicativeOperator",
# Statements
StatementEnd = "StatementEnd",
StmtBlockStart = "BlockStatementStart",
StmtBlockEnd = "BlockStatementEnd",
CommaDelimiter = "CommaDelimiter",
# Literals
Number = "Number",
String = "String",
# Symbols
VarDeclare = "Variable Declaration",
Identifier = "Identifier"
}
const TokenSpec = \
{
# Comments
TokenType.CommentLine : "^\\/\\/.*",
TokenType.CommentMultiLine : "^\\/\\*[\\s\\S]*?\\*\\/",
# Formatting
TokenType.Whitespace : "^\\s+",
# Expressions
TokenType.ExpressionPStart : "^\\(",
TokenType.ExpressionPEnd : "^\\)",
# Arithmetic
TokenType.ComplexAssignment : "^[*\\/\\+\\-]=",
TokenType.Assignment : "^=",
TokenType.AdditiveOp : "^[+\\-]",
TokenType.MultiplicativeOp : "^[*\\/]",
# Literal
TokenType.Number : "\\d+",
TokenType.String : "^\"[^\"]*\"",
# Statements
TokenType.StatementEnd : "^;",
TokenType.StmtBlockStart : "^{",
TokenType.StmtBlockEnd : "^}",
TokenType.CommaDelimiter : "^,",
# Symbols
TokenType.VarDeclare : "^\\blet\\b",
TokenType.Identifier : "^\\w+"
}
class Token:
var Type : String
var Value : String
func to_Dictionary():
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Tokenizer:
var SrcTxt : String
var Cursor : int;
# Sets up the tokenizer with the program source text.
func init(programSrcText):
SrcTxt = programSrcText
Cursor = 0
# Provides the next token in the source text.
func next_Token():
if reached_EndOfTxt() == true :
return null
var srcLeft = SrcTxt.substr(Cursor)
var regex = RegEx.new()
var token = Token.new()
for type in TokenSpec :
regex.compile(TokenSpec[type])
var result = regex.search(srcLeft)
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TokenType.CommentLine || type == TokenType.CommentMultiLine :
Cursor += result.get_string().length()
return next_Token()
# Skip Whitespace
if type == TokenType.Whitespace :
var addVal = result.get_string().length()
Cursor += addVal
return next_Token()
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
return token
var assertStrTmplt = "next_token: Source text not understood by tokenizer at Cursor pos: {value}"
var assertStr = assertStrTmplt.format({"value" : Cursor})
assert(true != true, assertStr)
return null
func reached_EndOfTxt():
return Cursor >= ( SrcTxt.length() )
var GTokenizer = Tokenizer.new()
const AST_Format = \
{
Dictionary = "Dictionary",
SExpression = "S-Expression"
}
const SyntaxNodeType = \
{
NumericLiteral = "NumericLiteral",
StringLiteral = "StringLiteral",
ExpressionStatement = "ExpressionStatement",
BlockStatement = "BlockStatement",
EmptyStatement = "EmptyStatement",
BinaryExpression = "BinaryExpression",
Identifier = "Identifier",
AssignmentExpression = "AssignmentExpression",
VariableStatement = "VariableStatement",
VariableDeclaration = "VariableDeclaration"
}
class SyntaxNode:
var Type : String
var Value # Not specifing a type implicity declares a Variant type.
func to_SExpression():
var expression = [ Type ]
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_SExpression() )
else :
array.append( entry )
expression.append(array)
return expression
if typeof(Value) == TYPE_OBJECT :
var result = [ Type, Value.to_SExpression() ]
return result
expression.append(Value)
return expression
func to_Dictionary():
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_Dictionary() )
else :
array.append( entry )
var result = \
{
Type = self.Type,
Value = array
}
return result
if typeof(Value) == TYPE_OBJECT :
var result = \
{
Type = self.Type,
Value = self.Value.to_Dictionary()
}
return result
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Parser:
var TokenizerRef : Tokenizer
var NextToken : Token
func is_Literal():
return NextToken.Type == TokenType.Number || NextToken.Type == TokenType.String
func eat(tokenType):
var currToken = self.NextToken
assert(currToken != null, "eat: NextToken was null")
var assertStrTmplt = "eat: Unexpected token: {value}, expected: {type}"
var assertStr = assertStrTmplt.format({"value" : currToken.Value, "type" : tokenType})
assert(currToken.Type == tokenType, assertStr)
NextToken = TokenizerRef.next_Token()
return currToken
# Literal
# : NumericLiteral
# : StringLiteral
# ;
func parse_Literal():
match NextToken.Type :
TokenType.Number:
return parse_NumericLiteral()
TokenType.String:
return parse_StringLiteral()
assert(false, "parse_Literal: Was not able to detect valid literal type from NextToken")
# NumericLiteral
# : Number
# ;
func parse_NumericLiteral():
var Token = eat(TokenType.Number)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NumericLiteral
node.Value = int( Token.Value )
return node
# StringLiteral
# : String
# ;
func parse_StringLiteral():
var Token = eat(TokenType.String)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.StringLiteral
node.Value = Token.Value.substr( 1, Token.Value.length() - 2 )
return node
# ParenthesizedExpression
# : ( Expression )
# ;
func parse_ParenthesizedExpression():
eat(TokenType.ExpressionPStart)
var expression = parse_Expression()
eat(TokenType.ExpressionPEnd)
return expression
# MultiplicativeExpression
# : PrimaryExpression
# : MultiplicativeExpression MultiplicativeOp PrimaryExpression -> PrimaryExpression MultiplicativeOp ... Literal
# ;
func parse_MultiplicativeExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_PrimaryExpression")
return parse_BinaryExpression(parseFn, TokenType.MultiplicativeOp)
# AdditiveExpression
# : MultiplicativeExpression
# | AdditiveExpression AdditiveOp MultiplicativeExpression -> MultiplicativeExpression AdditiveOp ... Literal
# ;
func parse_AdditiveExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_MultiplicativeExpression")
return parse_BinaryExpression(parseFn, TokenType.AdditiveOp)
# BinaryExpression
# : MultiplicativeExpression
# | AdditiveExpression
# ;
func parse_BinaryExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken:
var operator = eat(operatorToken)
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.BinaryExpression
nestedNode.Value = []
nestedNode.Value.append(operator.Value)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode;
return left
# Identifier
# : IdentifierSymbol
# ;
func parse_Identifier():
var name = eat(TokenType.Identifier).Value
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.Identifier
node.Value = name
return node
# ResolvedSymbol
# : Identiifer
# ;
func parse_ResolvedSymbol():
var resolvedSymbol = parse_Identifier()
if resolvedSymbol.Type == SyntaxNodeType.Identifier :
return resolvedSymbol
var assertStrTmplt = "parse_ResolvedSymbol: Unexpected symbol: {value}"
var assertStr = assertStrTmplt.format({"value" : resolvedSymbol.Type})
assert(true != true, assertStr)
# PrimaryExpression
# : Literal
# | ParenthesizedExpression
# | ResolvedSymbol
# ;
func parse_PrimaryExpression():
if is_Literal():
return parse_Literal()
match NextToken.Type:
TokenType.ExpressionPStart:
return parse_ParenthesizedExpression()
return parse_ResolvedSymbol()
# AssignmentExpression
# : AdditiveExpression
# | ResolvedSymbol AssignmentOperator AssignmetnExpression
# ;
func parse_AssignmentExpression():
var left = parse_AdditiveExpression()
if NextToken.Type != TokenType.Assignment && NextToken.Type != TokenType.ComplexAssignment :
return left
var assignmentOp;
if NextToken.Type == TokenType.Assignment :
assignmentOp = eat(TokenType.Assignment)
elif NextToken.Type == TokenType.ComplexAssignment :
assignmentOp = eat(TokenType.ComplexAssignment)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.AssignmentExpression
node.Value = \
[
assignmentOp.Value,
left,
parse_AssignmentExpression()
]
return node
# Expression
# : AssignmentExpression
# ;
func parse_Expression():
return parse_AssignmentExpression()
# EmptyStatement
# ;
func parse_EmptyStatement():
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.EmptyStatement
return node
# VariableInitializer
# : Assignment AssignmentExpression
# ;
func parse_VariableInitializer():
eat(TokenType.Assignment)
return parse_AssignmentExpression()
# VariableDeclaration
# : Identifier OptVariableInitalizer
# ;
func parse_VariableDeclaration():
var identifier = parse_Identifier()
var initalizer
if NextToken.Type != TokenType.StatementEnd && NextToken.Type != TokenType.CommaDelimiter :
initalizer = parse_VariableInitializer()
else :
initalizer = null
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableDeclaration
node.Value = [ identifier, initalizer ]
return node
# VariableDeclarationList
# : VariableDeclaration
# | VariableDelcarationList , VariableDeclaration -> VariableDelcaration , ...
func parse_VariableDeclarationList():
var \
declarations = []
declarations.append(parse_VariableDeclaration())
while NextToken.Type == TokenType.CommaDelimiter :
eat(TokenType.CommaDelimiter)
declarations.append(parse_VariableDeclaration())
return declarations
# VariableStatement
# : VarDeclare VariableDeclarationList StatementEnd
# ;
func parse_VariableStatement():
eat(TokenType.VarDeclare)
var declarations = parse_VariableDeclarationList()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableStatement
node.Value = declarations
return node
# BlockStatement
# : { OptStatementList }
# ;
func parse_BlockStatement():
eat(TokenType.StmtBlockStart)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BlockStatement
if NextToken.Type != TokenType.StmtBlockEnd :
node.Value = parse_StatementList(TokenType.StmtBlockEnd)
else :
node.Value = []
eat(TokenType.StmtBlockEnd)
return node
# ExpressionStatement
# : Expression
# ;
func parse_ExpressionStatement():
var expression = parse_Expression()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ExpressionStatement
node.Value = expression
return expression
# Statement
# : ExpressionStatement
# | BlockStatement
# | EmptyStatement
# | VariableStatement
# ;
func parse_Statement():
match NextToken.Type :
TokenType.StatementEnd :
return parse_EmptyStatement()
TokenType.StmtBlockStart :
return parse_BlockStatement()
TokenType.VarDeclare :
return parse_VariableStatement()
return parse_ExpressionStatement()
# StatementList
# : Statement
# | StatementList Statement -> Statement ...
# ;
func parse_StatementList(endToken):
var statementList = [ parse_Statement() ]
while NextToken != null && NextToken.Type != endToken :
statementList.append( parse_Statement() )
return statementList
# Program
# : StatementList
# : Literal
# ;
func parse_Program():
var \
node = SyntaxNode.new()
node.Type = TokenType.Program
node.Value = parse_StatementList(null)
return node
# Parses the text program description into an AST.
func parse(TokenizerRef):
self.TokenizerRef = TokenizerRef
NextToken = TokenizerRef.next_Token()
return parse_Program()
var GParser = Parser.new()
onready var TextOut = GScene.get_node("TextOutput")
func tout(text):
TextOut.insert_text_at_cursor(text)
const Tests = \
{
MultiStatement = \
{
Name = "Multi-Statement",
File = "1.Multi-Statement.uf"
},
BlockStatement = \
{
Name = "Block Statement",
File = "2.BlockStatement.uf"
},
BinaryExpression = \
{
Name = "Binary Expression",
File = "3.BinaryExpression.uf"
},
Assignment = \
{
Name = "Assignment",
File = "4.Assignment.uf"
},
VaraibleDeclaration = \
{
Name = "Variable Declaration",
File = "5.VariableDeclaration.uf"
}
}
func test(entry):
var introMessage = "Testing: {Name}\n"
var introMessageFormatted = introMessage.format({"Name" : entry.Name})
tout(introMessageFormatted)
var path
if Engine.editor_hint :
path = "res://../Tests/{TestName}"
else :
path = "res://../Builds/Tests/{TestName}"
var pathFormatted = path.format({"TestName" : entry.File})
var \
file = File.new()
file.open(pathFormatted, File.READ)
var programDescription = file.get_as_text()
file.close()
GTokenizer.init(programDescription)
var ast = GParser.parse(GTokenizer)
var json = JSON.print(ast.to_SExpression(), '\t')
tout(json + "\n")
tout("Passed!\n")
# Main Entry point.
func _ready():
for Key in Tests :
test(Tests[Key])

View File

@ -1,92 +1,92 @@
Following the first lecture of "Building a Parser from scratch" Following the first lecture of "Building a Parser from scratch"
By Dmitry Soshnikov. By Dmitry Soshnikov.
Lecture 1: Lecture 1:
Phases: Phases:
Data - Text Content Data - Text Content
Processor - Tokenizer Processor - Tokenizer
Data - Tokens Data - Tokens
Processor - Parser Processor - Parser
Data - AST Data - AST
Example of syntaxes : Example of syntaxes :
S-Expression : S-Expression :
(class Point (class Point
(begin (begin
(def constructor (self x y) (def constructor (self x y)
(begin (begin
(set (prop self x) x) (set (prop self x) x)
(set (prop self y) y) (set (prop self y) y)
) )
) )
(def calc (self) (def calc (self)
(+ (prop self x) (+ (prop self x)
(prop self y) (prop self y)
) )
) )
) )
) )
(var p (new Point 10 20)) (var p (new Point 10 20))
((prop p calc) p) ((prop p calc) p)
User Syntax : User Syntax :
class Point class Point
{ {
def constructor( x, y ) def constructor( x, y )
{ {
this.x = x; this.x = x;
this.y = y; this.y = y;
} }
def calc() { def calc() {
return this.x + this.y; return this.x + this.y;
} }
} }
let let
p = new Point(10, 20); p = new Point(10, 20);
p.calc(); p.calc();
Tokenizer - Lexial Analysis : Uses Regular Expressions (Optimal) Tokenizer - Lexial Analysis : Uses Regular Expressions (Optimal)
Parser - Syntactic Analysis : Uses Backus-Naur Form Parser - Syntactic Analysis : Uses Backus-Naur Form
Backus-Naur Example : Backus-Naur Example :
Program Program
: StatementList : StatementList
; ;
StatementList StatementList
: BlockStatement : BlockStatement
| IfStatement | IfStatement
| FunctionDeclaration | FunctionDeclaration
... ...
; ;
FunctionDeclaration FunctionDeclaration
: def Identifier ( Arguments ) BlockStatement : def Identifier ( Arguments ) BlockStatement
; ;
Hand-written parsers : Hand-written parsers :
Use recursive descent. Use recursive descent.
Automatically generated Automatically generated
All kinds of stuff... All kinds of stuff...

91
App/RDP/RDP_Viewer.tscn Normal file
View File

@ -0,0 +1,91 @@
[gd_scene load_steps=4 format=2]
[ext_resource path="res://Assets/Styles/Purple.EditorTheme.tres" type="Theme" id=1]
[ext_resource path="res://Assets/Branding/RDP_Class_cover_small.png" type="Texture" id=2]
[ext_resource path="res://RDP/Scripts/RDP_Viewer.gd" type="Script" id=3]
[node name="RDP_Panel" type="Panel"]
anchor_right = 1.0
anchor_bottom = 1.0
margin_left = -9.0
margin_top = 1.0
margin_right = -9.0
margin_bottom = 1.0
theme = ExtResource( 1 )
script = ExtResource( 3 )
[node name="CourseBrand" type="TextureRect" parent="."]
anchor_right = 1.0
anchor_bottom = 1.0
rect_scale = Vector2( 0.2, 0.2 )
texture = ExtResource( 2 )
expand = true
stretch_mode = 6
[node name="Tokens_TOut" type="TextEdit" parent="."]
anchor_left = 0.2
anchor_right = 0.465
anchor_bottom = 1.0
margin_left = 1.2
margin_right = -0.136017
grow_horizontal = 0
theme = ExtResource( 1 )
readonly = true
highlight_current_line = true
show_line_numbers = true
[node name="AST_TOut" type="TextEdit" parent="."]
anchor_left = 0.465
anchor_right = 1.0
anchor_bottom = 1.0
margin_left = 1.83997
grow_horizontal = 0
theme = ExtResource( 1 )
readonly = true
highlight_current_line = true
show_line_numbers = true
minimap_draw = true
[node name="Letter_FDialog" type="FileDialog" parent="."]
anchor_left = 0.35
anchor_top = 0.15
anchor_right = 0.45
anchor_bottom = 0.25
margin_right = 356.0
margin_bottom = 373.0
theme = ExtResource( 1 )
popup_exclusive = true
window_title = "Open a File"
mode = 0
access = 2
[node name="VBox" type="VBoxContainer" parent="."]
anchor_top = 0.196
anchor_right = 0.2
anchor_bottom = 1.0
margin_top = 2.39999
margin_right = -1.8
margin_bottom = -2.0
[node name="ParseLetterFile_Btn" type="Button" parent="VBox"]
margin_right = 203.0
margin_bottom = 32.0
focus_neighbour_top = NodePath("../../CourseBrand")
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
text = "Parse a letter file"
[node name="Separator" type="HSeparator" parent="VBox"]
modulate = Color( 0.145098, 0.145098, 0.164706, 0 )
margin_top = 36.0
margin_right = 203.0
margin_bottom = 441.0
size_flags_vertical = 15
[node name="Back_Btn" type="Button" parent="VBox"]
margin_top = 445.0
margin_right = 203.0
margin_bottom = 478.0
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
text = "Course Directory"

13
App/RDP/Readme.md Normal file
View File

@ -0,0 +1,13 @@
# Building a parser from scratch
![Img](https://i.imgur.com/rEzWwGs.png)
## Lecutres
Contains gd scripts that show the resulting progress at the end of each of their corresponding lectures.
## Scripts
Contains a final set of scripts for the course that cleans up the implementation to a final state.

View File

@ -0,0 +1,305 @@
extends Object
var SRegEx = preload("res://RegM/Scripts/SRegex.gd").new()
class_name RDP_Lexer
const TokenType : Dictionary = \
{
# Comments
cmt_SL = "Comment Single Line",
cmt_ML = "Comment Multi-Line",
# Formatting
fmt_S = "Formatting String",
# Delimiters
delim_Comma = "Comma Delimiter",
delim_SMR = "Symbol Member Resolution",
# Statements
def_End = "Statement End",
def_BStart = "Block Start",
def_BEnd = "Block End",
def_Var = "Variable Declaration",
def_Class = "Class",
# Iteration
def_While = "While",
def_Do = "Do-While",
def_For = "For",
# Procedures
def_Proc = "Procedure Declaration",
def_Return = "Return",
# Conditional
def_If = "If Statement",
def_Else = "Else Statement",
# Expressions
expr_PStart = "Parenthesis Start",
expr_PEnd = "Parenthesis End",
expr_SBStart = "Bracket Start",
expr_SBEnd = "Bracket End",
expr_New = "New Expression",
expr_Super = "Super Expression",
expr_Extends = "Class Extension",
# Operators
# Logical
op_Relational = "Relational",
op_Equality = "Equality",
op_LAnd = "Logical And",
op_LOr = "Logical Or",
op_LNot = "Logical Not",
# Arithmetic
op_CAssign = "ComplexAssignment",
op_Assign = "Assignment",
op_Additive = "AdditiveOperator",
op_Multiplicative = "MultiplicativeOperator",
# Literals
literal_BTrue = "True",
literal_BFalse = "False",
literal_Number = "Number",
literal_String = "String",
literal_Null = "Null Value",
# Symbols
sym_This = "This Reference",
sym_Identifier = "User Identifier",
}
const Spec : Dictionary = \
{
# Comments
TokenType.cmt_SL : "^\\/\\/.*",
TokenType.cmt_ML : "^\\/\\*[\\s\\S]*?\\*\\/",
# Formatting
TokenType.fmt_S : "^\\s+",
# Delimiters
TokenType.delim_Comma : "^,",
TokenType.delim_SMR : "^\\.",
# Statements
TokenType.def_End : "^;",
TokenType.def_BStart : "^{",
TokenType.def_BEnd : "^}",
TokenType.def_Var : "^\\blet\\b",
TokenType.def_Class : "^\\bclass\\b",
# Iteration
TokenType.def_While : "^\\bwhile\\b",
TokenType.def_Do : "^\\bdo\\b",
TokenType.def_For : "^\\bfor\\b",
# Procedures
TokenType.def_Proc : "^\\bdef\\b",
TokenType.def_Return : "^\\breturn\\b",
# Conditional
TokenType.def_If : "^\\bif\\b",
TokenType.def_Else : "^\\belse\\b",
# Expressions
TokenType.expr_PStart : "^\\(",
TokenType.expr_PEnd : "^\\)",
TokenType.expr_SBStart : "^\\[",
TokenType.expr_SBEnd : "^\\]",
TokenType.expr_New : "^\\bnew\\b",
TokenType.expr_Super : "^\\bsuper\\b",
TokenType.expr_Extends : "^\\bextends\\b",
#Operators
# Logical
TokenType.op_Relational : "^[><]=?",
TokenType.op_Equality : "^[=!]=",
TokenType.op_LAnd : "^&&",
TokenType.op_LOr : "^\\|\\|",
TokenType.op_LNot : "^!",
# Arithmetic
TokenType.op_CAssign : "^[\\*\\/+\\-]=",
TokenType.op_Assign : "^=",
TokenType.op_Additive : "^[+\\-]",
TokenType.op_Multiplicative : "^[\\*\\/]",
# Literals
TokenType.literal_BTrue : "^\\btrue\\b",
TokenType.literal_BFalse : "^\\bfalse\\b",
TokenType.literal_Number : "^\\d+",
TokenType.literal_String : "^\"[^\"]*\"",
TokenType.literal_Null : "^\\bnull\\b",
# Symbols
TokenType.sym_This : "^\\bthis\\b",
TokenType.sym_Identifier : "^\\w+"
}
const SSpec : Dictionary = \
{
# Comments
TokenType.cmt_SL : "start // inline.repeat(0-)",
TokenType.cmt_ML : "start /* set(whitespace !whitespace).repeat(0-).lazy */",
# Formatting
TokenType.fmt_S : "start whitespace.repeat(1-)",
# Delimiters
TokenType.delim_Comma : "start ,",
TokenType.delim_SMR : "start \\.",
# Statements
TokenType.def_End : "start ;",
TokenType.def_BStart : "start {",
TokenType.def_BEnd : "start }",
TokenType.def_Var : "start \"let\"",
TokenType.def_Class : "start \"class\"",
# Iteration
TokenType.def_While : "start \"while\"",
TokenType.def_Do : "start \"do\"",
TokenType.def_For : "start \"for\"",
# Procedures
TokenType.def_Proc : "start \"def\"",
TokenType.def_Return : "start \"return\"",
# Conditional
TokenType.def_If : "start \"if\"",
TokenType.def_Else : "start \"else\"",
# Expressions
TokenType.expr_PStart : "start \\(",
TokenType.expr_PEnd : "start \\)",
TokenType.expr_SBStart : "start [",
TokenType.expr_SBEnd : "start ]",
TokenType.expr_New : "start \"new\"",
TokenType.expr_Super : "start \"super\"",
TokenType.expr_Extends : "start \"extends\"",
#Operators
# Logical
TokenType.op_Relational : "start set(> <) =.repeat(0-1)",
TokenType.op_Equality : "start set(= \\!) =",
TokenType.op_LAnd : "start &&",
TokenType.op_LOr : "start \\| \\|",
TokenType.op_LNot : "start \\!",
# Arithmetic
TokenType.op_CAssign : "start set(* / + \\-) =",
TokenType.op_Assign : "start =",
TokenType.op_Additive : "start set(+ \\-)",
TokenType.op_Multiplicative : "start set(* /)",
# Literals
TokenType.literal_BTrue : "start \"true\"",
TokenType.literal_BFalse : "start \"false\"",
TokenType.literal_Number : "start digit.repeat(1-)",
TokenType.literal_String : "start \\\" !set( \\\" ).repeat(0-) \\\"",
TokenType.literal_Null : "start \"null\"",
# Symbols
TokenType.sym_This : "start \"this\"",
TokenType.sym_Identifier : "start word.repeat(1-)"
}
class Token:
var Type : String
var Value : String
var SourceText : String
var Cursor : int
var SpecRegex : Dictionary
var Tokens : Array
var TokenIndex : int = 0
func compile_regex():
for type in TokenType.values() :
var \
regex = RegEx.new()
var original = Spec[type]
var transpiled = SRegEx.compile(SSpec[type])
assert(transpiled == original, "transpiled did not match original")
regex.compile( transpiled )
SpecRegex[type] = regex
func init(programSrcText):
SourceText = programSrcText
Cursor = 0
TokenIndex = 0
if SpecRegex.size() == 0 :
compile_regex()
tokenize()
func next_Token():
var nextToken = null
if Tokens.size() > TokenIndex :
nextToken = Tokens[TokenIndex]
TokenIndex += 1
return nextToken
func reached_EndOfText():
return Cursor >= SourceText.length()
func tokenize():
Tokens.clear()
while reached_EndOfText() == false :
var srcLeft = SourceText.substr(Cursor)
var token = Token.new()
var error = true
for type in TokenType.values() :
var result = SpecRegex[type].search( srcLeft )
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TokenType.cmt_SL || type == TokenType.cmt_ML :
Cursor += result.get_string().length()
error = false
break
# Skip Whitespace
if type == TokenType.fmt_S :
var addVal = result.get_string().length()
Cursor += addVal
error = false
break
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
Tokens.append( token )
error = false
break;
if error :
var assertStrTmplt = "next_token: Source text not understood by tokenizer at Cursor pos: {value} -: {txt}"
var assertStr = assertStrTmplt.format({"value" : Cursor, "txt" : srcLeft})
assert(true != true, assertStr)
return

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,56 @@
extends Panel
var Lexer = preload("RDP_Lexer.gd").new()
var Parser = preload("RDP_Parser.gd").new()
onready var Tokens_TOut = get_node("Tokens_TOut")
onready var AST_TOut = get_node("AST_TOut")
onready var FDialog = get_node("Letter_FDialog")
onready var FD_Btn = get_node("VBox/ParseLetterFile_Btn")
onready var Back_Btn = get_node("VBox/Back_Btn")
func tokens_out(text):
Tokens_TOut.insert_text_at_cursor(text)
func ast_out(text):
AST_TOut.insert_text_at_cursor(text)
func parse_file(path):
var \
file = File.new()
file.open(path, File.READ)
var programDescription = file.get_as_text()
file.close()
Lexer.init(programDescription)
for token in Lexer.Tokens :
var string = "[" + token.Type + "] " + token.Value + "\n"
tokens_out( string )
var ast = Parser.parse(Lexer)
var json = JSON.print(ast.to_Dictionary(), '\t')
ast_out(json + "\n")
ast_out("Finished Parsing!\n")
func fd_btn_pressed():
FDialog.popup()
func fdialog_FSelected(path):
Tokens_TOut.text = ""
AST_TOut.text = ""
parse_file(path)
func backBtn_pressed():
queue_free()
# Called when the node enters the scene tree for the first time.
func _ready():
FDialog.connect("file_selected", self, "fdialog_FSelected")
FD_Btn.connect("pressed", self, "fd_btn_pressed")
Back_Btn.connect("pressed", self, "backBtn_pressed")

View File

@ -0,0 +1,35 @@
# Automata Theory: Building a RegExp machine
## Content:
State Machines
Formal Grammars
Implement a regular expression processor
## History:
*Pioneers:*
1951 - Stephen Kleene invented reg exp (sets).
Reuglar Langauge : Langauge recognized by a finite automata (state machines).
Kleene's Therem : Equivalence of regular expressions and finite automata.
Has a notation named after him:
Kleene-Closure (AKA: Kleene star) : A* (Stands for repetition)
1956 - Chomsky defines his hiearchy fo grammers
Regular grammers are considered a type 3.
See: https://en.wikipedia.org/wiki/Chomsky_hierarchy
![img](https://i.imgur.com/Pj2aFeg.png)
Thus they are the weakest form of grammars.
1968 - Ken Thompson used them for pattern matching in strings, and
lexical analysis (scanners)
NFA - Thompson construction

View File

@ -0,0 +1,74 @@
# Symbols, alphabets, and langauges and Regular Grammars
Alphabet : A set of characters.
Sigma = { a, b }
Langauge : A set of strings over a particular alphabet.
L1(Sigma) = { a, aa, b, ab, ba, bba, .. } (Infinite)
L2(Sigma) = { aa, bb, ab, ba }; (Length = 2, Finite)
Any time you constrain a langauge you are
defining a formal grammar.
## Formal Grammars:
FormalGrammer = (Non-Terminals, Terminals, Productions, Starting Symbol)
Non-Terminals : Variables (can be subsituted with a value)
Terminals : Cannot be replaced by anything (constant)
Productions : Rule in the grammar
**G = (N, T, P, S)**
Ex:
```
S -> aX
X -> b
```
**(This notation is known as BNF : Bakus-Naur Form)**
Ex.Non-Terminals = S, X
Ex.Terminals = a, b
Ex.Productions = S -> aX, X -> b (2)
Ex.Starting Symbol = S
Only valid string : "ab"
## Chomsky Hierachy :
0. Unrestricted : Natural Langauges, Turing Machines
1. Context-Sensitive : Programming Languages (Almost all in production)
2. Context-Free : Programming Langauges (Parsing Syntax only)
3. Regular : Regular Expressions
The lower in the hiearchy the less expressive it is.
RegExp is a vomit inducing terse notation that is equivalent to BNF.
BNF : RegExp
S -> aS :
S -> bA : `a*bc*`
A -> epsilon :
A -> cA :
epsilon : "The empty string".
Regular expressions may only have one non-terminal:
* A the very right side (right-linear, RHS)
* At the very left side (left-linear, LHS)
Regular expression have no support for *NESTING*
They can be *RECURSIVE*
Context-free grammers support nesting.
Ex:
(( () ))
`Parenthesis balacing`
Non-regular RegExp can support nesting but are not pure
finite automata and are slower implementation.

View File

@ -0,0 +1,85 @@
# Finite Automata
***(AKA: Finite State Machine)***
Mechanism and abstraction used behind regular grammars.
Usually has its state represented using nodes and edges.
Regular grammar:
```
S -> bA
A -> epsilon
```
Equivalent to: `\b\`
State transition:
--label--> : Transition symbol
O : State Symbol
(o) : Accepting State
->O.Start : Starting State (State transition to Start)
Ex:
->O.*Start* --*transition*--> (o).*Accepting*
*ε* - Epsilon (Empty String)
`I will be spelling it out as I do not enjoy single glyth representation`
Two main types of Finite Automtata :
FA w/ output
* Moore machine
* Mealy machine
FA w/o output
* DFA - Deterministic
* NFA - Non-deterministic
* epsilon-NFA - (Epsilon Transition) special case
NFA : Non-deterministic FA - Allos transition on the same symbol to
different states
```
a->o
/
->o.1---b-->o
\
a->o
```
epsilon-NFA : Extension of NFA that allows *epsilon* transitions
```
a--->o---epsi--->(o)
/ /
->o----b-->epsi--->o
\
a-->o--epsi-->(o)
```
DFA : A state machine which forbids multiple transitions on the same symbol, and *epsilon* transitions
```
a--->o
/
->o----b-->o
```
Use case:
Implementation Transformations:
```RegExp -> epsilon-NFA -> ... -> DFA```
## Formal Definition:
Non-deterministic finite automata is a tuple of five elements:
* All possible states
* Alphabet
* Transition Function
* Starting State
* Set of accepting states
NFA = ( States, Alphabet, TransitionFunction, StartingState, AcceptingStates )
NFA = ( Q, Σ, Δ, q0, F )

View File

@ -0,0 +1,27 @@
# Basic NFA Fragments
### Single Character
RegExp: `/^A$/`
Psuedo: `start A end`
^ : Beginning of string : Str.Start
$ : End of a string : Str.End
Machine:
->o.*Start* ---**Glyph**---> (o).*Accepting*
### Epsilon-Transition
RegExp: `/^$/`
Psuedo: `start end`
Machine:
```
->o --epsilon--> (o)
```
Everyhing else can be built on top of these machines.
```
Start = Input, Accepting = Output
```

View File

@ -0,0 +1,39 @@
## Concatenation
Regex : `/^AB$/`
Psuedo: `start AB end`
Machine:
```
->o --A--> o --epsilon--> o --B--> (o)
Submachine_A --epsilon--> Submachine_B
```
## Union
Regex : `/^A|B$/`
Psuedo: `start A | B end`
Machine:
```
epsilon--> o --A--> o --epsilon
/ \
->o ->(o)
\ /
epsilon--> o --B--> o --epsilon
```
## Kleene Closure
Regex : `/^A*$/`
Psuedo: `start A.repeat(0-) end`
Machine:
```
<------episilon-------
/ \
->o --epsilon--> o --A--> o --epsilon--> (o)
\ /
-------------epsilon---------------->
```

View File

@ -0,0 +1,30 @@
# Complex Machines
Ex:
RegEx : `/xy*|z`
SRegEx: `x y.repeat(0-) | z`
## Decomposition
### Stage 1: Union
```
->o.start (o)
\epsilon-> o --xy*-> o -epsilon-->/
\epsilon-> o --z---> o -epsilon->/
```
### Stage 2: Concatenation
```
->o.start (o)
\epsilon -> o --x--> o -epsilon-> o --y* -epsilon->/
\epsilon -> o --z--> o -epsilon------------------>/
```
### Stage 2: Kleene Closure
```
|<------------<|
->epsi -> o -x-> o -epsi-> o -epsi-> o -y-> -epsi-> o ->epsi->|
| |>---------------------->| /
->o.start (o)
\epsi -> o -z-> o -epsi------------------------------------>/
```

View File

@ -0,0 +1,39 @@
# Syntactic Sugar
Ex:
RegEx : `/a+|[0-3]/`
SRegEx: `a.repeat(1-) | set(0-3)`
`A+` === `AA*` === `A.repeat(1-)` === `AA.repeat(0-)`
`A?` === `A|ε` === `A.repeat(0-1)`
`[0-9]` === `0|1|2|3|4|5|6|7|8|9` === `set(0-9)`
# NFA Optimizations
Ex:
RegEx : `/[0-2]+/`
SRegEx: `set(0-2).repeat(1-)`
Machine (Optimized):
```
|<-epsi-<|
->o -0-> (o)
\--1-->/
\-2->/
```
A* (Optimized)
```
->o -A--> (o)
\-epsi->/
```
`[characters]`
```
->o --<num>--> (o)
..........
```

17
App/RegM/Readme.md Normal file
View File

@ -0,0 +1,17 @@
# Automata: RegEx Machines
![Img](https://i.imgur.com/NWwBjhN.png)
## Lectures
Contains notes taken from the lecture vods.
## Scripts
Contains some implementation from the lectures but I have no completed it yet...
I made my own context-free version of the language callled "SRegEx"
This is mainly what I wanted at the end of the day...
Currently the script only translates to RegEx,
I still need to make RegEx to SRegEx

99
App/RegM/RegM_Viewer.tscn Normal file
View File

@ -0,0 +1,99 @@
[gd_scene load_steps=4 format=2]
[ext_resource path="res://Assets/Branding/RegM_Class_cover_small.png" type="Texture" id=1]
[ext_resource path="res://Assets/Styles/Purple.EditorTheme.tres" type="Theme" id=2]
[ext_resource path="res://RegM/Scripts/RegM_Viewer.gd" type="Script" id=3]
[node name="RegM_Panel" type="Panel"]
anchor_right = 1.0
anchor_bottom = 1.0
theme = ExtResource( 2 )
script = ExtResource( 3 )
[node name="CourseBrand" type="TextureRect" parent="."]
anchor_right = 1.0
anchor_bottom = 1.0
rect_scale = Vector2( 0.2, 0.2 )
texture = ExtResource( 1 )
expand = true
stretch_mode = 6
[node name="RegEx_TEdit" type="TextEdit" parent="."]
anchor_left = 0.2
anchor_right = 0.526
anchor_bottom = 1.0
margin_left = 1.2
margin_right = 0.375977
grow_horizontal = 0
theme = ExtResource( 2 )
readonly = true
highlight_current_line = true
show_line_numbers = true
wrap_enabled = true
[node name="SRegEx_TEdit" type="TextEdit" parent="."]
anchor_left = 0.525
anchor_right = 1.0
anchor_bottom = 1.0
margin_left = 1.83997
grow_horizontal = 0
theme = ExtResource( 2 )
highlight_current_line = true
show_line_numbers = true
minimap_draw = true
[node name="Letter_FDialog" type="FileDialog" parent="."]
anchor_left = 0.35
anchor_top = 0.15
anchor_right = 0.45
anchor_bottom = 0.25
margin_right = 356.0
margin_bottom = 373.0
theme = ExtResource( 2 )
popup_exclusive = true
window_title = "Open a File"
mode = 0
access = 2
[node name="VBox" type="VBoxContainer" parent="."]
anchor_top = 0.196
anchor_right = 0.2
anchor_bottom = 1.0
margin_top = 2.39999
margin_right = -1.8
margin_bottom = -2.0
[node name="ToRegEx_Btn" type="Button" parent="VBox"]
margin_right = 203.0
margin_bottom = 30.0
focus_neighbour_top = NodePath("../../CourseBrand")
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
theme = ExtResource( 2 )
text = "To RegEx"
[node name="ToSRegEx_Btn" type="Button" parent="VBox"]
margin_top = 34.0
margin_right = 203.0
margin_bottom = 64.0
focus_neighbour_top = NodePath("../../CourseBrand")
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
theme = ExtResource( 2 )
disabled = true
text = "To SRegEx"
[node name="Separator" type="HSeparator" parent="VBox"]
modulate = Color( 0.145098, 0.145098, 0.164706, 0 )
margin_top = 68.0
margin_right = 203.0
margin_bottom = 443.0
size_flags_vertical = 15
[node name="Back_Btn" type="Button" parent="VBox"]
margin_top = 447.0
margin_right = 203.0
margin_bottom = 478.0
size_flags_vertical = 3
size_flags_stretch_ratio = 0.08
text = "Course Directory"

View File

@ -0,0 +1,105 @@
extends Object
const epsilon = 'ε'
class State :
var accepting : bool = false
var transitionMap : Dictionary
func _init(accepting : bool):
self.accepting = accepting
transitionMap[epsilon] = []
func add_Transition(symbol : string, state : State):
if symbol == epsilon :
transitionMap[symbol].append(state)
return
transitionMap[symbol] = state
func get_Transition(symbol : string) :
return transitionMap[symbol]
class NFA :
var input : State
var output : State
func _init(input : State, output : State):
self.input = input
self.output = output
func test(string : String) :
return input.test(string)
func concat(first : NFA, rest : Array):
for entry in rest :
first = concat_pair(first, entry)
return first
func concat_pair(first : NFA, second : NFA):
first.output.accepting = false
second.output.accepting = true
first.output.add_Transition(epsilon, second.input)
return NFA.new(first.input, second.output)
# Epsilon-Transition machine
func empty():
return glyph(epsilon)
# Single character machine.
func glyph(symbol : string):
var start = State.new(false)
var accepting = State.new(true)
start.add_Transition(symbol, accepting)
return NFA.new(start, accepting)
func repeat(entry : NFA):
var start = State.new(false)
var accepting = State.new(true)
start.add_Transition(epsilon, entry.input)
entry.output.accepting(false)
entry.output.add_Transition(epsilon, entry.input) # Repeater transition
entry.output.add_Transition(epsilon, accepting)
return NFA.new(start, accepting)
func union(first : NFA, rest : Array):
for entry in rest :
first = union_pair(first, entry)
return first
func union_pair(a : NFA, b : NFA):
var start = State.new(false)
var accepting = State.new(true)
start.add_Transition(epsilon, a.input)
start.add_Transition(epsilon, b.output)
a.output.accepting = false
b.output.accepting = false
a.output.add_Transition(epsilon, accepting)
b.output.add_Transition(epsilon, accepting)
return NFA.new(start, accepting)
func test():
var state_1 = State.new(false)
var state_2 = State.new(true)
state_1.add_Transition('A', state_2)
print("State 1 Transition for " + "A: " + state_1.get_Transition('A'))

View File

@ -0,0 +1,22 @@
extends Node
var SRegEx = preload("SRegEx.gd").new()
onready var RegEx_TEdit = get_node("RegEx_TEdit")
onready var SRegEx_TEdit = get_node("SRegEx_TEdit")
onready var ToRegEx_Btn = get_node("VBox/ToRegEx_Btn")
onready var Back_Btn = get_node("VBox/Back_Btn")
func to_RegExBtn_pressed():
RegEx_TEdit.text = SRegEx.transpile(SRegEx_TEdit.text)
# for line in SRegEx_TEdit.text.split("\n") :
# RegEx_TEdit.text += SRegEx.transpile( line ) + "\n"
func backBtn_pressed():
queue_free()
# Called when the node enters the scene tree for the first time.
func _ready():
Back_Btn.connect("pressed", self, "backBtn_pressed")
ToRegEx_Btn.connect("pressed", self, "to_RegExBtn_pressed")

View File

@ -0,0 +1,251 @@
SRegex Notes
Test Cases:
```
RegEx SRegex
. inline
\w word
\d digit
\s whitespace
\W !word
\D !digit
\S !whitespace
[abc] set(abc)
[^abc] !set(abc)
[a-g] set(a-g)
^abc$ start abc end
\bstring\b "string"
\Bnot this string\B !"not this string"
\- (NOTE: Used by between token)
\. (NOTE: Used by .lazy, .repeat)
\! (NOTE: Used as not operator)
\| (NOTE: Used as union operator)
\( (NOTE: Used for captures, set, ref)
\) (NOTE: Used for captures, set, ref)
\" (NOTE: Used for strings)
\t
\n
\r
(abc) ( abc )
\1 backref(1)
(?:abc) !( abc )
(?=abc) look(abc)
(?!abc) !look(abc)
a* a.repeat(0-)
a+ a.repeat(1-)
a? a.repeat(0-1)
a{5} a.repeat(5)
a{2,} a.repeat(2-)
a{1,3} a.repeat(1-3)
a{5} a.repeat(0-).lazy
a{2,}? a.repeat(2-).lazy
ab|cd ab | cd
/^\/\*[\s\S]*?\*\// start /* set(whitespace !whitespace).repeat(0-).lazy */
```
```
inline
word
digit
whitespace
!word
!digit
!whitespace
set(abc)
!set(abc)
set(a-g)
start abc end
"string"
!"not this string"
\-
\.
\!
\|
\(
\)
\"
\t
\n
\r
( abc )
backref(1)
!( abc )
look(abc)
!look(abc)
a.repeat(0-)
a.repeat(1-)
a.repeat(0-1)
a.repeat(5)
a.repeat(2-)
a.repeat(1-3)
a.repeat(0-).lazy
a.repeat(2-).lazy
ab | cd
start whitespace
start "start"
start "end"
start \" !set( \" ).repeat(0-) "\
start \ \(
start \ \)
start \(
start \)
start \-
start "digt"
start "inline"
start "word"
start "whitespace"
start "lazy"
start \."repeat"
start \\ \-
start \\ \.
start \\ \!
start \\ \|
start \\ \"
start "look"
start \!
start \|
start "backref"
start "set"
start !set(whitespace)
start // inline.repeat(0-)
start /* set(whitespace !whitespace).repeat(0-).lazy */ start
start whitespace.repeat(1-)
start ,
start \.
start ;
start {
start }
start "let"
start "class"
start "while"
start "do"
start "for"
start "def"
start "return"
start "if"
start "else"
start "new"
start "extends"
start "super"
start set(> <) =.repeat(0-1)
start set(= \!) =
start &&
start \| \|
start \!
start set( * / + \- ) =
start =
start set(+ \-)
start set( * / )
start "true"
start "false"
start digit.repeat(1-)
start \" !set( \" ).repeat(0-) \"
start "null"
start "this"
start word.repeat(1-)
(?# Url checker with or without http:// or https:// )
start(
http://www\.
| https://www\.
| http://
| https://
).repeat(0-1)
set(a-z 0-9).repeat(1-)
( (?# Check for any hypens or dot namespaces )
set(\- \. ).repeat(1)
set(a-z 0-9).repeat(1-)
)
.repeat(0-)
(?# Domain name )
\. set(a-z).repeat(2,5)
(?# Possibly for a port? )
( : set(0-9).repeat(1-5) ).repeat(0-1)
(?# I have no idea... )
( / \. \*).repeat(0-1)
end
(?# Validate an IP Address)
start(
( set(0-9)
| set(1-9) set(0-9)
| 1 set(0-9).repeat(2)
| 2 set(0-4) set(0-9)
| 25 set(0-5)
)
\.
)
.repeat(3)
(
set(0-9)
| set(1-9)set(0-9)
| 1 set(0-9).repeat(2)
| 2 set(0-4) set(0-9)
| 25 set(0-5)
)
end
(?# Match dates (M/D/YY, M/D/YYY, MM/DD/YY, MM/DD/YYYY) )
start
(
(?# Handle Jan, Mar, May, Jul, Aug, Oct, Dec )
( 0.repeat(0-1) set(1 3 5 7 8) | 10 | 12 )
( \- | / )
( (?# Handle Day )
( set(1-9))
| ( 0 set(1-9))
| ( set(1 2))
( set(0-9).repeat(0-1))
| ( 3 set(0 1).repeat(0-1))
)
( \- | / )
( (?# Handle Year)
(19)
( set(2-9))
( digit.repeat(1) )
| (20)
( set(0 1))
( digit.repeat(1) )
| ( set(8 9 0 1))
( digit.repeat(1))
)
| (?# Handle Feb, Apr, June, Sept )
( 0.repeat(2 4 6 9) | 11 )
( \- | /)
( (?# Handle Day )
( set(1-9))
| ( 0 set(1-9))
| ( set(1 2))
( set(0-9).repeat(0-1))
| ( 3 set(0 ).repeat(0-1))
)
( \- | / )
(
(?# Handle Year)
(19)
( set(2-9) )
( digit.repeat(1) )
| (20)
( set(0 1))
( digit.repeat(1) )
| ( set(8 9 0 1 ))
( digit.repeat(1))
)
)
end

1074
App/RegM/Scripts/SRegex.gd Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,74 @@
[gd_scene load_steps=6 format=2]
[ext_resource path="res://Scripts/CourseDirectory.gd" type="Script" id=1]
[ext_resource path="res://Assets/Branding/RDP_Class_cover_small.png" type="Texture" id=2]
[ext_resource path="res://Assets/Branding/RegM_Class_cover_small.png" type="Texture" id=3]
[ext_resource path="res://Assets/Styles/Purple.EditorTheme.tres" type="Theme" id=4]
[ext_resource path="res://Assets/Branding/EoI_Class_Cover.png" type="Texture" id=5]
[node name="Panel" type="Panel"]
anchor_right = 1.0
anchor_bottom = 1.0
script = ExtResource( 1 )
[node name="HBox" type="HBoxContainer" parent="."]
anchor_left = 0.5
anchor_top = 0.5
anchor_right = 0.5
anchor_bottom = 0.5
margin_left = -452.0
margin_top = -80.5
margin_right = 452.0
margin_bottom = 80.5
size_flags_horizontal = 3
size_flags_vertical = 3
alignment = 1
[node name="RDP_Btn" type="Button" parent="HBox"]
margin_right = 276.0
margin_bottom = 161.0
rect_min_size = Vector2( 255, 150 )
size_flags_horizontal = 3
size_flags_vertical = 7
theme = ExtResource( 4 )
icon = ExtResource( 2 )
icon_align = 1
expand_icon = true
[node name="VSeparator" type="VSeparator" parent="HBox"]
margin_left = 280.0
margin_right = 310.0
margin_bottom = 161.0
theme = ExtResource( 4 )
custom_constants/separation = 30
[node name="RegM_Btn" type="Button" parent="HBox"]
margin_left = 314.0
margin_right = 590.0
margin_bottom = 161.0
rect_min_size = Vector2( 255, 150 )
size_flags_horizontal = 3
size_flags_vertical = 3
theme = ExtResource( 4 )
icon = ExtResource( 3 )
icon_align = 1
expand_icon = true
[node name="VSeparator2" type="VSeparator" parent="HBox"]
margin_left = 594.0
margin_right = 624.0
margin_bottom = 161.0
theme = ExtResource( 4 )
custom_constants/separation = 30
[node name="EoI_Btn" type="Button" parent="HBox"]
margin_left = 628.0
margin_right = 904.0
margin_bottom = 161.0
rect_min_size = Vector2( 255, 150 )
size_flags_horizontal = 3
size_flags_vertical = 7
theme = ExtResource( 4 )
icon = ExtResource( 5 )
icon_align = 1
expand_icon = true

View File

@ -0,0 +1,28 @@
extends Panel
onready var RDP_Viewer = load("res://RDP/RDP_Viewer.tscn")
onready var RegM_Viewer = load("res://RegM/RegM_Viewer.tscn")
onready var EoI_Viewer = load("res://EoI/EoI_Viewer.tscn")
onready var RDP_Btn = get_node("HBox/RDP_Btn")
onready var RegM_Btn = get_node("HBox/RegM_Btn")
onready var EoI_Btn = get_node("HBox/EoI_Btn")
func rdp_pressed():
add_child( RDP_Viewer.instance() )
func regM_pressed():
add_child( RegM_Viewer.instance() )
func eoi_pressed():
add_child( EoI_Viewer.instance() )
# Called when the node enters the scene tree for the first time.
func _ready():
EoI_Btn.connect("pressed", self, "eoi_pressed")
RDP_Btn.connect("pressed", self, "rdp_pressed")
RegM_Btn.connect("pressed", self, "regM_pressed")

View File

View File

View File

View File

View File

Before

Width:  |  Height:  |  Size: 3.2 KiB

After

Width:  |  Height:  |  Size: 3.2 KiB

76
App/project.godot Normal file
View File

@ -0,0 +1,76 @@
; Engine configuration file.
; It's best edited using the editor UI and not directly,
; since the parameters that go here are not all obvious.
;
; Format:
; [section] ; section goes between []
; param=value ; assign values to parameters
config_version=4
_global_script_classes=[ {
"base": "Object",
"class": "Eva",
"language": "GDScript",
"path": "res://EoI/Scripts/Eva.gd"
}, {
"base": "Object",
"class": "EvaEnv",
"language": "GDScript",
"path": "res://EoI/Scripts/EvaEnv.gd"
}, {
"base": "Object",
"class": "Lexer",
"language": "GDScript",
"path": "res://EoI/Scripts/Lexer.gd"
}, {
"base": "Object",
"class": "Parser",
"language": "GDScript",
"path": "res://EoI/Scripts/Parser.gd"
}, {
"base": "Object",
"class": "RDP_Lexer",
"language": "GDScript",
"path": "res://RDP/Scripts/RDP_Lexer.gd"
}, {
"base": "Object",
"class": "RDP_Parser",
"language": "GDScript",
"path": "res://RDP/Scripts/RDP_Parser.gd"
} ]
_global_script_class_icons={
"Eva": "",
"EvaEnv": "",
"Lexer": "",
"Parser": "",
"RDP_Lexer": "",
"RDP_Parser": ""
}
[application]
config/name="LangStudies"
run/main_scene="res://Scenes/Persistent.tscn"
config/icon="res://Assets/Branding/RDP_Class_cover_small.png"
[autoload]
GScene="*res://Scenes/CourseDirectory.tscn"
[gui]
common/drop_mouse_on_gui_input_disabled=true
[physics]
common/enable_pause_aware_picking=true
[rendering]
quality/intended_usage/framebuffer_allocation=0
quality/intended_usage/framebuffer_allocation.mobile=0
quality/shadows/filter_mode=2
quality/shadows/filter_mode.mobile=2
quality/filters/msaa=1
environment/default_environment="res://default_env.tres"

View File

@ -1,14 +1,46 @@
where "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat" >nul 2>nul
if not ERRORLEVEL 0 (
echo Visual Studio 2019 not found... Remove this error message if you do have it.
pause
exit
)
where python >nul 2>nul
if not ERRORLEVEL 0 (
echo Need python not found... Remove this error message if you have it.
pause
exit
)
where scons >nul 2>nul
if not ERRORLEVEL 0 (
python pip install scons
)
git clone --recurse-submodules https://github.com/Ed94/LangStudies git clone --recurse-submodules https://github.com/Ed94/LangStudies
cd LangStudies cd LangStudies
start build_engine.debug.bat start build_engine.bat
start Engine\gd\bin\godot.windows.tools.64.exe -e Editor/project.godot :tools_wait
timeout 20 timeout 1
taskkill /f /im godot.windows.tools.64.exe if not exist Engine\gd\bin\godot.windows.opt.tools.64.exe (
goto :tools_wait
)
timeout 10
start Engine\gd\bin\godot.windows.opt.tools.64.exe -e App/project.godot
timeout 30
start /w build_engine.release.bat taskkill /f /im godot.windows.opt.tools.64.exe
:opt_wait
timeout 1
if not exist Engine\gd\bin\godot.windows.opt.64.exe (
goto :opt_wait
)
timeout 2
start /w build_project.bat start /w build_project.bat

View File

@ -0,0 +1,3 @@
// Unary
!x;
-x;

View File

@ -0,0 +1,32 @@
// While Statement
while ( x > 10 )
{
x -= 1;
}
// Do-While Statement
do
{
y -= 1;
}
while ( y > 10 );
// For Statement
for (let index = 0; index < 10; index += 1)
{
x += index;
}
// For Statement (Optionals)
for (;;)
{}
for (let index = 0, z = 12; index < 10; index += 1)
{
x += index;
}
for (index = 0; index < 10; index += 1)
{
x += index;
}

View File

@ -0,0 +1,15 @@
def square(value)
{
return value * value;
}
def empty() { return; }
def empty_2() { }
def multi_variable(a, b)
{
return a * b;
}
square(2);

View File

@ -0,0 +1,19 @@
Symbol.Member;
Symbol.Member = 1;
Symbol.Member[0] = 1;
Symbol.Member.AnotherMember["Something"];
let str = "Hello World!";
let index = 0;
while (index < str.Length)
{
str[index];
console.log(index, str[index]);
index += 1;
}

View File

@ -0,0 +1,32 @@
class Point
{
def constructor(x, y)
{
this.x = x;
this.y = y;
}
def sum()
{
return this.x + this.y;
}
}
class Point3D extends Point
{
def constructor(x, y, z)
{
super(x, y);
this.z = z;
}
def sum()
{
return super() + this.z;
}
}
let
point = new Point(10, 20, 30);
point.sum();

View File

@ -0,0 +1,18 @@
// Binary expression
2 + 2;
// Nested binary expressions:
// left : 3 + 2
// right : 2
3 + 2 - 2;
// Nested binary expressions:
3 * 3;
3 * 4 * 5;
3 + 6 * 2;
(2 + 2) * 3;
(2 + 2) * 3 + 10;

View File

@ -1,7 +0,0 @@
// Binary expression
2 + 2;
// Nested binary expressions:
// left : 3 + 2
// right : 2
3 + 2 - 2;

View File

@ -0,0 +1,11 @@
// Single
x = 42;
// Chained
x = (y = 42);
x + y;
x += y;
y += 5;

View File

@ -1,5 +0,0 @@
// Single
x = 42;
// Chained
x = (y = 42);

View File

@ -1,8 +0,0 @@
if (x)
{
x = 1;
}
else
{
x = 2;
}

View File

@ -5,4 +5,10 @@ let x;
let y = 42; let y = 42;
// Multiple declarations // Multiple declarations
let a, b; let a, b;
let c, d = 10;
let foo = bar = 10;
r = 10;

View File

@ -0,0 +1,28 @@
if (x)
{
x = 1;
}
else
{
x = 2;
}
if ( y )
{
x = 2;
}
if (a)
b = 1;
if (a)
if (b)
if (c)
if (d)
{}
else
{}
else
{}

View File

@ -1 +0,0 @@
x > 0;

View File

@ -0,0 +1,6 @@
x > 0;
x + 2 > 5;
y >= 4;
z <= 7;

View File

@ -0,0 +1,7 @@
x > 0 == true;
x > 0 != false;
x + 5 > 10 == false;
x == null;

Some files were not shown because too many files have changed in this diff Show More