BAPFS Lecture 12

I reorganized the functions by their precedence.
This commit is contained in:
Edward R. Gonzalez 2022-07-12 01:25:47 -04:00
parent d34ccb04d2
commit 7544c4bedc
5 changed files with 927 additions and 2 deletions

View File

@ -1,3 +1,8 @@
where "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat" >nul 2>nul
if not ERRORLEVEL 0 (
echo Visual Studio 2019 not found... Remove this error message if you do have it.
@ -13,6 +18,7 @@ if not ERRORLEVEL 0 (
git clone --recurse-submodules https://github.com/Ed94/LangStudies
cd LangStudies
pause
start build_engine.release.bat
timeout 10

3
Builds/Tests/10.Unary.uf Normal file
View File

@ -0,0 +1,3 @@
// Unary
!x;
-x;

View File

@ -817,7 +817,7 @@ func test(entry):
GTokenizer.init(programDescription)
var ast = GParser.parse(GTokenizer)
var json = JSON.print(ast.to_SExpression(), '\t')
var json = JSON.print(ast.to_Dictionary(), '\t')
tout(json + "\n")
tout("Passed!\n")

View File

@ -0,0 +1,916 @@
extends Node
# This closesly follows the source provided in the lectures.
# Later on after the lectures are complete or when I deem
# Necessary there will be heavy refactors.
const TokenType = \
{
Program = "Program",
# Comments
CommentLine = "CommentLine",
CommentMultiLine = "CommentMultiLine",
# Formatting
Whitespace = "Whitespace",
# Expressions
ExpressionPStart = "ExpresssionParenthesisStart",
ExpressionPEnd = "ExpressionParenthesisEnd",
# Logical
RelationalOp = "RelationalOperator",
EqualityOp = "EqualityOperator",
Logical_And = "Logical_And_Op",
Logical_Or = "Logical_Or_Op",
Logical_Not = "Logical_Not_Op",
# Arithmetic
ComplexAssignment = "ComplexAssignment",
Assignment = "Assignment",
AdditiveOp = "AdditiveOperator",
MultiplicativeOp = "MultiplicativeOperator",
# Conditional
Conditional_if = "if Conditional",
Conditional_else = "else Conditional",
# Statements
StatementEnd = "StatementEnd",
StmtBlockStart = "BlockStatementStart",
StmtBlockEnd = "BlockStatementEnd",
CommaDelimiter = "CommaDelimiter",
# Literals
Number = "Number",
String = "String",
# Symbols
Bool_true = "Boolean True",
Bool_false = "Boolean False",
VarDeclare = "Variable Declaration",
Identifier = "Identifier",
NullValue = "Null Value"
}
const TokenSpec = \
{
# Comments
TokenType.CommentLine : "^\/\/.*",
TokenType.CommentMultiLine : "^\/\\*[\\s\\S]*?\\*\/",
# Formatting
TokenType.Whitespace : "^\\s+",
# Expressions
TokenType.ExpressionPStart : "^\\(",
TokenType.ExpressionPEnd : "^\\)",
# Logical
TokenType.Logical_Not : "^!",
TokenType.RelationalOp : "^[>\\<]=?",
TokenType.EqualityOp : "^[=!]=",
TokenType.Logical_And : "^&&",
TokenType.Logical_Or : "^\\|\\|",
# Arithmetic
TokenType.ComplexAssignment : "^[*\\/\\+\\-]=",
TokenType.Assignment : "^=",
TokenType.AdditiveOp : "^[+\\-]",
TokenType.MultiplicativeOp : "^[*\\/]",
# Literal
TokenType.Number : "\\d+",
TokenType.String : "^\"[^\"]*\"",
TokenType.Conditional_if : "^\\bif\\b",
TokenType.Conditional_else : "^\\belse\\b",
# Statements
TokenType.StatementEnd : "^;",
TokenType.StmtBlockStart : "^{",
TokenType.StmtBlockEnd : "^}",
TokenType.CommaDelimiter : "^,",
# Symbols
TokenType.Bool_true : "^\\btrue\\b",
TokenType.Bool_false : "^\\bfalse\\b",
TokenType.VarDeclare : "^\\blet\\b",
TokenType.Identifier : "^\\w+",
TokenType.NullValue : "^\\bnull\\b"
}
class Token:
var Type : String
var Value : String
func to_Dictionary():
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Tokenizer:
var SrcTxt : String
var Cursor : int;
# Sets up the tokenizer with the program source text.
func init(programSrcText):
SrcTxt = programSrcText
Cursor = 0
# Provides the next token in the source text.
func next_Token():
if reached_EndOfTxt() == true :
return null
var srcLeft = SrcTxt.substr(Cursor)
var regex = RegEx.new()
var token = Token.new()
for type in TokenSpec :
regex.compile(TokenSpec[type])
var result = regex.search(srcLeft)
if result == null || result.get_start() != 0 :
continue
# Skip Comments
if type == TokenType.CommentLine || type == TokenType.CommentMultiLine :
Cursor += result.get_string().length()
return next_Token()
# Skip Whitespace
if type == TokenType.Whitespace :
var addVal = result.get_string().length()
Cursor += addVal
return next_Token()
token.Type = type
token.Value = result.get_string()
Cursor += ( result.get_string().length() )
return token
var assertStrTmplt = "next_token: Source text not understood by tokenizer at Cursor pos: {value}"
var assertStr = assertStrTmplt.format({"value" : Cursor})
assert(true != true, assertStr)
return null
func reached_EndOfTxt():
return Cursor >= ( SrcTxt.length() )
var GTokenizer = Tokenizer.new()
const AST_Format = \
{
Dictionary = "Dictionary",
SExpression = "S-Expression"
}
const SyntaxNodeType = \
{
NumericLiteral = "NumericLiteral",
StringLiteral = "StringLiteral",
ExpressionStatement = "ExpressionStatement",
BlockStatement = "BlockStatement",
EmptyStatement = "EmptyStatement",
BinaryExpression = "BinaryExpression",
Identifier = "Identifier",
AssignmentExpression = "AssignmentExpression",
VariableStatement = "VariableStatement",
VariableDeclaration = "VariableDeclaration",
ConditionalStatement = "ConditionalStatement",
BooleanLiteral = "BooleanLiteral",
NullLiteral = "NullLiteral",
LogicalExpression = "LogicalExpression",
UnaryExpression = "UnaryExpression"
}
class SyntaxNode:
var Type : String
var Value # Not specifing a type implicity declares a Variant type.
func to_SExpression():
var expression = [ Type ]
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_SExpression() )
else :
array.append( entry )
expression.append(array)
return expression
if typeof(Value) == TYPE_OBJECT :
var result = [ Type, Value.to_SExpression() ]
return result
expression.append(Value)
return expression
func to_Dictionary():
if typeof(Value) == TYPE_ARRAY :
var array = []
for entry in self.Value :
if typeof(entry) == TYPE_OBJECT :
array.append( entry.to_Dictionary() )
else :
array.append( entry )
var result = \
{
Type = self.Type,
Value = array
}
return result
if typeof(Value) == TYPE_OBJECT :
var result = \
{
Type = self.Type,
Value = self.Value.to_Dictionary()
}
return result
var result = \
{
Type = self.Type,
Value = self.Value
}
return result
class Parser:
var TokenizerRef : Tokenizer
var NextToken : Token
# --------------------------------------------------------------------- HELPERS
# Gets the next token only if the current token is the specified intended token (tokenType)
func eat(tokenType):
var currToken = self.NextToken
assert(currToken != null, "eat: NextToken was null")
var assertStrTmplt = "eat: Unexpected token: {value}, expected: {type}"
var assertStr = assertStrTmplt.format({"value" : currToken.Value, "type" : tokenType})
assert(currToken.Type == tokenType, assertStr)
NextToken = TokenizerRef.next_Token()
return currToken
func is_Literal():
return \
NextToken.Type == TokenType.Number \
|| NextToken.Type == TokenType.String \
|| NextToken.Type == TokenType.Bool_true \
|| NextToken.Type == TokenType.Bool_false \
|| NextToken.Type == TokenType.NullValue
# BinaryExpression
# : MultiplicativeExpression
# | AdditiveExpression
# ;
func parse_BinaryExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken:
var operator = eat(operatorToken)
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.BinaryExpression
nestedNode.Value = []
nestedNode.Value.append(operator.Value)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode;
return left
# LogicalExpression
# : LogicalAndExpression
# | LogicalOrExpression
# ;
func parse_LogicalExpression(parse_fn, operatorToken):
var left = parse_fn.call_func()
while NextToken.Type == operatorToken :
var operator = eat(operatorToken).Value
var right = parse_fn.call_func()
var \
nestedNode = SyntaxNode.new()
nestedNode.Type = SyntaxNodeType.LogicalExpression
nestedNode.Value = []
nestedNode.Value.append(operator)
nestedNode.Value.append(left)
nestedNode.Value.append(right)
left = nestedNode
return left
# ------------------------------------------------------------------ END HELPERS
# Parses the text program description into an AST.
func parse(TokenizerRef):
self.TokenizerRef = TokenizerRef
NextToken = TokenizerRef.next_Token()
return parse_Program()
# > parse
# Program
# : StatementList
# : Literal
# ;
func parse_Program():
var \
node = SyntaxNode.new()
node.Type = TokenType.Program
node.Value = parse_StatementList(null)
return node
# > Program
# > BlockStatement
# StatementList
# : Statement
# | StatementList Statement -> Statement ...
# ;
func parse_StatementList(endToken):
var statementList = [ parse_Statement() ]
while NextToken != null && NextToken.Type != endToken :
statementList.append( parse_Statement() )
return statementList
# > StatementList
# > If_Statement
# >
# Statement
# : ExpressionStatement
# | BlockStatement
# | EmptyStatement
# | VariableStatement
# | If_Statement
# ;
func parse_Statement():
if NextToken == null :
return null
match NextToken.Type :
TokenType.Conditional_if :
return parse_If_Statement()
TokenType.StatementEnd :
return parse_EmptyStatement()
TokenType.StmtBlockStart :
return parse_BlockStatement()
TokenType.VarDeclare :
return parse_VariableStatement()
return parse_ExpressionStatement()
# If Statement
# : if ( Expression ) Statement
# | if ( Expression ) Statement else Statement
# ;
func parse_If_Statement():
eat(TokenType.Conditional_if)
eat(TokenType.ExpressionPStart)
var condition = parse_Expression()
eat(TokenType.ExpressionPEnd)
var consequent = parse_Statement()
var alternative = null
if NextToken != null && NextToken.Type == TokenType.Conditional_else :
eat(TokenType.Conditional_else)
alternative = parse_Statement()
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ConditionalStatement
node.Value = [ condition, consequent, alternative ]
return node
# > Statement
# EmptyStatement
# ;
func parse_EmptyStatement():
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.EmptyStatement
return node
# > Statement
# BlockStatement
# : { OptStatementList }
# ;
func parse_BlockStatement():
eat(TokenType.StmtBlockStart)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BlockStatement
if NextToken.Type != TokenType.StmtBlockEnd :
node.Value = parse_StatementList(TokenType.StmtBlockEnd)
else :
node.Value = []
eat(TokenType.StmtBlockEnd)
return node
# > Statement
# VariableStatement
# : VarDeclare VariableDeclarationList StatementEnd
# ;
func parse_VariableStatement():
eat(TokenType.VarDeclare)
var declarations = parse_VariableDeclarationList()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableStatement
node.Value = declarations
return node
# > Statement
# ExpressionStatement
# : Expression
# ;
func parse_ExpressionStatement():
var expression = parse_Expression()
eat(TokenType.StatementEnd)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.ExpressionStatement
node.Value = expression
return expression
# > ExpressionStatement
# > If_Statement
# > PrimaryExpression
# Expression
# : AssignmentExpression
# ;
func parse_Expression():
return parse_AssignmentExpression()
# > VariableStatement
# VariableDeclarationList
# : VariableDeclaration
# | VariableDelcarationList , VariableDeclaration -> VariableDelcaration , ...
func parse_VariableDeclarationList():
var \
declarations = []
declarations.append(parse_VariableDeclaration())
while NextToken.Type == TokenType.CommaDelimiter :
eat(TokenType.CommaDelimiter)
declarations.append(parse_VariableDeclaration())
return declarations
# > VariableDeclarationList
# VariableDeclaration
# : Identifier OptVariableInitalizer
# ;
func parse_VariableDeclaration():
var identifier = parse_Identifier()
var initalizer
if NextToken.Type != TokenType.StatementEnd && NextToken.Type != TokenType.CommaDelimiter :
initalizer = parse_VariableInitializer()
else :
initalizer = null
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.VariableDeclaration
node.Value = [ identifier, initalizer ]
return node
# > VariableDeclaration
# VariableInitializer
# : Assignment AssignmentExpression
# ;
func parse_VariableInitializer():
eat(TokenType.Assignment)
return parse_AssignmentExpression()
# > Expression
# > VariableInitializer
# > AssignmentExpression
# AssignmentExpression
# : RelationalExpression
# | ResolvedSymbol AssignmentOperator AssignmetnExpression
# ;
func parse_AssignmentExpression():
var left = parse_LogicalOrExpression()
if NextToken.Type != TokenType.Assignment && NextToken.Type != TokenType.ComplexAssignment :
return left
var assignmentOp;
if NextToken.Type == TokenType.Assignment :
assignmentOp = eat(TokenType.Assignment)
elif NextToken.Type == TokenType.ComplexAssignment :
assignmentOp = eat(TokenType.ComplexAssignment)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.AssignmentExpression
node.Value = \
[
assignmentOp.Value,
left,
parse_AssignmentExpression()
]
return node
# > VariableDeclaration
# > ParenthesizedExpression
# Identifier
# : IdentifierSymbol
# ;
func parse_Identifier():
var name = eat(TokenType.Identifier).Value
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.Identifier
node.Value = name
return node
# > AssignmentExpression
# Logical Or Expression
# : LogicalAndExpression Logical_Or LogicalOrExpression
# | LogicalOrExpression
# ;
func parse_LogicalOrExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("pasre_LogicalAndExpression")
return parse_LogicalExpression(parseFn, TokenType.Logical_Or)
# > LogicaOrExpression
# Logical And Expression
# : EqualityExpression Logical_And LogicalAndExpression
# | EqualityExpression
# ;
func pasre_LogicalAndExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_EqualityExpression")
return parse_LogicalExpression(parseFn, TokenType.Logical_And)
# Equality Operators: ==, !=
#
# > LogicalAndExpression
# EqualityExpression
# : RelationalExpression EqualityOp RelationalExpression
# | RelationalExpression
# ;
func parse_EqualityExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_RelationalExpression")
return parse_BinaryExpression(parseFn, TokenType.EqualityOp)
# Relational Operators: >, >=, <, <=
#
# > EqualityExpression
# Relational Expression
# : AdditiveExpression
# | AdditiveExpression RelationalOp RelationalExpression
# ;
func parse_RelationalExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_AdditiveExpression")
return parse_BinaryExpression(parseFn, TokenType.RelationalOp)
# > RelationalExpression
# AdditiveExpression
# : MultiplicativeExpression
# | AdditiveExpression AdditiveOp MultiplicativeExpression -> MultiplicativeExpression AdditiveOp ... Literal
# ;
func parse_AdditiveExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_MultiplicativeExpression")
return parse_BinaryExpression(parseFn, TokenType.AdditiveOp)
# > AdditiveExpression
# MultiplicativeExpression
# : UnaryExpressioon
# : MultiplicativeExpression MultiplicativeOp UnaryExpression -> UnaryExpression MultiplicativeOp ... Literal
# ;
func parse_MultiplicativeExpression():
var \
parseFn = FuncRef.new()
parseFn.set_instance(self)
parseFn.set_function("parse_UnaryExpression")
return parse_BinaryExpression(parseFn, TokenType.MultiplicativeOp)
# > MultiplicativeExpression
# > UnaryExpression
# UnaryExpression
# : ResolvedSymbol
# | AdditiveOp UnaryExpression
# | Logical_Not UnaryExpression
# ;
func parse_UnaryExpression():
var operator
match NextToken.Type:
TokenType.AdditiveOp:
operator = eat(TokenType.AdditiveOp).Value
TokenType.Logical_Not:
operator = eat(TokenType.Logical_Not).Value
if operator == null :
return parse_ResolvedSymbol()
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.UnaryExpression
node.Value = [ operator, parse_UnaryExpression() ]
return node;
# > UnaryExpression
# > PrimaryExpression
# ResolvedSymbol (LeftHandExpression)
# : PrimaryExpression
# ;
func parse_ResolvedSymbol():
return parse_PrimaryExpression()
# var resolvedSymbol = parse_Identifier()
# if resolvedSymbol.Type == SyntaxNodeType.Identifier :
# return resolvedSymbol
# var assertStrTmplt = "parse_ResolvedSymbol: Unexpected symbol: {value}"
# var assertStr = assertStrTmplt.format({"value" : resolvedSymbol.Type})
# assert(true != true, assertStr)
# > ResolvedSymbol
# PrimaryExpression
# : Literal
# | ParenthesizedExpression
# ;
func parse_PrimaryExpression():
if is_Literal():
return parse_Literal()
match NextToken.Type:
TokenType.ExpressionPStart:
return parse_ParenthesizedExpression()
TokenType.Identifier:
var identifier = parse_Identifier()
if identifier.Type == SyntaxNodeType.Identifier :
return identifier
var assertStrTmplt = "parse_PrimaryExpression: (Identifier) Unexpected symbol: {value}"
var assertStr = assertStrTmplt.format({"value" : identifier.Type})
assert(true != true, assertStr)
return parse_ResolvedSymbol()
# > PrimaryExpression
# Literal
# : NumericLiteral
# | StringLiteral
# | BooleanLiteral
# | NullLiteral
# ;
func parse_Literal():
match NextToken.Type :
TokenType.Number:
return parse_NumericLiteral()
TokenType.String:
return parse_StringLiteral()
TokenType.Bool_true:
return parse_BooleanLiteral(TokenType.Bool_true)
TokenType.Bool_false:
return parse_BooleanLiteral(TokenType.Bool_false)
TokenType.NullValue:
return parse_NullLiteral()
assert(false, "parse_Literal: Was not able to detect valid literal type from NextToken")
# > PrimaryExpression
# ParenthesizedExpression
# : ( Expression )
# ;
func parse_ParenthesizedExpression():
eat(TokenType.ExpressionPStart)
var expression = parse_Expression()
eat(TokenType.ExpressionPEnd)
return expression
# > Literal
# NumericLiteral
# : Number
# ;
func parse_NumericLiteral():
var Token = eat(TokenType.Number)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NumericLiteral
node.Value = int( Token.Value )
return node
# > Literal
# StringLiteral
# : String
# ;
func parse_StringLiteral():
var Token = eat(TokenType.String)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.StringLiteral
node.Value = Token.Value.substr( 1, Token.Value.length() - 2 )
return node
# > Literal
# BooleanLiteral
# : true
# | false
# ;
func parse_BooleanLiteral(token):
eat(token)
var value
if (TokenType.Bool_true == token) :
value = true
elif (TokenType.Bool_false == token) :
value = false
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.BooleanLiteral
node.Value = value
return node
# > Literal
# NullLiteral
# : null
# ;
func parse_NullLiteral():
eat(TokenType.NullLiteral)
var \
node = SyntaxNode.new()
node.Type = SyntaxNodeType.NullLiteral
node.Value = null
return node
var GParser = Parser.new()
onready var TextOut = GScene.get_node("TextOutput")
func tout(text):
TextOut.insert_text_at_cursor(text)
const Tests = \
{
# MultiStatement = \
# {
# Name = "Multi-Statement",
# File = "1.Multi-Statement.uf"
# },
# BlockStatement = \
# {
# Name = "Block Statement",
# File = "2.BlockStatement.uf"
# },
# BinaryExpression = \
# {
# Name = "Binary Expression",
# File = "3.BinaryExpression.uf"
# },
# Assignment = \
# {
# Name = "Assignment",
# File = "4.Assignment.uf"
# },
# VaraibleDeclaration = \
# {
# Name = "Variable Declaration",
# File = "5.VariableDeclaration.uf"
# },
# Conditionals = \
# {
# Name = "Conditionals",
# File = "6.Conditionals.uf"
# },
# Relations = \
# {
# Name = "Relations",
# File = "7.Relations.uf"
# },
# Equality = \
# {
# Name = "Equality",
# File = "8.Equality.uf"
# },
# Logical = \
# {
# Name = "Logical",
# File = "9.Logical.uf"
# },
Unary = \
{
Name = "Unary",
File = "10.Unary.uf"
}
}
func test(entry):
var introMessage = "Testing: {Name}\n"
var introMessageFormatted = introMessage.format({"Name" : entry.Name})
tout(introMessageFormatted)
var path
if Engine.editor_hint :
path = "res://../Tests/{TestName}"
else :
path = "res://../Builds/Tests/{TestName}"
var pathFormatted = path.format({"TestName" : entry.File})
var \
file = File.new()
file.open(pathFormatted, File.READ)
var programDescription = file.get_as_text()
file.close()
GTokenizer.init(programDescription)
var ast = GParser.parse(GTokenizer)
var json = JSON.print(ast.to_Dictionary(), '\t')
tout(json + "\n")
tout("Passed!\n")
# Main Entry point.
func _ready():
for Key in Tests :
test(Tests[Key])

View File

@ -18,7 +18,7 @@ config/icon="res://Assets/Branding/RDP_Class_cover_small.png"
[autoload]
GScene="*res://Lectures/Lecture.tscn"
GScript="*res://Lectures/Lecture.11.gd"
GScript="*res://Lectures/Lecture.12.gd"
[gui]