mirror of
https://github.com/Ed94/metadesk.git
synced 2026-08-06 23:58:48 +00:00
initial review of parse_from_text_tokens
This commit is contained in:
@@ -1,10 +0,0 @@
|
|||||||
#ifdef INTELLISENSE_DIRECTIVES
|
|
||||||
#pragma once
|
|
||||||
#include "base/memory.h"
|
|
||||||
#endif
|
|
||||||
|
|
||||||
typedef struct Context Context;
|
|
||||||
struct Context
|
|
||||||
{
|
|
||||||
Allocator allocator;
|
|
||||||
};
|
|
||||||
+235
-220
@@ -608,6 +608,45 @@ tokenize_from_text(Arena* arena, String8 text)
|
|||||||
////////////////////////////////
|
////////////////////////////////
|
||||||
//~ rjf: Tokens -> Tree Functions
|
//~ rjf: Tokens -> Tree Functions
|
||||||
|
|
||||||
|
typedef enum ParseWorkKind ParseWorkKind;
|
||||||
|
enum ParseWorkKind
|
||||||
|
{
|
||||||
|
ParseWorkKind_Main,
|
||||||
|
ParseWorkKind_MainImplicit,
|
||||||
|
ParseWorkKind_NodeOptionalFollowUp,
|
||||||
|
ParseWorkKind_NodeChildrenStyleScan,
|
||||||
|
|
||||||
|
ParseWorkKind_UnderlyingType = MAX_U32,
|
||||||
|
};
|
||||||
|
|
||||||
|
typedef struct ParseWorkNode ParseWorkNode;
|
||||||
|
struct ParseWorkNode
|
||||||
|
{
|
||||||
|
ParseWorkNode* next;
|
||||||
|
ParseWorkKind kind;
|
||||||
|
Node* parent;
|
||||||
|
Node* first_gathered_tag;
|
||||||
|
Node* last_gathered_tag;
|
||||||
|
NodeFlags gathered_node_flags;
|
||||||
|
S32 counted_newlines;
|
||||||
|
};
|
||||||
|
|
||||||
|
#if 0
|
||||||
|
void parser_work_push(work_kind, work_parent, work_free)
|
||||||
|
{
|
||||||
|
ParseWorkNode* work_node = work_free;
|
||||||
|
if (work_node == 0) {
|
||||||
|
work_node = push_array(scratch.arena, MD_ParseWorkNode, 1);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
sll_stack_pop(work_free);
|
||||||
|
}
|
||||||
|
work_node->kind = (work_kind);
|
||||||
|
work_node->parent = (work_parent);
|
||||||
|
sll_stack_push(work_top, work_node);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
ParseResult
|
ParseResult
|
||||||
parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray tokens)
|
parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray tokens)
|
||||||
{
|
{
|
||||||
@@ -618,301 +657,277 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
|
|||||||
Node *root = push_node(arena, NodeKind_File, 0, filename, text, 0);
|
Node *root = push_node(arena, NodeKind_File, 0, filename, text, 0);
|
||||||
|
|
||||||
//- rjf: set up parse rule stack
|
//- rjf: set up parse rule stack
|
||||||
typedef enum MD_ParseWorkKind
|
|
||||||
{
|
ParseWorkNode first_work = {
|
||||||
MD_ParseWorkKind_Main,
|
|
||||||
MD_ParseWorkKind_MainImplicit,
|
|
||||||
MD_ParseWorkKind_NodeOptionalFollowUp,
|
|
||||||
MD_ParseWorkKind_NodeChildrenStyleScan,
|
|
||||||
}
|
|
||||||
MD_ParseWorkKind;
|
|
||||||
typedef struct MD_ParseWorkNode MD_ParseWorkNode;
|
|
||||||
struct MD_ParseWorkNode
|
|
||||||
{
|
|
||||||
MD_ParseWorkNode *next;
|
|
||||||
MD_ParseWorkKind kind;
|
|
||||||
Node *parent;
|
|
||||||
Node *first_gathered_tag;
|
|
||||||
Node *last_gathered_tag;
|
|
||||||
NodeFlags gathered_node_flags;
|
|
||||||
S32 counted_newlines;
|
|
||||||
};
|
|
||||||
MD_ParseWorkNode first_work =
|
|
||||||
{
|
|
||||||
0,
|
0,
|
||||||
MD_ParseWorkKind_Main,
|
ParseWorkKind_Main,
|
||||||
root,
|
root,
|
||||||
};
|
};
|
||||||
MD_ParseWorkNode broken_work = { 0, MD_ParseWorkKind_Main, root,};
|
|
||||||
MD_ParseWorkNode *work_top = &first_work;
|
ParseWorkNode broken_work = { 0, ParseWorkKind_Main, root,};
|
||||||
MD_ParseWorkNode *work_free = 0;
|
ParseWorkNode* work_top = &first_work;
|
||||||
#define MD_ParseWorkPush(work_kind, work_parent) do\
|
ParseWorkNode* work_free = 0;
|
||||||
{\
|
|
||||||
MD_ParseWorkNode *work_node = work_free;\
|
#define parser_work_push(work_kind, work_parent) do \
|
||||||
if(work_node == 0) {work_node = push_array(scratch.arena, MD_ParseWorkNode, 1);}\
|
{ \
|
||||||
else { SLLStackPop(work_free); }\
|
ParseWorkNode* work_node = work_free; \
|
||||||
work_node->kind = (work_kind);\
|
if (work_node == 0) { \
|
||||||
work_node->parent = (work_parent);\
|
work_node = push_array(scratch.arena, MD_ParseWorkNode, 1); \
|
||||||
SLLStackPush(work_top, work_node);\
|
} \
|
||||||
}while(0)
|
else { \
|
||||||
#define MD_ParseWorkPop() do\
|
sll_stack_pop(work_free); \
|
||||||
{\
|
} \
|
||||||
SLLStackPop(work_top);\
|
work_node->kind = (work_kind); \
|
||||||
if(work_top == 0) {work_top = &broken_work;}\
|
work_node->parent = (work_parent); \
|
||||||
}while(0)
|
sll_stack_push(work_top, work_node); \
|
||||||
|
} while(0)
|
||||||
|
|
||||||
|
#define parse_work_pop() do \
|
||||||
|
{ \
|
||||||
|
sll_stack_pop(work_top); \
|
||||||
|
if (work_top == 0) { \
|
||||||
|
work_top = &broken_work; \
|
||||||
|
} \
|
||||||
|
} while(0)
|
||||||
|
|
||||||
//- rjf: parse
|
//- rjf: parse
|
||||||
Token *tokens_first = tokens.v;
|
Token* tokens_first = tokens.v;
|
||||||
Token *tokens_opl = tokens_first + tokens.count;
|
Token* tokens_opl = tokens_first + tokens.count;
|
||||||
Token *token = tokens_first;
|
Token* token = tokens_first;
|
||||||
for(;token < tokens_opl;)
|
for(;token < tokens_opl;)
|
||||||
{
|
{
|
||||||
//- rjf: unpack token
|
//- rjf: unpack token
|
||||||
String8 token_string = str8_substr(text, token[0].range);
|
String8 token_string = str8_substr(text, token[0].range);
|
||||||
|
|
||||||
//- rjf: whitespace -> always no-op & inc
|
//- rjf: whitespace -> always no-op & inc
|
||||||
if(token->flags & TokenFlag_Whitespace)
|
if (token->flags & TokenFlag_Whitespace) {
|
||||||
{
|
token += 1;
|
||||||
token += 1;
|
goto end_consume;
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: comments -> always no-op & inc
|
//- rjf: comments -> always no-op & inc
|
||||||
if(token->flags & TokenGroup_Comment)
|
if (token->flags & TokenGroup_Comment) {
|
||||||
{
|
token += 1;
|
||||||
token += 1;
|
goto end_consume;
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [node follow up] : following label -> work top parent has children. we need
|
//- rjf: [node follow up] : following label -> work top parent has children. we need
|
||||||
// to scan for explicit delimiters, else parse an implicitly delimited set of children
|
// to scan for explicit delimiters, else parse an implicitly delimited set of children
|
||||||
if(work_top->kind == MD_ParseWorkKind_NodeOptionalFollowUp && str8_match(token_string, str8_lit(":"), 0))
|
if (work_top->kind == ParseWorkKind_NodeOptionalFollowUp && str8_match(token_string, str8_lit(":"), 0)) {
|
||||||
{
|
Node* parent = work_top->parent;
|
||||||
Node *parent = work_top->parent;
|
parse_work_pop();
|
||||||
MD_ParseWorkPop();
|
parse_work_push(ParseWorkKind_NodeChildrenStyleScan, parent);
|
||||||
MD_ParseWorkPush(MD_ParseWorkKind_NodeChildrenStyleScan, parent);
|
token += 1;
|
||||||
token += 1;
|
goto end_consume;
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [node follow up] anything but : following label -> node has no children. just
|
//- rjf: [node follow up] anything but : following label -> node has no children. just
|
||||||
// pop & move on
|
// pop & move on
|
||||||
if(work_top->kind == MD_ParseWorkKind_NodeOptionalFollowUp)
|
if (work_top->kind == ParseWorkKind_NodeOptionalFollowUp) {
|
||||||
{
|
parse_work_pop();
|
||||||
MD_ParseWorkPop();
|
goto end_consume;
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [main] separators -> mark & inc
|
//- rjf: [main] separators -> mark & inc
|
||||||
if(work_top->kind == MD_ParseWorkKind_Main && token->flags & TokenFlag_Reserved &&
|
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit(","), 0) || str8_match(token_string, str8_lit(";"), 0)))
|
||||||
(str8_match(token_string, str8_lit(","), 0) ||
|
|
||||||
str8_match(token_string, str8_lit(";"), 0)))
|
|
||||||
{
|
{
|
||||||
Node *parent = work_top->parent;
|
Node* parent = work_top->parent;
|
||||||
if(!node_is_nil(parent->last))
|
if (!node_is_nil(parent->last))
|
||||||
{
|
{
|
||||||
parent->last->flags |= NodeFlag_IsBeforeComma*!!str8_match(token_string, str8_lit(","), 0);
|
parent->last->flags |= NodeFlag_IsBeforeComma *!! str8_match(token_string, str8_lit(","), 0);
|
||||||
parent->last->flags |= NodeFlag_IsBeforeSemicolon*!!str8_match(token_string, str8_lit(";"), 0);
|
parent->last->flags |= NodeFlag_IsBeforeSemicolon *!! str8_match(token_string, str8_lit(";"), 0);
|
||||||
work_top->gathered_node_flags |= NodeFlag_IsAfterComma*!!str8_match(token_string, str8_lit(","), 0);
|
work_top->gathered_node_flags |= NodeFlag_IsAfterComma *!! str8_match(token_string, str8_lit(","), 0);
|
||||||
work_top->gathered_node_flags |= NodeFlag_IsAfterSemicolon*!!str8_match(token_string, str8_lit(";"), 0);
|
work_top->gathered_node_flags |= NodeFlag_IsAfterSemicolon *!! str8_match(token_string, str8_lit(";"), 0);
|
||||||
}
|
}
|
||||||
token += 1;
|
token += 1;
|
||||||
goto end_consume;
|
goto end_consume;
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [main_implicit] separators -> pop
|
//- rjf: [main_implicit] separators -> pop
|
||||||
if(work_top->kind == MD_ParseWorkKind_MainImplicit && token->flags & TokenFlag_Reserved &&
|
if(work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit(","), 0) || str8_match(token_string, str8_lit(";"), 0))) {
|
||||||
(str8_match(token_string, str8_lit(","), 0) ||
|
parse_work_pop();
|
||||||
str8_match(token_string, str8_lit(";"), 0)))
|
goto end_consume;
|
||||||
{
|
|
||||||
MD_ParseWorkPop();
|
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [main, main_implicit] unexpected reserved tokens
|
//- rjf: [main, main_implicit] unexpected reserved tokens
|
||||||
if((work_top->kind == MD_ParseWorkKind_Main || work_top->kind == MD_ParseWorkKind_MainImplicit) &&
|
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("#"), 0) || str8_match(token_string, str8_lit("\\"), 0) || str8_match(token_string, str8_lit(":"), 0)))
|
||||||
token->flags & TokenFlag_Reserved &&
|
|
||||||
(str8_match(token_string, str8_lit("#"), 0) ||
|
|
||||||
str8_match(token_string, str8_lit("\\"), 0) ||
|
|
||||||
str8_match(token_string, str8_lit(":"), 0)))
|
|
||||||
{
|
{
|
||||||
Node *error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
|
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
|
||||||
String8 error_string = push_str8f(arena, "Unexpected reserved symbol \"%S\".", token_string);
|
String8 error_string = push_str8f(arena, "Unexpected reserved symbol \"%S\".", token_string);
|
||||||
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
|
|
||||||
token += 1;
|
|
||||||
goto end_consume;
|
|
||||||
}
|
|
||||||
|
|
||||||
//- rjf: [main, main_implicit] tag signifier -> create new tag
|
|
||||||
if((work_top->kind == MD_ParseWorkKind_Main || work_top->kind == MD_ParseWorkKind_MainImplicit) &&
|
|
||||||
token[0].flags & TokenFlag_Reserved && str8_match(token_string, str8_lit("@"), 0))
|
|
||||||
{
|
|
||||||
if(token+1 >= tokens_opl ||
|
|
||||||
!(token[1].flags & TokenGroup_Label))
|
|
||||||
{
|
|
||||||
Node *error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
|
|
||||||
String8 error_string = str8_lit("Tag label expected after @ symbol.");
|
|
||||||
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
|
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
|
||||||
token += 1;
|
token += 1;
|
||||||
goto end_consume;
|
goto end_consume;
|
||||||
}
|
}
|
||||||
else
|
|
||||||
|
//- rjf: [main, main_implicit] tag signifier -> create new tag
|
||||||
|
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token[0].flags & TokenFlag_Reserved && str8_match(token_string, str8_lit("@"), 0))
|
||||||
{
|
{
|
||||||
String8 tag_name_raw = str8_substr(text, token[1].range);
|
if (token+1 >= tokens_opl || !(token[1].flags & TokenGroup_Label))
|
||||||
String8 tag_name = content_string_from_token_flags_str8(token[1].flags, tag_name_raw);
|
|
||||||
Node *node = push_node(arena, NodeKind_Tag, node_flags_from_token_flags(token[1].flags), tag_name, tag_name_raw, token[0].range.min);
|
|
||||||
dll_push_back_npz(nil_node(), work_top->first_gathered_tag, work_top->last_gathered_tag, node, next, prev);
|
|
||||||
if(token+2 < tokens_opl && token[2].flags & TokenFlag_Reserved && str8_match(str8_substr(text, token[2].range), str8_lit("("), 0))
|
|
||||||
{
|
{
|
||||||
token += 3;
|
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
|
||||||
MD_ParseWorkPush(MD_ParseWorkKind_Main, node);
|
String8 error_string = str8_lit("Tag label expected after @ symbol.");
|
||||||
|
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
|
||||||
|
|
||||||
|
token += 1;
|
||||||
|
goto end_consume;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
token += 2;
|
String8 tag_name_raw = str8_substr(text, token[1].range);
|
||||||
|
String8 tag_name = content_string_from_token_flags_str8(token[1].flags, tag_name_raw);
|
||||||
|
|
||||||
|
Node* node = push_node(arena, NodeKind_Tag, node_flags_from_token_flags(token[1].flags), tag_name, tag_name_raw, token[0].range.min);
|
||||||
|
dll_push_back_npz(nil_node(), work_top->first_gathered_tag, work_top->last_gathered_tag, node, next, prev);
|
||||||
|
|
||||||
|
if (token + 2 < tokens_opl && token[2].flags & TokenFlag_Reserved && str8_match(str8_substr(text, token[2].range), str8_lit("("), 0)) {
|
||||||
|
token += 3;
|
||||||
|
parse_work_push(ParseWorkKind_Main, node);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
token += 2;
|
||||||
|
}
|
||||||
|
goto end_consume;
|
||||||
}
|
}
|
||||||
goto end_consume;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [main, main_implicit] label -> create new main
|
//- rjf: [main, main_implicit] label -> create new main
|
||||||
if((work_top->kind == MD_ParseWorkKind_Main || work_top->kind == MD_ParseWorkKind_MainImplicit) &&
|
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token->flags & TokenGroup_Label)
|
||||||
token->flags & TokenGroup_Label)
|
|
||||||
{
|
{
|
||||||
String8 node_string_raw = token_string;
|
String8 node_string_raw = token_string;
|
||||||
String8 node_string = content_string_from_token_flags_str8(token->flags, node_string_raw);
|
String8 node_string = content_string_from_token_flags_str8(token->flags, node_string_raw);
|
||||||
NodeFlags flags = node_flags_from_token_flags(token->flags)|work_top->gathered_node_flags;
|
NodeFlags flags = node_flags_from_token_flags(token->flags)|work_top->gathered_node_flags;
|
||||||
work_top->gathered_node_flags = 0;
|
|
||||||
Node *node = push_node(arena, NodeKind_Main, flags, node_string, node_string_raw, token[0].range.min);
|
work_top->gathered_node_flags = 0;
|
||||||
node->first_tag = work_top->first_gathered_tag;
|
|
||||||
node->last_tag = work_top->last_gathered_tag;
|
Node* node = push_node(arena, NodeKind_Main, flags, node_string, node_string_raw, token[0].range.min);
|
||||||
for(Node *tag = work_top->first_gathered_tag; !node_is_nil(tag); tag = tag->next)
|
node->first_tag = work_top->first_gathered_tag;
|
||||||
{
|
node->last_tag = work_top->last_gathered_tag;
|
||||||
tag->parent = node;
|
for (Node* tag = work_top->first_gathered_tag; !node_is_nil(tag); tag = tag->next) {
|
||||||
}
|
tag->parent = node;
|
||||||
work_top->first_gathered_tag = work_top->last_gathered_tag = nil_node();
|
}
|
||||||
node_push_child(work_top->parent, node);
|
work_top->first_gathered_tag = work_top->last_gathered_tag = nil_node();
|
||||||
MD_ParseWorkPush(MD_ParseWorkKind_NodeOptionalFollowUp, node);
|
|
||||||
token += 1;
|
node_push_child(work_top->parent, node);
|
||||||
goto end_consume;
|
parse_work_push(ParseWorkKind_NodeOptionalFollowUp, node);
|
||||||
|
token += 1;
|
||||||
|
goto end_consume;
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [main] {s, [s, and (s -> create new main
|
//- rjf: [main] {s, [s, and (s -> create new main
|
||||||
if(work_top->kind == MD_ParseWorkKind_Main && token->flags & TokenFlag_Reserved &&
|
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved &&
|
||||||
(str8_match(token_string, str8_lit("{"), 0) ||
|
(
|
||||||
str8_match(token_string, str8_lit("["), 0) ||
|
str8_match(token_string, str8_lit("{"), 0) ||
|
||||||
str8_match(token_string, str8_lit("("), 0)))
|
str8_match(token_string, str8_lit("["), 0) ||
|
||||||
|
str8_match(token_string, str8_lit("("), 0)
|
||||||
|
)
|
||||||
|
)
|
||||||
{
|
{
|
||||||
NodeFlags flags = node_flags_from_token_flags(token->flags)|work_top->gathered_node_flags;
|
NodeFlags
|
||||||
flags |= NodeFlag_HasBraceLeft*!!str8_match(token_string, str8_lit("{"), 0);
|
flags = node_flags_from_token_flags(token->flags) | work_top->gathered_node_flags;
|
||||||
flags |= NodeFlag_HasBracketLeft*!!str8_match(token_string, str8_lit("["), 0);
|
flags |= NodeFlag_HasBraceLeft *!! str8_match(token_string, str8_lit("{"), 0);
|
||||||
flags |= NodeFlag_HasParenLeft*!!str8_match(token_string, str8_lit("("), 0);
|
flags |= NodeFlag_HasBracketLeft *!! str8_match(token_string, str8_lit("["), 0);
|
||||||
work_top->gathered_node_flags = 0;
|
flags |= NodeFlag_HasParenLeft *!! str8_match(token_string, str8_lit("("), 0);
|
||||||
Node *node = push_node(arena, NodeKind_Main, flags, str8_lit(""), str8_lit(""), token[0].range.min);
|
|
||||||
node->first_tag = work_top->first_gathered_tag;
|
work_top->gathered_node_flags = 0;
|
||||||
node->last_tag = work_top->last_gathered_tag;
|
|
||||||
for(Node *tag = work_top->first_gathered_tag; !node_is_nil(tag); tag = tag->next)
|
Node*
|
||||||
{
|
node = push_node(arena, NodeKind_Main, flags, str8_lit(""), str8_lit(""), token[0].range.min);
|
||||||
tag->parent = node;
|
node->first_tag = work_top->first_gathered_tag;
|
||||||
}
|
node->last_tag = work_top->last_gathered_tag;
|
||||||
work_top->first_gathered_tag = work_top->last_gathered_tag = nil_node();
|
for (Node *tag = work_top->first_gathered_tag; !node_is_nil(tag); tag = tag->next) {
|
||||||
node_push_child(work_top->parent, node);
|
tag->parent = node;
|
||||||
MD_ParseWorkPush(MD_ParseWorkKind_Main, node);
|
}
|
||||||
token += 1;
|
work_top->first_gathered_tag = work_top->last_gathered_tag = nil_node();
|
||||||
goto end_consume;
|
|
||||||
|
node_push_child(work_top->parent, node);
|
||||||
|
parse_work_push(ParseWorkKind_Main, node);
|
||||||
|
token += 1;
|
||||||
|
goto end_consume;
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [node children style scan] {s, [s, and (s -> explicitly delimited children
|
//- rjf: [node children style scan] {s, [s, and (s -> explicitly delimited children
|
||||||
if(work_top->kind == MD_ParseWorkKind_NodeChildrenStyleScan && token->flags & TokenFlag_Reserved &&
|
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && token->flags & TokenFlag_Reserved &&
|
||||||
(str8_match(token_string, str8_lit("{"), 0) ||
|
(
|
||||||
str8_match(token_string, str8_lit("["), 0) ||
|
str8_match(token_string, str8_lit("{"), 0) ||
|
||||||
str8_match(token_string, str8_lit("("), 0)))
|
str8_match(token_string, str8_lit("["), 0) ||
|
||||||
|
str8_match(token_string, str8_lit("("), 0)
|
||||||
|
)
|
||||||
|
)
|
||||||
{
|
{
|
||||||
Node *parent = work_top->parent;
|
Node *parent = work_top->parent;
|
||||||
parent->flags |= NodeFlag_HasBraceLeft*!!str8_match(token_string, str8_lit("{"), 0);
|
parent->flags |= NodeFlag_HasBraceLeft*!!str8_match(token_string, str8_lit("{"), 0);
|
||||||
parent->flags |= NodeFlag_HasBracketLeft*!!str8_match(token_string, str8_lit("["), 0);
|
parent->flags |= NodeFlag_HasBracketLeft*!!str8_match(token_string, str8_lit("["), 0);
|
||||||
parent->flags |= NodeFlag_HasParenLeft*!!str8_match(token_string, str8_lit("("), 0);
|
parent->flags |= NodeFlag_HasParenLeft*!!str8_match(token_string, str8_lit("("), 0);
|
||||||
MD_ParseWorkPop();
|
parse_work_pop();
|
||||||
MD_ParseWorkPush(MD_ParseWorkKind_Main, parent);
|
parse_work_push(ParseWorkKind_Main, parent);
|
||||||
token += 1;
|
token += 1;
|
||||||
goto end_consume;
|
goto end_consume;
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [node children style scan] count newlines
|
//- rjf: [node children style scan] count newlines
|
||||||
if(work_top->kind == MD_ParseWorkKind_NodeChildrenStyleScan && token->flags & TokenFlag_Newline)
|
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && token->flags & TokenFlag_Newline) {
|
||||||
{
|
work_top->counted_newlines += 1;
|
||||||
work_top->counted_newlines += 1;
|
token += 1;
|
||||||
token += 1;
|
goto end_consume;
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [main_implicit] newline -> pop
|
//- rjf: [main_implicit] newline -> pop
|
||||||
if(work_top->kind == MD_ParseWorkKind_MainImplicit && token->flags & TokenFlag_Newline)
|
if (work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Newline) {
|
||||||
{
|
parss_work_pop();
|
||||||
MD_ParseWorkPop();
|
token += 1;
|
||||||
token += 1;
|
goto end_consume;
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [all but main_implicit] newline -> no-op & inc
|
//- rjf: [all but main_implicit] newline -> no-op & inc
|
||||||
if(work_top->kind != MD_ParseWorkKind_MainImplicit && token->flags & TokenFlag_Newline)
|
if (work_top->kind != ParseWorkKind_MainImplicit && token->flags & TokenFlag_Newline) {
|
||||||
{
|
token += 1;
|
||||||
token += 1;
|
goto end_consume;
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [node children style scan] anything causing implicit set -> <2 newlines, all good,
|
//- rjf: [node children style scan] anything causing implicit set -> <2 newlines, all good,
|
||||||
// >=2 newlines, houston we have a problem
|
// >=2 newlines, houston we have a problem
|
||||||
if(work_top->kind == MD_ParseWorkKind_NodeChildrenStyleScan)
|
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan)
|
||||||
{
|
{
|
||||||
if(work_top->counted_newlines >= 2)
|
if (work_top->counted_newlines >= 2)
|
||||||
{
|
{
|
||||||
Node *node = work_top->parent;
|
Node* node = work_top->parent;
|
||||||
Node *error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
|
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
|
||||||
String8 error_string = push_str8f(arena, "More than two newlines following \"%S\", which has implicitly-delimited children, resulting in an empty list of children.", node->string);
|
String8 error_string = push_str8f(arena, "More than two newlines following \"%S\", which has implicitly-delimited children, resulting in an empty list of children.", node->string);
|
||||||
msg_list_push(arena, &msgs, error, MsgKind_Warning, error_string);
|
msg_list_push(arena, &msgs, error, MsgKind_Warning, error_string);
|
||||||
MD_ParseWorkPop();
|
MD_ParseWorkPop();
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
Node *parent = work_top->parent;
|
Node *parent = work_top->parent;
|
||||||
MD_ParseWorkPop();
|
MD_ParseWorkPop();
|
||||||
MD_ParseWorkPush(MD_ParseWorkKind_MainImplicit, parent);
|
MD_ParseWorkPush(ParseWorkKind_MainImplicit, parent);
|
||||||
}
|
}
|
||||||
goto end_consume;
|
goto end_consume;
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [main] }s, ]s, and )s -> pop
|
//- rjf: [main] }s, ]s, and )s -> pop
|
||||||
if(work_top->kind == MD_ParseWorkKind_Main && token->flags & TokenFlag_Reserved &&
|
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("}"), 0) || str8_match(token_string, str8_lit("]"), 0) || str8_match(token_string, str8_lit(")"), 0))) {
|
||||||
(str8_match(token_string, str8_lit("}"), 0) ||
|
Node *parent = work_top->parent;
|
||||||
str8_match(token_string, str8_lit("]"), 0) ||
|
parent->flags |= NodeFlag_HasBraceRight *!! str8_match(token_string, str8_lit("}"), 0);
|
||||||
str8_match(token_string, str8_lit(")"), 0)))
|
parent->flags |= NodeFlag_HasBracketRight *!! str8_match(token_string, str8_lit("]"), 0);
|
||||||
{
|
parent->flags |= NodeFlag_HasParenRight *!! str8_match(token_string, str8_lit(")"), 0);
|
||||||
Node *parent = work_top->parent;
|
parse_work_pop();
|
||||||
parent->flags |= NodeFlag_HasBraceRight*!!str8_match(token_string, str8_lit("}"), 0);
|
token += 1;
|
||||||
parent->flags |= NodeFlag_HasBracketRight*!!str8_match(token_string, str8_lit("]"), 0);
|
goto end_consume;
|
||||||
parent->flags |= NodeFlag_HasParenRight*!!str8_match(token_string, str8_lit(")"), 0);
|
|
||||||
MD_ParseWorkPop();
|
|
||||||
token += 1;
|
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: [main implicit] }s, ]s, and )s -> pop without advancing
|
//- rjf: [main implicit] }s, ]s, and )s -> pop without advancing
|
||||||
if(work_top->kind == MD_ParseWorkKind_MainImplicit && token->flags & TokenFlag_Reserved &&
|
if(work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("}"), 0) || str8_match(token_string, str8_lit("]"), 0) || str8_match(token_string, str8_lit(")"), 0))) {
|
||||||
(str8_match(token_string, str8_lit("}"), 0) ||
|
parse_work_pop();
|
||||||
str8_match(token_string, str8_lit("]"), 0) ||
|
goto end_consume;
|
||||||
str8_match(token_string, str8_lit(")"), 0)))
|
|
||||||
{
|
|
||||||
MD_ParseWorkPop();
|
|
||||||
goto end_consume;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
//- rjf: no consumption -> unexpected token! we don't know what to do with this.
|
//- rjf: no consumption -> unexpected token! we don't know what to do with this.
|
||||||
{
|
{
|
||||||
Node *error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
|
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
|
||||||
String8 error_string = push_str8f(arena, "Unexpected \"%S\" token.", token_string);
|
String8 error_string = push_str8f(arena, "Unexpected \"%S\" token.", token_string);
|
||||||
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
|
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
|
||||||
token += 1;
|
token += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
end_consume:;
|
end_consume:;
|
||||||
@@ -929,12 +944,12 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
|
|||||||
////////////////////////////////
|
////////////////////////////////
|
||||||
//~ rjf: Bundled Text -> Tree Functions
|
//~ rjf: Bundled Text -> Tree Functions
|
||||||
|
|
||||||
internal ParseResult
|
ParseResult
|
||||||
parse_from_text(Arena *arena, String8 filename, String8 text)
|
parse_from_text(Arena* arena, String8 filename, String8 text)
|
||||||
{
|
{
|
||||||
TempArena scratch = scratch_begin(&arena, 1);
|
TempArena scratch = scratch_begin(&arena, 1);
|
||||||
TokenizeResult tokenize = tokenize_from_text(scratch.arena, text);
|
TokenizeResult tokenize = tokenize_from_text(scratch.arena, text);
|
||||||
ParseResult parse = parse_from_text_tokens(arena, filename, text, tokenize.tokens);
|
ParseResult parse = parse_from_text_tokens(arena, filename, text, tokenize.tokens);
|
||||||
scratch_end(scratch);
|
scratch_end(scratch);
|
||||||
return parse;
|
return parse;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user