ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-08-06 07:38:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

mdesk: at Tokens -> Tree Functions

Browse Source
This commit is contained in:
ed
2025-02-08 14:11:56 -05:00
parent 31b8084dcf
commit af2c9eceaa
2 changed files with 730 additions and 710 deletions
Download Patch File Download Diff File Expand all files Collapse all files
code/mdesk/mdesk.c
+124 -102
View File
@@ -180,31 +180,25 @@ string_from_children(Arena* arena, Node* root)
//- rjf: tree comparison //- rjf: tree comparison
internal B32 B32
node_match(Node* a, Node* b, StringMatchFlags flags) node_match(Node* a, Node* b, StringMatchFlags flags)
{ {
B32 result = 0; B32 result = 0;
if (a->kind == b->kind && str8_match(a->string, b->string, flags)) if (a->kind == b->kind && str8_match(a->string, b->string, flags))
{ {
result = 1; result = 1;
if(result) if (result) {
{
result = result && a->flags == b->flags; result = result && a->flags == b->flags;
} }
if (result && a->kind != NodeKind_Tag) if (result && a->kind != NodeKind_Tag)
{ {
for(Node *a_tag = a->first_tag, *b_tag = b->first_tag; for (Node* a_tag = a->first_tag, *b_tag = b->first_tag; !node_is_nil(a_tag) || !node_is_nil(b_tag); a_tag = a_tag->next, b_tag = b_tag->next)
!node_is_nil(a_tag) || !node_is_nil(b_tag);
a_tag = a_tag->next, b_tag = b_tag->next)
{ {
if (node_match(a_tag, b_tag, flags)) if (node_match(a_tag, b_tag, flags))
{ {
for(Node *a_tag_arg = a_tag->first, *b_tag_arg = b_tag->first; for (Node* a_tag_arg = a_tag->first, *b_tag_arg = b_tag->first; !node_is_nil(a_tag_arg) || !node_is_nil(b_tag_arg); a_tag_arg = a_tag_arg->next, b_tag_arg = b_tag_arg->next)
!node_is_nil(a_tag_arg) || !node_is_nil(b_tag_arg);
a_tag_arg = a_tag_arg->next, b_tag_arg = b_tag_arg->next)
{
if(!tree_match(a_tag_arg, b_tag_arg, flags))
{ {
if (!tree_match(a_tag_arg, b_tag_arg, flags)) {
result = 0; result = 0;
goto end; goto end;
} }
@@ -222,15 +216,13 @@ node_match(Node *a, Node *b, StringMatchFlags flags)
return result; return result;
} }
internal B32 B32
tree_match(Node *a, Node *b, StringMatchFlags flags) tree_match(Node *a, Node *b, StringMatchFlags flags)
{ {
B32 result = node_match(a, b, flags); B32 result = node_match(a, b, flags);
if (result) if (result)
{ {
for(Node *a_child = a->first, *b_child = b->first; for(Node *a_child = a->first, *b_child = b->first; !node_is_nil(a_child) || !node_is_nil(b_child); a_child = a_child->next, b_child = b_child->next)
!node_is_nil(a_child) || !node_is_nil(b_child);
a_child = a_child->next, b_child = b_child->next)
{ {
if (!tree_match(a_child, b_child, flags)) if (!tree_match(a_child, b_child, flags))
{ {
@@ -245,7 +237,7 @@ tree_match(Node *a, Node *b, StringMatchFlags flags)
//- rjf: tree duplication //- rjf: tree duplication
internal Node * Node*
tree_copy(Arena* arena, Node* src_root) tree_copy(Arena* arena, Node* src_root)
{ {
Node* dst_root = nil_node(); Node* dst_root = nil_node();
@@ -263,21 +255,18 @@ tree_copy(Arena *arena, Node *src_root)
dst->raw_string = push_str8_copy(arena, src->raw_string); dst->raw_string = push_str8_copy(arena, src->raw_string);
dst->src_offset = src->src_offset; dst->src_offset = src->src_offset;
dst->parent = dst_parent; dst->parent = dst_parent;
if(dst_parent != nil_node()) if (dst_parent != nil_node()) {
{
dll_push_back_npz(nil_node(), dst_parent->first, dst_parent->last, dst, next, prev); dll_push_back_npz(nil_node(), dst_parent->first, dst_parent->last, dst, next, prev);
} }
else else {
{
dst_root = dst_parent = dst; dst_root = dst_parent = dst;
} }
rec = node_rec_depth_first_pre(src, src_root); rec = node_rec_depth_first_pre(src, src_root);
if(rec.push_count != 0) if (rec.push_count != 0) {
{
dst_parent = dst; dst_parent = dst;
} }
else for(U64 idx = 0; idx < rec.pop_count; idx += 1) else for (U64 idx = 0; idx < rec.pop_count; idx += 1) {
{
dst_parent = dst_parent->parent; dst_parent = dst_parent->parent;
} }
} }
@@ -288,14 +277,16 @@ tree_copy(Arena *arena, Node *src_root)
//////////////////////////////// ////////////////////////////////
//~ rjf: Text -> Tokens Functions //~ rjf: Text -> Tokens Functions
internal TokenizeResult TokenizeResult
tokenize_from_text(Arena* arena, String8 text) tokenize_from_text(Arena* arena, String8 text)
{ {
TempArena scratch = scratch_begin(&arena, 1); TempArena scratch = scratch_begin(&arena, 1);
TokenChunkList tokens = {0}; TokenChunkList tokens = {0};
MsgList msgs = {0}; MsgList msgs = {0};
U8* byte_first = text.str; U8* byte_first = text.str;
U8 *byte_opl = byte_first + text.size; U8* byte_opl = byte_first + text.size; // one-past-last
U8* byte = byte_first; U8* byte = byte_first;
//- rjf: scan string & produce tokens //- rjf: scan string & produce tokens
@@ -305,18 +296,20 @@ tokenize_from_text(Arena *arena, String8 text)
U8* token_start = 0; U8* token_start = 0;
U8* token_opl = 0; U8* token_opl = 0;
#define is_whitespace(byte) (*byte == ' ' || *byte == '\t' || *byte == '\v' || *byte == '\r')
//- rjf: whitespace //- rjf: whitespace
if(token_flags == 0 && (*byte == ' ' || *byte == '\t' || *byte == '\v' || *byte == '\r')) if (token_flags == 0 && is_whitespace(byte))
{ {
token_flags = TokenFlag_Whitespace; token_flags = TokenFlag_Whitespace;
token_start = byte; token_start = byte;
token_opl = byte; token_opl = byte;
byte += 1; byte += 1;
for (;byte <= byte_opl; byte += 1) for (;byte <= byte_opl; byte += 1)
{ {
token_opl += 1; token_opl += 1;
if(byte == byte_opl || (*byte != ' ' && *byte != '\t' && *byte != '\v' && *byte != '\r')) if (byte == byte_opl || !is_whitespace(byte)) {
{
break; break;
} }
} }
@@ -328,6 +321,7 @@ tokenize_from_text(Arena *arena, String8 text)
token_flags = TokenFlag_Newline; token_flags = TokenFlag_Newline;
token_start = byte; token_start = byte;
token_opl = byte+1; token_opl = byte+1;
byte += 1; byte += 1;
} }
@@ -337,27 +331,25 @@ tokenize_from_text(Arena *arena, String8 text)
token_flags = TokenFlag_Comment; token_flags = TokenFlag_Comment;
token_start = byte; token_start = byte;
token_opl = byte+2; token_opl = byte+2;
byte += 2; byte += 2;
B32 escaped = 0; B32 escaped = 0;
for (;byte <= byte_opl; byte += 1) for (;byte <= byte_opl; byte += 1)
{ {
token_opl += 1; token_opl += 1;
if(byte == byte_opl) if (byte == byte_opl) {
{
break; break;
} }
if(escaped) if (escaped) {
{
escaped = 0; escaped = 0;
} }
else else
{ {
if(*byte == '\n') if (*byte == '\n') {
{
break; break;
} }
else if(*byte == '\\') else if (*byte == '\\') {
{
escaped = 1; escaped = 1;
} }
} }
@@ -370,78 +362,93 @@ tokenize_from_text(Arena *arena, String8 text)
token_flags = TokenFlag_Comment; token_flags = TokenFlag_Comment;
token_start = byte; token_start = byte;
token_opl = byte + 2; token_opl = byte + 2;
byte += 2; byte += 2;
for (;byte <= byte_opl; byte += 1) for (;byte <= byte_opl; byte += 1)
{ {
token_opl += 1; token_opl += 1;
if(byte == byte_opl) if (byte == byte_opl) {
{
token_flags |= TokenFlag_BrokenComment; token_flags |= TokenFlag_BrokenComment;
break; break;
} }
if(byte+1 < byte_opl && byte[0] == '*' && byte[1] == '/') if (byte + 1 < byte_opl && byte[0] == '*' && byte[1] == '/') {
{
token_opl += 2; token_opl += 2;
break; break;
} }
} }
} }
#define is_identifier(byte) ( \
('A' <= *byte && *byte <= 'Z') || \
('a' <= *byte && *byte <= 'z') || \
*byte == '_' || \
utf8_class(*byte >> 3) >= 2 \
)
#if 0
(
!('A' <= *byte && *byte <= 'Z') &&
!('a' <= *byte && *byte <= 'z') &&
!('0' <= *byte && *byte <= '9') &&
*byte != '_' &&
utf8_class(*byte>>3) < 2
)
#endif
//- rjf: identifiers //- rjf: identifiers
if(token_flags == 0 && (('A' <= *byte && *byte <= 'Z') || if (token_flags == 0 && is_identifier(byte))
('a' <= *byte && *byte <= 'z') ||
*byte == '_' ||
utf8_class(*byte>>3) >= 2 ))
{ {
token_flags = TokenFlag_Identifier; token_flags = TokenFlag_Identifier;
token_start = byte; token_start = byte;
token_opl = byte; token_opl = byte;
byte += 1; byte += 1;
for(;byte <= byte_opl; byte += 1) for(;byte <= byte_opl; byte += 1)
{ {
token_opl += 1; token_opl += 1;
if(byte == byte_opl || if (byte == byte_opl || !is_identifier(byte)) {
(!('A' <= *byte && *byte <= 'Z') &&
!('a' <= *byte && *byte <= 'z') &&
!('0' <= *byte && *byte <= '9') &&
*byte != '_' &&
utf8_class(*byte>>3) < 2))
{
break; break;
} }
} }
} }
#define is_numeric(byte) ( \
('0' <= *byte && *byte <= '9') || \
(*byte == '.' && byte + 1 < byte_opl && '0' <= byte[1] && byte[1] <= '9') || \
(*byte == '-' && byte + 1 < byte_opl && '0' <= byte[1] && byte[1] <= '9') || \
*byte == '_' \
)
#define is_not_numeric(byte) ( \
!('A' <= *byte && *byte <= 'Z') && \
!('a' <= *byte && *byte <= 'z') && \
!('0' <= *byte && *byte <= '9') && \
*byte != '_' && *byte != '.' \
)
//- rjf: numerics //- rjf: numerics
if(token_flags == 0 && (('0' <= *byte && *byte <= '9') || if (token_flags == 0 && is_numeric(byte))
(*byte == '.' && byte+1 < byte_opl && '0' <= byte[1] && byte[1] <= '9') ||
(*byte == '-' && byte+1 < byte_opl && '0' <= byte[1] && byte[1] <= '9') ||
*byte == '_'))
{ {
token_flags = TokenFlag_Numeric; token_flags = TokenFlag_Numeric;
token_start = byte; token_start = byte;
token_opl = byte; token_opl = byte;
byte += 1; byte += 1;
for (;byte <= byte_opl; byte += 1) for (;byte <= byte_opl; byte += 1)
{ {
token_opl += 1; token_opl += 1;
if(byte == byte_opl || if (byte == byte_opl || is_not_numeric(byte)) {
(!('A' <= *byte && *byte <= 'Z') &&
!('a' <= *byte && *byte <= 'z') &&
!('0' <= *byte && *byte <= '9') &&
*byte != '_' &&
*byte != '.'))
{
break; break;
} }
} }
} }
#define is_triple_string_literal(byte) ( \
(byte[0] == '"' && byte[1] == '"' && byte[2] == '"' ) || \
(byte[0] == '\''&& byte[1] == '\''&& byte[2] == '\'') || \
(byte[0] == '`' && byte[1] == '`' && byte[2] == '`' ) \
)
//- rjf: triplet string literals //- rjf: triplet string literals
if(token_flags == 0 && byte+2 < byte_opl && if (token_flags == 0 && byte + 2 < byte_opl && is_triple_string_literal(byte))
((byte[0] == '"' && byte[1] == '"' && byte[2] == '"') ||
(byte[0] == '\''&& byte[1] == '\''&& byte[2] == '\'') ||
(byte[0] == '`' && byte[1] == '`' && byte[2] == '`')))
{ {
U8 literal_style = byte[0]; U8 literal_style = byte[0];
token_flags = TokenFlag_StringLiteral | TokenFlag_StringTriplet; token_flags = TokenFlag_StringLiteral | TokenFlag_StringTriplet;
@@ -450,17 +457,16 @@ tokenize_from_text(Arena *arena, String8 text)
token_flags |= (literal_style == '`') * TokenFlag_StringTick; token_flags |= (literal_style == '`') * TokenFlag_StringTick;
token_start = byte; token_start = byte;
token_opl = byte + 3; token_opl = byte + 3;
byte += 3; byte += 3;
for (;byte <= byte_opl; byte += 1) for (;byte <= byte_opl; byte += 1)
{ {
if(byte == byte_opl) if (byte == byte_opl) {
{
token_flags |= TokenFlag_BrokenStringLiteral; token_flags |= TokenFlag_BrokenStringLiteral;
token_opl = byte; token_opl = byte;
break; break;
} }
if(byte+2 < byte_opl && (byte[0] == literal_style && byte[1] == literal_style && byte[2] == literal_style)) if (byte + 2 < byte_opl && (byte[0] == literal_style && byte[1] == literal_style && byte[2] == literal_style)) {
{
byte += 3; byte += 3;
token_opl = byte; token_opl = byte;
break; break;
@@ -478,80 +484,93 @@ tokenize_from_text(Arena *arena, String8 text)
token_flags |= (literal_style == '`') * TokenFlag_StringTick; token_flags |= (literal_style == '`') * TokenFlag_StringTick;
token_start = byte; token_start = byte;
token_opl = byte + 1; token_opl = byte + 1;
byte += 1; byte += 1;
B32 escaped = 0; B32 escaped = 0;
for (;byte <= byte_opl; byte += 1) for (;byte <= byte_opl; byte += 1)
{ {
if(byte == byte_opl || *byte == '\n') if (byte == byte_opl || *byte == '\n') {
{
token_opl = byte; token_opl = byte;
token_flags |= TokenFlag_BrokenStringLiteral; token_flags |= TokenFlag_BrokenStringLiteral;
break; break;
} }
if(!escaped && byte[0] == '\\') if (!escaped && byte[0] == '\\') {
{
escaped = 1; escaped = 1;
} }
else if(!escaped && byte[0] == literal_style) else if (!escaped && byte[0] == literal_style) {
{
token_opl = byte+1; token_opl = byte+1;
byte += 1; byte += 1;
break; break;
} }
else if(escaped) else if (escaped) {
{
escaped = 0; escaped = 0;
} }
} }
} }
#define is_non_reserved_symbol(byte) ( \
*byte == '~' || *byte == '!' || *byte == '$' || *byte == '%' || *byte == '^' || \
*byte == '&' || *byte == '*' || *byte == '-' || *byte == '=' || *byte == '+' || \
*byte == '<' || *byte == '.' || *byte == '>' || *byte == '/' || *byte == '?' || \
*byte == '|' \
)
#if 0
(
*byte != '~' && *byte != '!' && *byte != '$' && *byte != '%' && *byte != '^' &&
*byte != '&' && *byte != '*' && *byte != '-' && *byte != '=' && *byte != '+' &&
*byte != '<' && *byte != '.' && *byte != '>' && *byte != '/' && *byte != '?' &&
*byte != '|'
)
#endif
//- rjf: non-reserved symbols //- rjf: non-reserved symbols
if(token_flags == 0 && (*byte == '~' || *byte == '!' || *byte == '$' || *byte == '%' || *byte == '^' || if (token_flags == 0 && is_non_reserved_symbol(byte))
*byte == '&' || *byte == '*' || *byte == '-' || *byte == '=' || *byte == '+' ||
*byte == '<' || *byte == '.' || *byte == '>' || *byte == '/' || *byte == '?' ||
*byte == '|'))
{ {
token_flags = TokenFlag_Symbol; token_flags = TokenFlag_Symbol;
token_start = byte; token_start = byte;
token_opl = byte; token_opl = byte;
byte += 1; byte += 1;
for (;byte <= byte_opl; byte += 1) for (;byte <= byte_opl; byte += 1)
{ {
token_opl += 1; token_opl += 1;
if(byte == byte_opl || if (byte == byte_opl || !is_non_reserved_symbol(byte)) {
(*byte != '~' && *byte != '!' && *byte != '$' && *byte != '%' && *byte != '^' &&
*byte != '&' && *byte != '*' && *byte != '-' && *byte != '=' && *byte != '+' &&
*byte != '<' && *byte != '.' && *byte != '>' && *byte != '/' && *byte != '?' &&
*byte != '|'))
{
break; break;
} }
} }
} }
#define is_reserved_symbol(byte) ( \
*byte == '{' || *byte == '}' || \
*byte == '(' || *byte == ')' || \
*byte == '[' || *byte == ']' || \
*byte == '#' || \
*byte == ',' || \
*byte == '\\'|| \
*byte == ':' || *byte == ';' || \
*byte == '@' \
)
//- rjf: reserved symbols //- rjf: reserved symbols
if(token_flags == 0 && (*byte == '{' || *byte == '}' || *byte == '(' || *byte == ')' || if (token_flags == 0 && is_reserved_symbol(byte)) {
*byte == '[' || *byte == ']' || *byte == '#' || *byte == ',' ||
*byte == '\\'|| *byte == ':' || *byte == ';' || *byte == '@'))
{
token_flags = TokenFlag_Reserved; token_flags = TokenFlag_Reserved;
token_start = byte; token_start = byte;
token_opl = byte+1; token_opl = byte+1;
byte += 1; byte += 1;
} }
//- rjf: bad characters in all other cases //- rjf: bad characters in all other cases
if(token_flags == 0) if (token_flags == 0) {
{
token_flags = TokenFlag_BadCharacter; token_flags = TokenFlag_BadCharacter;
token_start = byte; token_start = byte;
token_opl = byte+1; token_opl = byte+1;
byte += 1; byte += 1;
} }
//- rjf; push token if formed //- rjf; push token if formed
if(token_flags != 0 && token_start != 0 && token_opl > token_start) if (token_flags != 0 && token_start != 0 && token_opl > token_start) {
{
Token token = {{(U64)(token_start - byte_first), (U64)(token_opl - byte_first)}, token_flags}; Token token = {{(U64)(token_start - byte_first), (U64)(token_opl - byte_first)}, token_flags};
token_chunk_list_push(scratch.arena, &tokens, 4096, token); token_chunk_list_push(scratch.arena, &tokens, 4096, token);
} }
@@ -565,8 +584,7 @@ tokenize_from_text(Arena *arena, String8 text)
} }
//- rjf: push errors on unterminated strings //- rjf: push errors on unterminated strings
if(token_flags & TokenFlag_BrokenStringLiteral) if (token_flags & TokenFlag_BrokenStringLiteral) {
{
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first); Node* error = push_node(arena, NodeKind_ErrorMarker, 0, str8_lit(""), str8_lit(""), token_start - byte_first);
String8 error_string = str8_lit("Unterminated string literal."); String8 error_string = str8_lit("Unterminated string literal.");
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string); msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
@@ -574,19 +592,23 @@ tokenize_from_text(Arena *arena, String8 text)
} }
//- rjf: bake, fill & return //- rjf: bake, fill & return
TokenizeResult result = {0}; TokenizeResult result = {0}; {
{
result.tokens = token_array_from_chunk_list(arena, &tokens); result.tokens = token_array_from_chunk_list(arena, &tokens);
result.msgs = msgs; result.msgs = msgs;
} }
scratch_end(scratch); scratch_end(scratch);
return result; return result;
#undef byte_is_whitespace
#undef byte_is_identifier
#undef byte_is_numeric
#undef byte_is_not_numeric
} }
//////////////////////////////// ////////////////////////////////
//~ rjf: Tokens -> Tree Functions //~ rjf: Tokens -> Tree Functions
internal ParseResult ParseResult
parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray tokens) parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray tokens)
{ {
TempArena scratch = scratch_begin(&arena, 1); TempArena scratch = scratch_begin(&arena, 1);
code/mdesk/mdesk.h
+7 -9
View File
@@ -482,32 +482,30 @@ tag_count_from_node(Node* node) {
//- rjf: tree comparison //- rjf: tree comparison
internal B32 tree_match(Node *a, Node *b, StringMatchFlags flags); MD_API B32 tree_match(Node* a, Node* b, StringMatchFlags flags);
internal B32 node_match(Node *a, Node *b, StringMatchFlags flags); MD_API B32 node_match(Node* a, Node* b, StringMatchFlags flags);
//- rjf: tree duplication //- rjf: tree duplication
internal Node* tree_copy(Arena *arena, Node *src_root); MD_API Node* tree_copy(Arena* arena, Node* src_root);
// Node* tree_copy_arena(Node* src_root, Arena* arena);
// Node* tree_copy_ainfo(Node* src_root, AllocatorInfo info);
//////////////////////////////// ////////////////////////////////
//~ rjf: Text -> Tokens Functions //~ rjf: Text -> Tokens Functions
internal TokenizeResult tokenize_from_text(Arena *arena, String8 text); MD_API TokenizeResult tokenize_from_text(Arena* arena, String8 text);
//////////////////////////////// ////////////////////////////////
//~ rjf: Tokens -> Tree Functions //~ rjf: Tokens -> Tree Functions
internal ParseResult parse_from_text_tokens(Arena *arena, String8 filename, String8 text, TokenArray tokens); MD_API ParseResult parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray tokens);
//////////////////////////////// ////////////////////////////////
//~ rjf: Bundled Text -> Tree Functions //~ rjf: Bundled Text -> Tree Functions
internal ParseResult parse_from_text(Arena *arena, String8 filename, String8 text); ParseResult parse_from_text(Arena* arena, String8 filename, String8 text);
#define tree_from_string(arena, string) (parse_from_text((arena), str8_zero(), (string)).root) #define tree_from_string(arena, string) (parse_from_text((arena), str8_zero(), (string)).root)
//////////////////////////////// ////////////////////////////////
//~ rjf: Tree -> Text Functions //~ rjf: Tree -> Text Functions
internal String8List debug_string_list_from_tree(Arena *arena, Node *root); String8List debug_string_list_from_tree(Arena* arena, Node* root);