ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-08-06 15:48:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

adjustments to parse_from_text_tokens

Browse Source
This commit is contained in:
ed
2025-02-08 18:08:27 -05:00
parent 2ef486b7a0
commit 843cc7fa90
6 changed files with 124 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Readme.md
-1
View File
@@ -18,4 +18,3 @@ docs will be updated referencing content procued by Ryan Fleury and content base
* [examples](./examples/Readme.md) * [examples](./examples/Readme.md)
* [gen_c11](./gen_c11/Readme.md) * [gen_c11](./gen_c11/Readme.md)
* [gen_cpp17](./gen_cpp17/Readme.md) * [gen_cpp17](./gen_cpp17/Readme.md)
code/base/memory.h
+12 -1
View File
@@ -452,9 +452,11 @@ dll__insert_npz(
) \ ) \
) \ ) \
) )
// ! MD_LINKED_LIST_PURE_MACRO
#endif
// dll_insert_npz
#endif #endif
#endif
#ifndef dll_push_back_npz #ifndef dll_push_back_npz
// push-back next-previous with nil // push-back next-previous with nil
#define dll_push_back_npz(nil, f, l, n, next, prev) dll_insert_npz(nil, f, l, l, n, next, prev) #define dll_push_back_npz(nil, f, l, n, next, prev) dll_insert_npz(nil, f, l, l, n, next, prev)
@@ -517,7 +519,9 @@ dll__remove_npz(
: ((n)->next->prev = (n)->prev) \ : ((n)->next->prev = (n)->prev) \
) \ ) \
) )
// ! MD_LINKED_LIST_PURE_MACRO
#endif #endif
// dll_remove_npz
#endif #endif
//- rjf: singly-linked, doubly-headed lists (queues) //- rjf: singly-linked, doubly-headed lists (queues)
@@ -558,7 +562,9 @@ sll__queue_push_nz(
set_nil(nil,(n)->next) \ set_nil(nil,(n)->next) \
) \ ) \
) )
// ! MD_LINKED_LIST_PURE_MACRO
#endif #endif
// sll_queue_push_nz
#endif #endif
#ifndef sll_queue_push_front_nz #ifndef sll_queue_push_front_nz
@@ -589,6 +595,8 @@ sll__queue_push_front_nz(void* nil, void** f, void** l, void* n, void** n_next)
(f) = (n) \ (f) = (n) \
) \ ) \
) )
// ! MD_LINKED_LIST_PURE_MACRO
#endif
#endif #endif
#ifndef sll_queue_pop_nz #ifndef sll_queue_pop_nz
@@ -618,6 +626,9 @@ sll__queue_pop_nz(void* nil, void** f, void* f_next, void** l)
(f)=(f)->next \ (f)=(f)->next \
) \ ) \
) )
// ! MD_LINKED_LIST_PURE_MACRO
#endif
// sll_queue_pop_nz
#endif #endif
//- rjf: singly-linked, singly-headed lists (stacks) //- rjf: singly-linked, singly-headed lists (stacks)
code/base/memory_substrate.c
+18 -9
View File
@@ -17,25 +17,29 @@ struct _heap_stats
global _heap_stats _heap_stats_info; global _heap_stats _heap_stats_info;
void heap_stats_init( void ) void
heap_stats_init( void )
{ {
memory_zero_struct( &_heap_stats_info ); memory_zero_struct( &_heap_stats_info );
_heap_stats_info.magic = GEN_HEAP_STATS_MAGIC; _heap_stats_info.magic = GEN_HEAP_STATS_MAGIC;
} }
SSIZE heap_stats_used_memory( void ) SSIZE
heap_stats_used_memory( void )
{ {
assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" ); assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" );
return _heap_stats_info.used_memory; return _heap_stats_info.used_memory;
} }
SSIZE heap_stats_alloc_count( void ) SSIZE
heap_stats_alloc_count( void )
{ {
assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" ); assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" );
return _heap_stats_info.alloc_count; return _heap_stats_info.alloc_count;
} }
void heap_stats_check( void ) void
heap_stats_check( void )
{ {
assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" ); assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" );
assert( _heap_stats_info.used_memory == 0 ); assert( _heap_stats_info.used_memory == 0 );
@@ -50,7 +54,8 @@ struct _heap_alloc_info
}; };
#endif #endif
void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags ) void*
heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags )
{ {
void* ptr = nullptr; void* ptr = nullptr;
// unused( allocator_data ); // unused( allocator_data );
@@ -184,7 +189,8 @@ void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size,
return ptr; return ptr;
} }
VArena varena__alloc(VArenaParams params) VArena
varena__alloc(VArenaParams params)
{ {
if (params.reserve_size == 0) { if (params.reserve_size == 0) {
params.reserve_size = VARENA_DEFAULT_RESERVE; params.reserve_size = VARENA_DEFAULT_RESERVE;
@@ -238,13 +244,15 @@ VArena varena__alloc(VArenaParams params)
vm->flags = params.flags; vm->flags = params.flags;
} }
void varena_release(VArena* arena) void
varena_release(VArena* arena)
{ {
os_release(arena, arena->reserve); os_release(arena, arena->reserve);
arena = nullptr; arena = nullptr;
} }
void* varena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE requested_size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags) void*
varena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE requested_size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags)
{ {
OS_SystemInfo const* info = os_get_system_info(); OS_SystemInfo const* info = os_get_system_info();
@@ -372,7 +380,8 @@ void* varena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE requ
return allocated_mem; return allocated_mem;
} }
void* farena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags) void*
farena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags)
{ {
FArena* arena = rcast(FArena*, allocator_data); FArena* arena = rcast(FArena*, allocator_data);
code/base/memory_substrate.h
+19 -18
View File
@@ -106,6 +106,7 @@ void* resize_align( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size,
#ifndef alloc_array #ifndef alloc_array
// Allocate memory for an array of items. // Allocate memory for an array of items.
#define alloc_array( allocator_, Type, count ) (Type*)memory_zero(alloc( allocator_, size_of(Type) * (count) ), size_of(Type) * (count)) #define alloc_array( allocator_, Type, count ) (Type*)memory_zero(alloc( allocator_, size_of(Type) * (count) ), size_of(Type) * (count))
// Allocate memory for an array of items. (Don't zero initialize)
#define alloc_array_no_zero( allocator_, Type, count ) (Type*) alloc( allocator_, size_of(Type) * (count) ) #define alloc_array_no_zero( allocator_, Type, count ) (Type*) alloc( allocator_, size_of(Type) * (count) )
#endif #endif
@@ -221,40 +222,40 @@ MD_API void* farena_allocator_proc(void* allocator_data, AllocatorMode mode, SSI
// Inlines // Inlines
inline inline AllocatorType
AllocatorType allocator_type(AllocatorInfo a) { allocator_type(AllocatorInfo a) {
if (a.proc == nullptr) { if (a.proc == nullptr) {
a = default_allocator(); a = default_allocator();
} }
return (AllocatorType) a.proc(a.data, AllocatorMode_QueryType, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS); return (AllocatorType) a.proc(a.data, AllocatorMode_QueryType, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS);
} }
inline inline AllocatorQueryFlags
AllocatorQueryFlags allocator_query_support(AllocatorInfo a) { allocator_query_support(AllocatorInfo a) {
if (a.proc == nullptr) { if (a.proc == nullptr) {
a = default_allocator(); a = default_allocator();
} }
return (AllocatorType) a.proc(a.data, AllocatorMode_QuerySupport, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS); return (AllocatorType) a.proc(a.data, AllocatorMode_QuerySupport, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS);
} }
inline inline void*
void* alloc_align( AllocatorInfo a, SSIZE size, SSIZE alignment ) { alloc_align( AllocatorInfo a, SSIZE size, SSIZE alignment ) {
if (a.proc == nullptr) { if (a.proc == nullptr) {
a = default_allocator(); a = default_allocator();
} }
return a.proc( a.data, AllocatorMode_Alloc, size, alignment, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS ); return a.proc( a.data, AllocatorMode_Alloc, size, alignment, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS );
} }
inline inline void*
void* alloc( AllocatorInfo a, SSIZE size ) { alloc( AllocatorInfo a, SSIZE size ) {
if (a.proc == nullptr) { if (a.proc == nullptr) {
a = default_allocator(); a = default_allocator();
} }
return alloc_align( a, size, MD_DEFAULT_MEMORY_ALIGNMENT ); return alloc_align( a, size, MD_DEFAULT_MEMORY_ALIGNMENT );
} }
inline inline void
void alloc_free( AllocatorInfo a, void* ptr ) { alloc_free( AllocatorInfo a, void* ptr ) {
if (a.proc == nullptr) { if (a.proc == nullptr) {
a = default_allocator(); a = default_allocator();
} }
@@ -263,32 +264,32 @@ void alloc_free( AllocatorInfo a, void* ptr ) {
} }
} }
inline inline void
void free_all( AllocatorInfo a ) { free_all( AllocatorInfo a ) {
if (a.proc == nullptr) { if (a.proc == nullptr) {
a = default_allocator(); a = default_allocator();
} }
a.proc( a.data, AllocatorMode_FreeAll, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS ); a.proc( a.data, AllocatorMode_FreeAll, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS );
} }
inline inline void*
void* resize( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size ) { resize( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size ) {
if (a.proc == nullptr) { if (a.proc == nullptr) {
a = default_allocator(); a = default_allocator();
} }
return resize_align( a, ptr, old_size, new_size, MD_DEFAULT_ALLOCATOR_FLAGS ); return resize_align( a, ptr, old_size, new_size, MD_DEFAULT_ALLOCATOR_FLAGS );
} }
inline inline void*
void* resize_align( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size, SSIZE alignment ) { resize_align( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size, SSIZE alignment ) {
if (a.proc == nullptr) { if (a.proc == nullptr) {
a = default_allocator(); a = default_allocator();
} }
return a.proc( a.data, AllocatorMode_Resize, new_size, alignment, ptr, old_size, MD_DEFAULT_ALLOCATOR_FLAGS ); return a.proc( a.data, AllocatorMode_Resize, new_size, alignment, ptr, old_size, MD_DEFAULT_ALLOCATOR_FLAGS );
} }
inline inline void*
void* default_resize_align( AllocatorInfo a, void* old_memory, SSIZE old_size, SSIZE new_size, SSIZE alignment ) default_resize_align( AllocatorInfo a, void* old_memory, SSIZE old_size, SSIZE new_size, SSIZE alignment )
{ {
if ( ! old_memory ) if ( ! old_memory )
return alloc_align( a, new_size, alignment ); return alloc_align( a, new_size, alignment );
code/mdesk/mdesk.c
+75 -61
View File
@@ -314,6 +314,7 @@ tokenize_from_text(Arena* arena, String8 text)
} }
} }
} }
#undef is_whitspace
//- rjf: newlines //- rjf: newlines
if (token_flags == 0 && *byte == '\n') if (token_flags == 0 && *byte == '\n')
@@ -410,6 +411,7 @@ tokenize_from_text(Arena* arena, String8 text)
} }
} }
} }
#undef is_identifier
#define is_numeric(byte) ( \ #define is_numeric(byte) ( \
('0' <= *byte && *byte <= '9') || \ ('0' <= *byte && *byte <= '9') || \
@@ -440,6 +442,8 @@ tokenize_from_text(Arena* arena, String8 text)
} }
} }
} }
#undef is_numeric
#undef is_not_numeric
#define is_triple_string_literal(byte) ( \ #define is_triple_string_literal(byte) ( \
(byte[0] == '"' && byte[1] == '"' && byte[2] == '"' ) || \ (byte[0] == '"' && byte[1] == '"' && byte[2] == '"' ) || \
@@ -473,6 +477,7 @@ tokenize_from_text(Arena* arena, String8 text)
} }
} }
} }
#undef is_triple_string_literal
//- rjf: singlet string literals //- rjf: singlet string literals
if (token_flags == 0 && (byte[0] == '"' || byte[0] == '\'' || byte[0] == '`')) if (token_flags == 0 && (byte[0] == '"' || byte[0] == '\'' || byte[0] == '`'))
@@ -539,6 +544,7 @@ tokenize_from_text(Arena* arena, String8 text)
} }
} }
} }
#undef is_non_reserved_symbol
#define is_reserved_symbol(byte) ( \ #define is_reserved_symbol(byte) ( \
*byte == '{' || *byte == '}' || \ *byte == '{' || *byte == '}' || \
@@ -559,6 +565,7 @@ tokenize_from_text(Arena* arena, String8 text)
byte += 1; byte += 1;
} }
#undef is_reserved_symbol
//- rjf: bad characters in all other cases //- rjf: bad characters in all other cases
if (token_flags == 0) { if (token_flags == 0) {
@@ -598,11 +605,6 @@ tokenize_from_text(Arena* arena, String8 text)
} }
scratch_end(scratch); scratch_end(scratch);
return result; return result;
#undef byte_is_whitespace
#undef byte_is_identifier
#undef byte_is_numeric
#undef byte_is_not_numeric
} }
//////////////////////////////// ////////////////////////////////
@@ -631,21 +633,28 @@ struct ParseWorkNode
S32 counted_newlines; S32 counted_newlines;
}; };
#if 0 inline
void parser_work_push(work_kind, work_parent, work_free) void parse__work_push(ParseWorkKind work_kind, Node* work_parent, ParseWorkNode* work_top, ParseWorkNode* work_free, TempArena* scratch)
{ {
ParseWorkNode* work_node = work_free; ParseWorkNode* work_node = work_free;
if (work_node == 0) { if (work_node == 0) {
work_node = push_array(scratch.arena, MD_ParseWorkNode, 1); work_node = push_array(scratch->arena, ParseWorkNode, 1);
} }
else { else {
sll_stack_pop(work_free); sll_stack_pop(work_free);
} }
work_node->kind = (work_kind); work_node->kind = (work_kind);
work_node->parent = (work_parent); work_node->parent = (work_parent);
sll_stack_push(work_top, work_node); sll_stack_push(work_top, work_node);
} }
#endif
inline
void parse__work_pop(ParseWorkNode* work_top, ParseWorkNode* broken_work) {
sll_stack_pop(work_top);
if (work_top == 0) {
work_top = broken_work;
}
}
ParseResult ParseResult
parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray tokens) parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray tokens)
@@ -654,115 +663,104 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
//- rjf: set up outputs //- rjf: set up outputs
MsgList msgs = {0}; MsgList msgs = {0};
Node *root = push_node(arena, NodeKind_File, 0, filename, text, 0); Node* root = push_node(arena, NodeKind_File, 0, filename, text, 0);
//- rjf: set up parse rule stack //- rjf: set up parse rule stack
ParseWorkNode first_work = { 0, ParseWorkKind_Main, root };
ParseWorkNode first_work = { ParseWorkNode broken_work = { 0, ParseWorkKind_Main, root };
0,
ParseWorkKind_Main,
root,
};
ParseWorkNode broken_work = { 0, ParseWorkKind_Main, root,};
ParseWorkNode* work_top = &first_work; ParseWorkNode* work_top = &first_work;
ParseWorkNode* work_free = 0; ParseWorkNode* work_free = 0;
#define parser_work_push(work_kind, work_parent) do \ #define parse_work_push(work_kind, work_parent) parse__work_push(work_kind, work_parent, work_top, work_free, &scratch)
{ \ #define parse_work_pop() parse__work_pop (work_top, &broken_work)
ParseWorkNode* work_node = work_free; \
if (work_node == 0) { \
work_node = push_array(scratch.arena, MD_ParseWorkNode, 1); \
} \
else { \
sll_stack_pop(work_free); \
} \
work_node->kind = (work_kind); \
work_node->parent = (work_parent); \
sll_stack_push(work_top, work_node); \
} while(0)
#define parse_work_pop() do \
{ \
sll_stack_pop(work_top); \
if (work_top == 0) { \
work_top = &broken_work; \
} \
} while(0)
//- rjf: parse //- rjf: parse
Token* tokens_first = tokens.v; Token* tokens_first = tokens.v;
Token* tokens_opl = tokens_first + tokens.count; Token* tokens_opl = tokens_first + tokens.count;
Token* token = tokens_first; Token* token = tokens_first;
for(;token < tokens_opl;) for (;token < tokens_opl;)
{ {
//- rjf: unpack token //- rjf: unpack token
String8 token_string = str8_substr(text, token[0].range); String8 token_string = str8_substr(text, token[0].range);
// Note(Ed): Each of the the follwoing conditionals will always terminate the iteration path to end_consume label.
// TODO(Ed): Add opt-in support for whitespace awareness
//- rjf: whitespace -> always no-op & inc //- rjf: whitespace -> always no-op & inc
if (token->flags & TokenFlag_Whitespace) { if (token->flags & TokenFlag_Whitespace) {
token += 1; token += 1;
goto end_consume; goto end_consume;
} }
// TODO(Ed): Add opt-in support for comment awareness
//- rjf: comments -> always no-op & inc //- rjf: comments -> always no-op & inc
if (token->flags & TokenGroup_Comment) { if (token->flags & TokenGroup_Comment) {
token += 1; token += 1;
goto end_consume; goto end_consume;
} }
//- rjf: [node follow up] : following label -> work top parent has children. we need //- rjf: [node follow up] : following label -> work top parent has children.
// to scan for explicit delimiters, else parse an implicitly delimited set of children // we need to scan for explicit delimiters, else parse an implicitly delimited set of children
if (work_top->kind == ParseWorkKind_NodeOptionalFollowUp && str8_match(token_string, str8_lit(":"), 0)) { if (work_top->kind == ParseWorkKind_NodeOptionalFollowUp && str8_match(token_string, str8_lit(":"), 0)) {
Node* parent = work_top->parent; Node* parent = work_top->parent;
parse_work_pop(); parse_work_pop();
parse_work_push(ParseWorkKind_NodeChildrenStyleScan, parent); parse_work_push(ParseWorkKind_NodeChildrenStyleScan, parent);
token += 1; token += 1;
goto end_consume; goto end_consume;
// Transition to a state where we scan for // explicit delimiters (e.g., '{', '[', '(').
} }
//- rjf: [node follow up] anything but : following label -> node has no children. just //- rjf: [node follow up] anything but : following label -> node has no children.
// pop & move on // just pop & move on
if (work_top->kind == ParseWorkKind_NodeOptionalFollowUp) { if (work_top->kind == ParseWorkKind_NodeOptionalFollowUp) {
parse_work_pop(); parse_work_pop();
goto end_consume; goto end_consume;
} }
B32 reserved_token = token->flags & TokenFlag_Reserved;
B32 found_seprator = reserved_token && (str8_match(token_string, str8_lit(","), 0) || str8_match(token_string, str8_lit(";"), 0));
//- rjf: [main] separators -> mark & inc //- rjf: [main] separators -> mark & inc
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit(","), 0) || str8_match(token_string, str8_lit(";"), 0))) if (work_top->kind == ParseWorkKind_Main && found_seprator)
{ {
Node* parent = work_top->parent; Node* parent = work_top->parent;
if (!node_is_nil(parent->last)) if (!node_is_nil(parent->last))
{ {
parent->last->flags |= NodeFlag_IsBeforeComma *!! str8_match(token_string, str8_lit(","), 0); parent->last->flags |= NodeFlag_IsBeforeComma *!! str8_match(token_string, str8_lit(","), 0);
parent->last->flags |= NodeFlag_IsBeforeSemicolon *!! str8_match(token_string, str8_lit(";"), 0); parent->last->flags |= NodeFlag_IsBeforeSemicolon *!! str8_match(token_string, str8_lit(";"), 0);
work_top->gathered_node_flags |= NodeFlag_IsAfterComma *!! str8_match(token_string, str8_lit(","), 0); work_top->gathered_node_flags |= NodeFlag_IsAfterComma *!! str8_match(token_string, str8_lit(","), 0);
work_top->gathered_node_flags |= NodeFlag_IsAfterSemicolon *!! str8_match(token_string, str8_lit(";"), 0); work_top->gathered_node_flags |= NodeFlag_IsAfterSemicolon *!! str8_match(token_string, str8_lit(";"), 0);
} }
token += 1; token += 1;
goto end_consume; goto end_consume;
//
} }
//- rjf: [main_implicit] separators -> pop //- rjf: [main_implicit] separators -> pop
if(work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit(","), 0) || str8_match(token_string, str8_lit(";"), 0))) { if(work_top->kind == ParseWorkKind_MainImplicit && token->flags & found_seprator) {
parse_work_pop(); parse_work_pop();
goto end_consume; goto end_consume;
} }
B32 mode_main_and_implict = (work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit);
B32 found_unexpected = reserved_token && (str8_match(token_string, str8_lit("#"), 0) || str8_match(token_string, str8_lit("\\"), 0) || str8_match(token_string, str8_lit(":"), 0));
//- rjf: [main, main_implicit] unexpected reserved tokens //- rjf: [main, main_implicit] unexpected reserved tokens
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("#"), 0) || str8_match(token_string, str8_lit("\\"), 0) || str8_match(token_string, str8_lit(":"), 0))) if (mode_main_and_implict && found_unexpected) {
{
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min); Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
String8 error_string = push_str8f(arena, "Unexpected reserved symbol \"%S\".", token_string); String8 error_string = push_str8f(arena, "Unexpected reserved symbol \"%S\".", token_string);
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string); msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
token += 1; token += 1;
goto end_consume; goto end_consume;
} }
B32 found_tag = reserved_token && str8_match(token_string, str8_lit("@"), 0);
//- rjf: [main, main_implicit] tag signifier -> create new tag //- rjf: [main, main_implicit] tag signifier -> create new tag
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token[0].flags & TokenFlag_Reserved && str8_match(token_string, str8_lit("@"), 0)) if (mode_main_and_implict && found_tag)
{ {
if (token+1 >= tokens_opl || !(token[1].flags & TokenGroup_Label)) if (token + 1 >= tokens_opl || !(token[1].flags & TokenGroup_Label))
{ {
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min); Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
String8 error_string = str8_lit("Tag label expected after @ symbol."); String8 error_string = str8_lit("Tag label expected after @ symbol.");
@@ -791,7 +789,7 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
} }
//- rjf: [main, main_implicit] label -> create new main //- rjf: [main, main_implicit] label -> create new main
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token->flags & TokenGroup_Label) if (mode_main_and_implict && token->flags & TokenGroup_Label)
{ {
String8 node_string_raw = token_string; String8 node_string_raw = token_string;
String8 node_string = content_string_from_token_flags_str8(token->flags, node_string_raw); String8 node_string = content_string_from_token_flags_str8(token->flags, node_string_raw);
@@ -814,7 +812,7 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
} }
//- rjf: [main] {s, [s, and (s -> create new main //- rjf: [main] {s, [s, and (s -> create new main
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved && if (work_top->kind == ParseWorkKind_Main && reserved_token &&
( (
str8_match(token_string, str8_lit("{"), 0) || str8_match(token_string, str8_lit("{"), 0) ||
str8_match(token_string, str8_lit("["), 0) || str8_match(token_string, str8_lit("["), 0) ||
@@ -846,7 +844,7 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
} }
//- rjf: [node children style scan] {s, [s, and (s -> explicitly delimited children //- rjf: [node children style scan] {s, [s, and (s -> explicitly delimited children
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && token->flags & TokenFlag_Reserved && if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && reserved_token &&
( (
str8_match(token_string, str8_lit("{"), 0) || str8_match(token_string, str8_lit("{"), 0) ||
str8_match(token_string, str8_lit("["), 0) || str8_match(token_string, str8_lit("["), 0) ||
@@ -863,23 +861,25 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
token += 1; token += 1;
goto end_consume; goto end_consume;
} }
B32 newline_token = token->flags & TokenFlag_Newline;
//- rjf: [node children style scan] count newlines //- rjf: [node children style scan] count newlines
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && token->flags & TokenFlag_Newline) { if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && newline_token) {
work_top->counted_newlines += 1; work_top->counted_newlines += 1;
token += 1; token += 1;
goto end_consume; goto end_consume;
} }
//- rjf: [main_implicit] newline -> pop //- rjf: [main_implicit] newline -> pop
if (work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Newline) { if (work_top->kind == ParseWorkKind_MainImplicit && newline_token) {
parss_work_pop(); parss_work_pop();
token += 1; token += 1;
goto end_consume; goto end_consume;
} }
//- rjf: [all but main_implicit] newline -> no-op & inc //- rjf: [all but main_implicit] newline -> no-op & inc
if (work_top->kind != ParseWorkKind_MainImplicit && token->flags & TokenFlag_Newline) { if (work_top->kind != ParseWorkKind_MainImplicit && newline_token) {
token += 1; token += 1;
goto end_consume; goto end_consume;
} }
@@ -906,7 +906,14 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
} }
//- rjf: [main] }s, ]s, and )s -> pop //- rjf: [main] }s, ]s, and )s -> pop
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("}"), 0) || str8_match(token_string, str8_lit("]"), 0) || str8_match(token_string, str8_lit(")"), 0))) { if (work_top->kind == ParseWorkKind_Main && reserved_token &&
(
str8_match(token_string, str8_lit("}"), 0) ||
str8_match(token_string, str8_lit("]"), 0) ||
str8_match(token_string, str8_lit(")"), 0)
)
)
{
Node* parent = work_top->parent; Node* parent = work_top->parent;
parent->flags |= NodeFlag_HasBraceRight *!! str8_match(token_string, str8_lit("}"), 0); parent->flags |= NodeFlag_HasBraceRight *!! str8_match(token_string, str8_lit("}"), 0);
parent->flags |= NodeFlag_HasBracketRight *!! str8_match(token_string, str8_lit("]"), 0); parent->flags |= NodeFlag_HasBracketRight *!! str8_match(token_string, str8_lit("]"), 0);
@@ -917,7 +924,14 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
} }
//- rjf: [main implicit] }s, ]s, and )s -> pop without advancing //- rjf: [main implicit] }s, ]s, and )s -> pop without advancing
if(work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("}"), 0) || str8_match(token_string, str8_lit("]"), 0) || str8_match(token_string, str8_lit(")"), 0))) { if (work_top->kind == ParseWorkKind_MainImplicit && reserved_token &&
(
str8_match(token_string, str8_lit("}"), 0) ||
str8_match(token_string, str8_lit("]"), 0) ||
str8_match(token_string, str8_lit(")"), 0)
)
)
{
parse_work_pop(); parse_work_pop();
goto end_consume; goto end_consume;
} }
docs/mdesk.pur
BIN
View File
Binary file not shown.