ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-06-12 23:51:37 -07:00
Code Issues Packages Projects Releases Wiki Activity

adjustments to parse_from_text_tokens

Browse Source
This commit is contained in:
Edward R. Gonzalez
2025-02-08 18:08:27 -05:00
parent 2ef486b7a0
commit 843cc7fa90
6 changed files with 124 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Readme.md
-1
View File
@@ -18,4 +18,3 @@ docs will be updated referencing content procued by Ryan Fleury and content base
* [examples](./examples/Readme.md)
* [gen_c11](./gen_c11/Readme.md)
* [gen_cpp17](./gen_cpp17/Readme.md)
code/base/memory.h
+12 -1
View File
@@ -452,9 +452,11 @@ dll__insert_npz(
) \
) \
)
// ! MD_LINKED_LIST_PURE_MACRO
#endif
// dll_insert_npz
#endif
#endif
#ifndef dll_push_back_npz
// push-back next-previous with nil
#define dll_push_back_npz(nil, f, l, n, next, prev) dll_insert_npz(nil, f, l, l, n, next, prev)
@@ -517,7 +519,9 @@ dll__remove_npz(
: ((n)->next->prev = (n)->prev) \
) \
)
// ! MD_LINKED_LIST_PURE_MACRO
#endif
// dll_remove_npz
#endif
//- rjf: singly-linked, doubly-headed lists (queues)
@@ -558,7 +562,9 @@ sll__queue_push_nz(
set_nil(nil,(n)->next) \
) \
)
// ! MD_LINKED_LIST_PURE_MACRO
#endif
// sll_queue_push_nz
#endif
#ifndef sll_queue_push_front_nz
@@ -589,6 +595,8 @@ sll__queue_push_front_nz(void* nil, void** f, void** l, void* n, void** n_next)
(f) = (n) \
) \
)
// ! MD_LINKED_LIST_PURE_MACRO
#endif
#endif
#ifndef sll_queue_pop_nz
@@ -618,6 +626,9 @@ sll__queue_pop_nz(void* nil, void** f, void* f_next, void** l)
(f)=(f)->next \
) \
)
// ! MD_LINKED_LIST_PURE_MACRO
#endif
// sll_queue_pop_nz
#endif
//- rjf: singly-linked, singly-headed lists (stacks)
code/base/memory_substrate.c
+18 -9
View File
@@ -17,25 +17,29 @@ struct _heap_stats
global _heap_stats _heap_stats_info;
void heap_stats_init( void )
void
heap_stats_init( void )
{
memory_zero_struct( &_heap_stats_info );
_heap_stats_info.magic = GEN_HEAP_STATS_MAGIC;
}
SSIZE heap_stats_used_memory( void )
SSIZE
heap_stats_used_memory( void )
{
assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" );
return _heap_stats_info.used_memory;
}
SSIZE heap_stats_alloc_count( void )
SSIZE
heap_stats_alloc_count( void )
{
assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" );
return _heap_stats_info.alloc_count;
}
void heap_stats_check( void )
void
heap_stats_check( void )
{
assert_msg( _heap_stats_info.magic == GEN_HEAP_STATS_MAGIC, "heap_stats is not initialised yet, call heap_stats_init first!" );
assert( _heap_stats_info.used_memory == 0 );
@@ -50,7 +54,8 @@ struct _heap_alloc_info
};
#endif
void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags )
void*
heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags )
{
void* ptr = nullptr;
// unused( allocator_data );
@@ -184,7 +189,8 @@ void* heap_allocator_proc( void* allocator_data, AllocatorMode mode, SSIZE size,
return ptr;
}
VArena varena__alloc(VArenaParams params)
VArena
varena__alloc(VArenaParams params)
{
if (params.reserve_size == 0) {
params.reserve_size = VARENA_DEFAULT_RESERVE;
@@ -238,13 +244,15 @@ VArena varena__alloc(VArenaParams params)
vm->flags = params.flags;
}
void varena_release(VArena* arena)
void
varena_release(VArena* arena)
{
os_release(arena, arena->reserve);
arena = nullptr;
}
void* varena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE requested_size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags)
void*
varena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE requested_size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags)
{
OS_SystemInfo const* info = os_get_system_info();
@@ -372,7 +380,8 @@ void* varena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE requ
return allocated_mem;
}
void* farena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags)
void*
farena_allocator_proc(void* allocator_data, AllocatorMode mode, SSIZE size, SSIZE alignment, void* old_memory, SSIZE old_size, U64 flags)
{
FArena* arena = rcast(FArena*, allocator_data);
code/base/memory_substrate.h
+19 -18
View File
@@ -106,6 +106,7 @@ void* resize_align( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size,
#ifndef alloc_array
// Allocate memory for an array of items.
#define alloc_array( allocator_, Type, count ) (Type*)memory_zero(alloc( allocator_, size_of(Type) * (count) ), size_of(Type) * (count))
// Allocate memory for an array of items. (Don't zero initialize)
#define alloc_array_no_zero( allocator_, Type, count ) (Type*) alloc( allocator_, size_of(Type) * (count) )
#endif
@@ -221,40 +222,40 @@ MD_API void* farena_allocator_proc(void* allocator_data, AllocatorMode mode, SSI
// Inlines
inline
AllocatorType allocator_type(AllocatorInfo a) {
inline AllocatorType
allocator_type(AllocatorInfo a) {
if (a.proc == nullptr) {
a = default_allocator();
}
return (AllocatorType) a.proc(a.data, AllocatorMode_QueryType, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS);
}
inline
AllocatorQueryFlags allocator_query_support(AllocatorInfo a) {
inline AllocatorQueryFlags
allocator_query_support(AllocatorInfo a) {
if (a.proc == nullptr) {
a = default_allocator();
}
return (AllocatorType) a.proc(a.data, AllocatorMode_QuerySupport, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS);
}
inline
void* alloc_align( AllocatorInfo a, SSIZE size, SSIZE alignment ) {
inline void*
alloc_align( AllocatorInfo a, SSIZE size, SSIZE alignment ) {
if (a.proc == nullptr) {
a = default_allocator();
}
return a.proc( a.data, AllocatorMode_Alloc, size, alignment, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS );
}
inline
void* alloc( AllocatorInfo a, SSIZE size ) {
inline void*
alloc( AllocatorInfo a, SSIZE size ) {
if (a.proc == nullptr) {
a = default_allocator();
}
return alloc_align( a, size, MD_DEFAULT_MEMORY_ALIGNMENT );
}
inline
void alloc_free( AllocatorInfo a, void* ptr ) {
inline void
alloc_free( AllocatorInfo a, void* ptr ) {
if (a.proc == nullptr) {
a = default_allocator();
}
@@ -263,32 +264,32 @@ void alloc_free( AllocatorInfo a, void* ptr ) {
}
}
inline
void free_all( AllocatorInfo a ) {
inline void
free_all( AllocatorInfo a ) {
if (a.proc == nullptr) {
a = default_allocator();
}
a.proc( a.data, AllocatorMode_FreeAll, 0, 0, nullptr, 0, MD_DEFAULT_ALLOCATOR_FLAGS );
}
inline
void* resize( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size ) {
inline void*
resize( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size ) {
if (a.proc == nullptr) {
a = default_allocator();
}
return resize_align( a, ptr, old_size, new_size, MD_DEFAULT_ALLOCATOR_FLAGS );
}
inline
void* resize_align( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size, SSIZE alignment ) {
inline void*
resize_align( AllocatorInfo a, void* ptr, SSIZE old_size, SSIZE new_size, SSIZE alignment ) {
if (a.proc == nullptr) {
a = default_allocator();
}
return a.proc( a.data, AllocatorMode_Resize, new_size, alignment, ptr, old_size, MD_DEFAULT_ALLOCATOR_FLAGS );
}
inline
void* default_resize_align( AllocatorInfo a, void* old_memory, SSIZE old_size, SSIZE new_size, SSIZE alignment )
inline void*
default_resize_align( AllocatorInfo a, void* old_memory, SSIZE old_size, SSIZE new_size, SSIZE alignment )
{
if ( ! old_memory )
return alloc_align( a, new_size, alignment );
code/mdesk/mdesk.c
+75 -61
View File
@@ -314,6 +314,7 @@ tokenize_from_text(Arena* arena, String8 text)
}
}
}
#undef is_whitspace
//- rjf: newlines
if (token_flags == 0 && *byte == '\n')
@@ -410,6 +411,7 @@ tokenize_from_text(Arena* arena, String8 text)
}
}
}
#undef is_identifier
#define is_numeric(byte) ( \
('0' <= *byte && *byte <= '9') || \
@@ -440,6 +442,8 @@ tokenize_from_text(Arena* arena, String8 text)
}
}
}
#undef is_numeric
#undef is_not_numeric
#define is_triple_string_literal(byte) ( \
(byte[0] == '"' && byte[1] == '"' && byte[2] == '"' ) || \
@@ -473,6 +477,7 @@ tokenize_from_text(Arena* arena, String8 text)
}
}
}
#undef is_triple_string_literal
//- rjf: singlet string literals
if (token_flags == 0 && (byte[0] == '"' || byte[0] == '\'' || byte[0] == '`'))
@@ -539,6 +544,7 @@ tokenize_from_text(Arena* arena, String8 text)
}
}
}
#undef is_non_reserved_symbol
#define is_reserved_symbol(byte) ( \
*byte == '{' || *byte == '}' || \
@@ -559,6 +565,7 @@ tokenize_from_text(Arena* arena, String8 text)
byte += 1;
}
#undef is_reserved_symbol
//- rjf: bad characters in all other cases
if (token_flags == 0) {
@@ -598,11 +605,6 @@ tokenize_from_text(Arena* arena, String8 text)
}
scratch_end(scratch);
return result;
#undef byte_is_whitespace
#undef byte_is_identifier
#undef byte_is_numeric
#undef byte_is_not_numeric
}
////////////////////////////////
@@ -631,21 +633,28 @@ struct ParseWorkNode
S32 counted_newlines;
};
#if 0
void parser_work_push(work_kind, work_parent, work_free)
inline
void parse__work_push(ParseWorkKind work_kind, Node* work_parent, ParseWorkNode* work_top, ParseWorkNode* work_free, TempArena* scratch)
{
ParseWorkNode* work_node = work_free;
if (work_node == 0) {
work_node = push_array(scratch.arena, MD_ParseWorkNode, 1);
work_node = push_array(scratch->arena, ParseWorkNode, 1);
}
else {
sll_stack_pop(work_free);
}
work_node->kind = (work_kind);
work_node->kind = (work_kind);
work_node->parent = (work_parent);
sll_stack_push(work_top, work_node);
}
#endif
inline
void parse__work_pop(ParseWorkNode* work_top, ParseWorkNode* broken_work) {
sll_stack_pop(work_top);
if (work_top == 0) {
work_top = broken_work;
}
}
ParseResult
parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray tokens)
@@ -654,115 +663,104 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
//- rjf: set up outputs
MsgList msgs = {0};
Node *root = push_node(arena, NodeKind_File, 0, filename, text, 0);
Node* root = push_node(arena, NodeKind_File, 0, filename, text, 0);
//- rjf: set up parse rule stack
ParseWorkNode first_work = {
0,
ParseWorkKind_Main,
root,
};
ParseWorkNode broken_work = { 0, ParseWorkKind_Main, root,};
ParseWorkNode first_work = { 0, ParseWorkKind_Main, root };
ParseWorkNode broken_work = { 0, ParseWorkKind_Main, root };
ParseWorkNode* work_top = &first_work;
ParseWorkNode* work_free = 0;
#define parser_work_push(work_kind, work_parent) do \
{ \
ParseWorkNode* work_node = work_free; \
if (work_node == 0) { \
work_node = push_array(scratch.arena, MD_ParseWorkNode, 1); \
} \
else { \
sll_stack_pop(work_free); \
} \
work_node->kind = (work_kind); \
work_node->parent = (work_parent); \
sll_stack_push(work_top, work_node); \
} while(0)
#define parse_work_pop() do \
{ \
sll_stack_pop(work_top); \
if (work_top == 0) { \
work_top = &broken_work; \
} \
} while(0)
#define parse_work_push(work_kind, work_parent) parse__work_push(work_kind, work_parent, work_top, work_free, &scratch)
#define parse_work_pop() parse__work_pop (work_top, &broken_work)
//- rjf: parse
Token* tokens_first = tokens.v;
Token* tokens_opl = tokens_first + tokens.count;
Token* token = tokens_first;
for(;token < tokens_opl;)
for (;token < tokens_opl;)
{
//- rjf: unpack token
String8 token_string = str8_substr(text, token[0].range);
// Note(Ed): Each of the the follwoing conditionals will always terminate the iteration path to end_consume label.
// TODO(Ed): Add opt-in support for whitespace awareness
//- rjf: whitespace -> always no-op & inc
if (token->flags & TokenFlag_Whitespace) {
token += 1;
goto end_consume;
}
// TODO(Ed): Add opt-in support for comment awareness
//- rjf: comments -> always no-op & inc
if (token->flags & TokenGroup_Comment) {
token += 1;
goto end_consume;
}
//- rjf: [node follow up] : following label -> work top parent has children. we need
// to scan for explicit delimiters, else parse an implicitly delimited set of children
//- rjf: [node follow up] : following label -> work top parent has children.
// we need to scan for explicit delimiters, else parse an implicitly delimited set of children
if (work_top->kind == ParseWorkKind_NodeOptionalFollowUp && str8_match(token_string, str8_lit(":"), 0)) {
Node* parent = work_top->parent;
parse_work_pop();
parse_work_push(ParseWorkKind_NodeChildrenStyleScan, parent);
token += 1;
goto end_consume;
// Transition to a state where we scan for // explicit delimiters (e.g., '{', '[', '(').
}
//- rjf: [node follow up] anything but : following label -> node has no children. just
// pop & move on
//- rjf: [node follow up] anything but : following label -> node has no children.
// just pop & move on
if (work_top->kind == ParseWorkKind_NodeOptionalFollowUp) {
parse_work_pop();
goto end_consume;
}
B32 reserved_token = token->flags & TokenFlag_Reserved;
B32 found_seprator = reserved_token && (str8_match(token_string, str8_lit(","), 0) || str8_match(token_string, str8_lit(";"), 0));
//- rjf: [main] separators -> mark & inc
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit(","), 0) || str8_match(token_string, str8_lit(";"), 0)))
if (work_top->kind == ParseWorkKind_Main && found_seprator)
{
Node* parent = work_top->parent;
if (!node_is_nil(parent->last))
{
parent->last->flags |= NodeFlag_IsBeforeComma *!! str8_match(token_string, str8_lit(","), 0);
parent->last->flags |= NodeFlag_IsBeforeComma *!! str8_match(token_string, str8_lit(","), 0);
parent->last->flags |= NodeFlag_IsBeforeSemicolon *!! str8_match(token_string, str8_lit(";"), 0);
work_top->gathered_node_flags |= NodeFlag_IsAfterComma *!! str8_match(token_string, str8_lit(","), 0);
work_top->gathered_node_flags |= NodeFlag_IsAfterComma *!! str8_match(token_string, str8_lit(","), 0);
work_top->gathered_node_flags |= NodeFlag_IsAfterSemicolon *!! str8_match(token_string, str8_lit(";"), 0);
}
token += 1;
goto end_consume;
//
}
//- rjf: [main_implicit] separators -> pop
if(work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit(","), 0) || str8_match(token_string, str8_lit(";"), 0))) {
if(work_top->kind == ParseWorkKind_MainImplicit && token->flags & found_seprator) {
parse_work_pop();
goto end_consume;
}
B32 mode_main_and_implict = (work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit);
B32 found_unexpected = reserved_token && (str8_match(token_string, str8_lit("#"), 0) || str8_match(token_string, str8_lit("\\"), 0) || str8_match(token_string, str8_lit(":"), 0));
//- rjf: [main, main_implicit] unexpected reserved tokens
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("#"), 0) || str8_match(token_string, str8_lit("\\"), 0) || str8_match(token_string, str8_lit(":"), 0)))
{
if (mode_main_and_implict && found_unexpected) {
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
String8 error_string = push_str8f(arena, "Unexpected reserved symbol \"%S\".", token_string);
msg_list_push(arena, &msgs, error, MsgKind_Error, error_string);
token += 1;
goto end_consume;
}
B32 found_tag = reserved_token && str8_match(token_string, str8_lit("@"), 0);
//- rjf: [main, main_implicit] tag signifier -> create new tag
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token[0].flags & TokenFlag_Reserved && str8_match(token_string, str8_lit("@"), 0))
if (mode_main_and_implict && found_tag)
{
if (token+1 >= tokens_opl || !(token[1].flags & TokenGroup_Label))
if (token + 1 >= tokens_opl || !(token[1].flags & TokenGroup_Label))
{
Node* error = push_node(arena, NodeKind_ErrorMarker, 0, token_string, token_string, token->range.min);
String8 error_string = str8_lit("Tag label expected after @ symbol.");
@@ -791,7 +789,7 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
}
//- rjf: [main, main_implicit] label -> create new main
if ((work_top->kind == ParseWorkKind_Main || work_top->kind == ParseWorkKind_MainImplicit) && token->flags & TokenGroup_Label)
if (mode_main_and_implict && token->flags & TokenGroup_Label)
{
String8 node_string_raw = token_string;
String8 node_string = content_string_from_token_flags_str8(token->flags, node_string_raw);
@@ -814,7 +812,7 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
}
//- rjf: [main] {s, [s, and (s -> create new main
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved &&
if (work_top->kind == ParseWorkKind_Main && reserved_token &&
(
str8_match(token_string, str8_lit("{"), 0) ||
str8_match(token_string, str8_lit("["), 0) ||
@@ -846,7 +844,7 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
}
//- rjf: [node children style scan] {s, [s, and (s -> explicitly delimited children
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && token->flags & TokenFlag_Reserved &&
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && reserved_token &&
(
str8_match(token_string, str8_lit("{"), 0) ||
str8_match(token_string, str8_lit("["), 0) ||
@@ -863,23 +861,25 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
token += 1;
goto end_consume;
}
B32 newline_token = token->flags & TokenFlag_Newline;
//- rjf: [node children style scan] count newlines
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && token->flags & TokenFlag_Newline) {
if (work_top->kind == ParseWorkKind_NodeChildrenStyleScan && newline_token) {
work_top->counted_newlines += 1;
token += 1;
goto end_consume;
}
//- rjf: [main_implicit] newline -> pop
if (work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Newline) {
if (work_top->kind == ParseWorkKind_MainImplicit && newline_token) {
parss_work_pop();
token += 1;
goto end_consume;
}
//- rjf: [all but main_implicit] newline -> no-op & inc
if (work_top->kind != ParseWorkKind_MainImplicit && token->flags & TokenFlag_Newline) {
if (work_top->kind != ParseWorkKind_MainImplicit && newline_token) {
token += 1;
goto end_consume;
}
@@ -906,7 +906,14 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
}
//- rjf: [main] }s, ]s, and )s -> pop
if (work_top->kind == ParseWorkKind_Main && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("}"), 0) || str8_match(token_string, str8_lit("]"), 0) || str8_match(token_string, str8_lit(")"), 0))) {
if (work_top->kind == ParseWorkKind_Main && reserved_token &&
(
str8_match(token_string, str8_lit("}"), 0) ||
str8_match(token_string, str8_lit("]"), 0) ||
str8_match(token_string, str8_lit(")"), 0)
)
)
{
Node* parent = work_top->parent;
parent->flags |= NodeFlag_HasBraceRight *!! str8_match(token_string, str8_lit("}"), 0);
parent->flags |= NodeFlag_HasBracketRight *!! str8_match(token_string, str8_lit("]"), 0);
@@ -917,7 +924,14 @@ parse_from_text_tokens(Arena* arena, String8 filename, String8 text, TokenArray
}
//- rjf: [main implicit] }s, ]s, and )s -> pop without advancing
if(work_top->kind == ParseWorkKind_MainImplicit && token->flags & TokenFlag_Reserved && (str8_match(token_string, str8_lit("}"), 0) || str8_match(token_string, str8_lit("]"), 0) || str8_match(token_string, str8_lit(")"), 0))) {
if (work_top->kind == ParseWorkKind_MainImplicit && reserved_token &&
(
str8_match(token_string, str8_lit("}"), 0) ||
str8_match(token_string, str8_lit("]"), 0) ||
str8_match(token_string, str8_lit(")"), 0)
)
)
{
parse_work_pop();
goto end_consume;
}
docs/mdesk.pur
BIN
View File
Binary file not shown.