From 204af74b319b2e02b09bc143ee5294e657a774b2 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 8 Oct 2021 15:25:58 -0700 Subject: [PATCH 01/28] [examples] starting notes in overrides example; start cleaning up overrides --- examples/advanced_usages/overrides.c | 32 +++++++++++++++++++++++++--- source/md.c | 6 ++++++ source/md.h | 18 +++++++++++----- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/examples/advanced_usages/overrides.c b/examples/advanced_usages/overrides.c index cf72b30..00a8acb 100644 --- a/examples/advanced_usages/overrides.c +++ b/examples/advanced_usages/overrides.c @@ -1,13 +1,28 @@ /* ** Example: overrides ** -** TODO full commentary +** This example shows using the overrides system in Metadesk to plug in a +** custom memory allocator and file loading routine. There are more options +** in the overrides than are presented here. A full list of the overrides +** options is kept in md.c 'Overrides & Options Macros' +** +** A few of the reasons one might want to use the Metadesk overrides are: +** 1. Plugging in a custom allocator to control the memory allocations +** 2. Plugging in a custom arena implementation for more seamless +** interoperation between the codebase and library +** 3. Provide implementation for unsupported OSes without having to modify + ** md.h or md.c +** 4. Remove dependency on CRT +** 5. Remove dependency on OS headers ** */ - //~ example allocator ///////////////////////////////////////////////////////// +// @notes This isn't really "the example" but we need something to play the +// role of a custom allocator, imagine this is any alloc & free style +// allocator you might already have in a codebase. + typedef struct ExampleAllocatorNode{ struct ExampleAllocatorNode *next; struct ExampleAllocatorNode *prev; @@ -25,6 +40,11 @@ void examp_free(ExampleAllocator *a, void *ptr); //~ include metadesk header /////////////////////////////////////////////////// +// @notes We include the metadesk header before we define the overrides because +// some overrides require that metadesk base types be visible. There are +// exceptions to this pattern, in particular overrides for types need to be +// defined before including md.h, we aren't going that far here. + #include "md.h" @@ -32,6 +52,12 @@ void examp_free(ExampleAllocator *a, void *ptr); // override memory to use malloc/free +// @notes A common practice in setting up allocator overrides is to use a pass +// through opaque user context pointer. We took a look at that for *long time* +// and just really don't like the amount of baggage it adds to the entire API. +// Instead we recommend handling the context pointer with a global for single +// threaded use cases, or with a pointer in thread local storage for +// multi-threaded cases. ExampleAllocator* md_example_allocator = 0; void* md_reserve_by_example_allocator(unsigned long long size); @@ -40,7 +66,7 @@ void md_release_by_example_allocator(void *ptr, unsigned long long ignore); #define MD_IMPL_Reserve md_reserve_by_example_allocator #define MD_IMPL_Commit(p,z) (1) #define MD_IMPL_Decommit(p,z) ((void)0) -#define MD_IMPL_Release(p,z) md_release_by_example_allocator +#define MD_IMPL_Release md_release_by_example_allocator // override file loading diff --git a/source/md.c b/source/md.c index 6b19270..defa30c 100644 --- a/source/md.c +++ b/source/md.c @@ -4,6 +4,11 @@ ** Overrides & Options Macros ** ** Overridable +** "basic types" ** REQUIRED +** #define/typedef MD_i8, MD_i16, MD_i32, MD_i64 +** #define/typedef MD_u8, MD_u16, MD_u32, MD_u64 +** #define/typedef MD_f32, MD_f64 +** ** "memset" ** REQUIRED ** #define MD_IMPL_Memset (void*, int, uint64) -> void* ** #define MD_IMPL_Memmove (void*, void*, uint64) -> void* @@ -42,6 +47,7 @@ ** ** Default Implementation Controls ** These controls default to '1' i.e. 'enabled' +** #define MD_DEFAULT_BASIC_TYPES -> construct "basic types" from stdint.h header ** #define MD_DEFAULT_MEMSET -> construct "memset" from CRT ** #define MD_DEFAULT_FILE_ITER -> construct "file iteration" from OS headers ** #define MD_DEFAULT_MEMORY -> construct "low level memory" from OS headers diff --git a/source/md.h b/source/md.h index 5d158d2..970eb32 100644 --- a/source/md.h +++ b/source/md.h @@ -29,6 +29,9 @@ #define MD_VERSION_MIN 0 //~ Set default values for controls +#if !defined(MD_DEFAULT_BASIC_TYPES) +# define MD_DEFAULT_BASIC_TYPES 1 +#endif #if !defined(MD_DEFAULT_MEMSET) # define MD_DEFAULT_MEMSET 1 #endif @@ -374,9 +377,11 @@ //~ Basic Types -#include #include +#if defined(MD_DEFAULT_BASIC_TYPES) + +#include typedef int8_t MD_i8; typedef int16_t MD_i16; typedef int32_t MD_i32; @@ -385,13 +390,16 @@ typedef uint8_t MD_u8; typedef uint16_t MD_u16; typedef uint32_t MD_u32; typedef uint64_t MD_u64; -typedef int8_t MD_b8; -typedef int16_t MD_b16; -typedef int32_t MD_b32; -typedef int64_t MD_b64; typedef float MD_f32; typedef double MD_f64; +#endif + +typedef MD_i8 MD_b8; +typedef MD_i16 MD_b16; +typedef MD_i32 MD_b32; +typedef MD_i64 MD_b64; + //~ Default Arena #if MD_DEFAULT_ARENA From feee8cf74519276dce12dc0746a4ed0f7033cf24 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 8 Oct 2021 15:32:16 -0700 Subject: [PATCH 02/28] [examples] finish overrides touch ups; tweak overrides example --- examples/advanced_usages/overrides.c | 3 +++ source/md.c | 31 ++++++++++++++++------------ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/examples/advanced_usages/overrides.c b/examples/advanced_usages/overrides.c index 00a8acb..b845b25 100644 --- a/examples/advanced_usages/overrides.c +++ b/examples/advanced_usages/overrides.c @@ -68,6 +68,9 @@ void md_release_by_example_allocator(void *ptr, unsigned long long ignore); #define MD_IMPL_Decommit(p,z) ((void)0) #define MD_IMPL_Release md_release_by_example_allocator +#define MD_DEFAULT_ARENA_RES_SIZE (1 << 20) +#define MD_DEFAULT_ARENA_CMT_SIZE (1 << 20) + // override file loading diff --git a/source/md.c b/source/md.c index defa30c..84f1da2 100644 --- a/source/md.c +++ b/source/md.c @@ -27,15 +27,16 @@ ** #define MD_IMPL_Decommit (void*, uint64) -> void ** #define MD_IMPL_Release (void*, uint64) -> void ** +** ** "arena" ** REQUIRED ** #define MD_IMPL_Arena (must set before including md.h) +** #define MD_IMPL_ArenaMinPos uint64 ** #define MD_IMPL_ArenaAlloc () -> MD_IMPL_Arena* ** #define MD_IMPL_ArenaRelease (MD_IMPL_Arena*) -> void ** #define MD_IMPL_ArenaGetPos (MD_IMPL_Arena*) -> uint64 ** #define MD_IMPL_ArenaPush (MD_IMPL_Arena*, uint64) -> void* ** #define MD_IMPL_ArenaPopTo (MD_IMPL_Arena*, uint64) -> void ** #define MD_IMPL_ArenaSetAutoAlign (MD_IMPL_Arena*, uint64) -> void -** #define MD_IMPL_ArenaHeaderSize uint64 ** ** "scratch" ** REQUIRED ** #define MD_IMPL_GetScratch (MD_IMPL_Arena**, uint64) -> MD_IMPL_Arena* @@ -45,6 +46,10 @@ ** "sprintf" ** REQUIRED ** #define MD_IMPL_Vsnprintf (char*, uint64, char const*, va_list) -> uint64 ** +** Static Parameters to the Default Arena Implementation +** #define MD_DEFAULT_ARENA_RES_SIZE uint64 [default 64 megabytes] +** #define MD_DEFAULT_ARENA_CMT_SIZE uint64 [default 64 kilabytes] +** ** Default Implementation Controls ** These controls default to '1' i.e. 'enabled' ** #define MD_DEFAULT_BASIC_TYPES -> construct "basic types" from stdint.h header @@ -413,7 +418,7 @@ MD_LINUX_Release(void *ptr, MD_u64 size) # define MD_DEFAULT_ARENA_CMT_SIZE (64 << 10) #endif -#define MD_DEFAULT_ARENA_VERY_BIG (MD_DEFAULT_ARENA_RES_SIZE - MD_IMPL_ArenaHeaderSize)/2 +#define MD_DEFAULT_ARENA_VERY_BIG (MD_DEFAULT_ARENA_RES_SIZE - MD_IMPL_ArenaMinPos)/2 //- "low level memory" implementation check #if !defined(MD_IMPL_Reserve) @@ -429,8 +434,8 @@ MD_LINUX_Release(void *ptr, MD_u64 size) # error Missing implementation for MD_IMPL_Release #endif -#define MD_IMPL_ArenaHeaderSize 64 -MD_StaticAssert(sizeof(MD_ArenaDefault) <= MD_IMPL_ArenaHeaderSize, arena_def_size_check); +#define MD_IMPL_ArenaMinPos 64 +MD_StaticAssert(sizeof(MD_ArenaDefault) <= MD_IMPL_ArenaMinPos, arena_def_size_check); #define MD_IMPL_ArenaAlloc MD_ArenaDefaultAlloc #define MD_IMPL_ArenaRelease MD_ArenaDefaultRelease @@ -442,7 +447,7 @@ MD_StaticAssert(sizeof(MD_ArenaDefault) <= MD_IMPL_ArenaHeaderSize, arena_def_si static MD_ArenaDefault* MD_ArenaDefaultAlloc__Size(MD_u64 cmt, MD_u64 res) { - MD_Assert(MD_IMPL_ArenaHeaderSize < cmt && cmt <= res); + MD_Assert(MD_IMPL_ArenaMinPos < cmt && cmt <= res); MD_u64 cmt_clamped = MD_ClampTop(cmt, res); MD_ArenaDefault *result = 0; void *mem = MD_IMPL_Reserve(res); @@ -452,7 +457,7 @@ MD_ArenaDefaultAlloc__Size(MD_u64 cmt, MD_u64 res) result->prev = 0; result->current = result; result->base_pos = 0; - result->pos = MD_IMPL_ArenaHeaderSize; + result->pos = MD_IMPL_ArenaMinPos; result->cmt = cmt_clamped; result->cap = res; result->align = 8; @@ -512,7 +517,7 @@ MD_ArenaDefaultPush(MD_ArenaDefault *arena, MD_u64 size) MD_ArenaDefault *new_arena = 0; if (size > MD_DEFAULT_ARENA_VERY_BIG) { - MD_u64 big_size_unrounded = size + MD_IMPL_ArenaHeaderSize; + MD_u64 big_size_unrounded = size + MD_IMPL_ArenaMinPos; MD_u64 big_size = MD_AlignPow2(big_size_unrounded, (4 << 10)); new_arena = MD_ArenaDefaultAlloc__Size(big_size, big_size); } @@ -564,7 +569,7 @@ static void MD_ArenaDefaultPopTo(MD_ArenaDefault *arena, MD_u64 pos) { // pop chunks in the chain - MD_u64 pos_clamped = MD_ClampBot(MD_IMPL_ArenaHeaderSize, pos); + MD_u64 pos_clamped = MD_ClampBot(MD_IMPL_ArenaMinPos, pos); { MD_ArenaDefault *node = arena->current; for (MD_ArenaDefault *prev = 0; @@ -581,7 +586,7 @@ MD_ArenaDefaultPopTo(MD_ArenaDefault *arena, MD_u64 pos) { MD_ArenaDefault *current = arena->current; MD_u64 local_pos_unclamped = pos - current->base_pos; - MD_u64 local_pos = MD_ClampBot(local_pos_unclamped, MD_IMPL_ArenaHeaderSize); + MD_u64 local_pos = MD_ClampBot(local_pos_unclamped, MD_IMPL_ArenaMinPos); current->pos = local_pos; } } @@ -613,8 +618,8 @@ MD_ArenaDefaultSetAutoAlign(MD_ArenaDefault *arena, MD_u64 align) #if !defined(MD_IMPL_ArenaSetAutoAlign) # error Missing implementation for MD_IMPL_ArenaSetAutoAlign #endif -#if !defined(MD_IMPL_ArenaHeaderSize) -# error Missing implementation for MD_IMPL_ArenaHeaderSize +#if !defined(MD_IMPL_ArenaMinPos) +# error Missing implementation for MD_IMPL_ArenaMinPos #endif //~///////////////////////////////////////////////////////////////////////////// @@ -728,7 +733,7 @@ MD_ArenaPutBack(MD_Arena *arena, MD_u64 size) { MD_u64 pos = MD_IMPL_ArenaGetPos(arena); MD_u64 new_pos = pos - size; - MD_u64 new_pos_clamped = MD_ClampBot(MD_IMPL_ArenaHeaderSize, new_pos); + MD_u64 new_pos_clamped = MD_ClampBot(MD_IMPL_ArenaMinPos, new_pos); MD_IMPL_ArenaPopTo(arena, new_pos_clamped); } @@ -754,7 +759,7 @@ MD_ArenaPushAlign(MD_Arena *arena, MD_u64 boundary) MD_FUNCTION void MD_ArenaClear(MD_Arena *arena) { - MD_IMPL_ArenaPopTo(arena, MD_IMPL_ArenaHeaderSize); + MD_IMPL_ArenaPopTo(arena, MD_IMPL_ArenaMinPos); } MD_FUNCTION MD_ArenaTemp From e5e65e472488d278d1b08810e5e7a0a25178971d Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 8 Oct 2021 15:34:43 -0700 Subject: [PATCH 03/28] [maintenance] cleaning up old code, sticking to 'data desk' name, shrinking name of 'advanced_usages' --- bin/build_examples.sh | 10 +- .../overrides.c | 0 ...e_template.c => data_desk_like_template.c} | 2 +- examples_todo.txt | 2 +- source/md_c_helpers.c | 722 ------------------ source/md_c_helpers.h | 131 ---- 6 files changed, 7 insertions(+), 860 deletions(-) rename examples/{advanced_usages => integration}/overrides.c (100%) rename examples/intro/{datadesk_like_template.c => data_desk_like_template.c} (97%) delete mode 100644 source/md_c_helpers.c delete mode 100644 source/md_c_helpers.h diff --git a/bin/build_examples.sh b/bin/build_examples.sh index d8e3e10..92eee3c 100755 --- a/bin/build_examples.sh +++ b/bin/build_examples.sh @@ -11,17 +11,17 @@ bin/bld_core.sh show_ctx examps="examples" -bin/bld_core.sh unit hello_world $examps/intro/hello_world.c -bin/bld_core.sh unit parse_check $examps/intro/parse_check.c -bin/bld_core.sh unit datadesk_like $examps/intro/datadesk_like_template.c -bin/bld_core.sh unit user_errors $examps/user_errors/user_errors.c +bin/bld_core.sh unit hello_world $examps/intro/hello_world.c +bin/bld_core.sh unit parse_check $examps/intro/parse_check.c +bin/bld_core.sh unit data_desk_like $examps/intro/data_desk_like_template.c +bin/bld_core.sh unit user_errors $examps/user_errors/user_errors.c bin/bld_core.sh unit type_metadata $examps/type_metadata/type_metadata.c if [ -d $examps/type_metadata/generated/type_info_meta.h ]; then bin/bld_core.sh unit type_info $examps/type_metadata/type_info_final_program.c fi -bin/bld_core.sh unit overrides $examps/advanced_usages/overrides.c +bin/bld_core.sh unit overrides $examps/integration/overrides.c echo diff --git a/examples/advanced_usages/overrides.c b/examples/integration/overrides.c similarity index 100% rename from examples/advanced_usages/overrides.c rename to examples/integration/overrides.c diff --git a/examples/intro/datadesk_like_template.c b/examples/intro/data_desk_like_template.c similarity index 97% rename from examples/intro/datadesk_like_template.c rename to examples/intro/data_desk_like_template.c index e11be85..3f32309 100644 --- a/examples/intro/datadesk_like_template.c +++ b/examples/intro/data_desk_like_template.c @@ -2,7 +2,7 @@ ** Example: data desk like template ** ** This example is setup as a copy-pastable template for creating metadesk -** based metaprograms that have the same structure as datadesk metaprograms. +** based metaprograms that have the same structure as data desk metaprograms. ** ** Data Desk was a precursor language to metadesk. This example is mostly meant ** to help Data Desk users understand metadesk and migrate onto it. diff --git a/examples_todo.txt b/examples_todo.txt index f232eb2..202b81e 100644 --- a/examples_todo.txt +++ b/examples_todo.txt @@ -2,7 +2,7 @@ Example Programs: [x] Metadesk hello world [x] Metadesk parse checker [x] User error checking -[x] Datadesk-like setup +[x] Data Desk like setup [ ] Example type metadata [x] Cleanup & Simplification Pass [ ] Commentary diff --git a/source/md_c_helpers.c b/source/md_c_helpers.c deleted file mode 100644 index c431cce..0000000 --- a/source/md_c_helpers.c +++ /dev/null @@ -1,722 +0,0 @@ -//~ Expression and Type-Expression Helper - -MD_GLOBAL MD_C_Expr _md_nil_expr = -{ - &_md_nil_node, - MD_C_ExprKind_Nil, - &_md_nil_expr, - {&_md_nil_expr, &_md_nil_expr }, -}; - -MD_FUNCTION_IMPL MD_C_Expr * -MD_C_NilExpr(void) -{ - return &_md_nil_expr; -} - -MD_FUNCTION_IMPL MD_b32 -MD_C_ExprIsNil(MD_C_Expr *expr) -{ - return expr == 0 || expr == &_md_nil_expr || expr->kind == MD_C_ExprKind_Nil; -} - -typedef struct _MD_C_ExprKindMetadata _MD_C_ExprKindMetadata; -struct _MD_C_ExprKindMetadata -{ - MD_C_ExprKindGroup group; - MD_C_ExprPrec prec; - char *symbol; - char *pre_symbol; - char *post_symbol; -}; - -MD_FUNCTION_IMPL _MD_C_ExprKindMetadata * -_MD_MetadataFromExprKind(MD_C_ExprKind kind) -{ - // 0: Invalid - // 12: (unary) - ~ ! - // 11: . -> () [] - // 10: * / % - // 9: + - - // 8: << >> - // 7: < <= > >= - // 6: == != - // 5: (bitwise) & - // 4: ^ - // 3: | - // 2: && - // 1: || - static _MD_C_ExprKindMetadata metadata[] = - { - {MD_C_ExprKindGroup_Nil, +0, "NIL", "", "" }, // MD_C_ExprKind_Nil - {MD_C_ExprKindGroup_Atom, +0, "NIL", "", "" }, // MD_C_ExprKind_Atom - {MD_C_ExprKindGroup_Binary, +11, ".", "", "" }, // MD_C_ExprKind_Dot - {MD_C_ExprKindGroup_Binary, +11, "->", "", "" }, // MD_C_ExprKind_Arrow - {MD_C_ExprKindGroup_PostUnary, +11, "", "(", ")"}, // MD_C_ExprKind_Call - {MD_C_ExprKindGroup_PostUnary, +11, "", "[", "]"}, // MD_C_ExprKind_Subscript - {MD_C_ExprKindGroup_PreUnary, +12, "", "*", "" }, // MD_C_ExprKind_Dereference - {MD_C_ExprKindGroup_PreUnary, +12, "", "&", "" }, // MD_C_ExprKind_Reference - {MD_C_ExprKindGroup_Binary, +9, "+", "", "" }, // MD_C_ExprKind_Add - {MD_C_ExprKindGroup_Binary, +9, "-", "", "" }, // MD_C_ExprKind_Subtract - {MD_C_ExprKindGroup_Binary, +10, "*", "", "" }, // MD_C_ExprKind_Multiply - {MD_C_ExprKindGroup_Binary, +10, "/", "", "" }, // MD_C_ExprKind_Divide - {MD_C_ExprKindGroup_Binary, +10, "%", "", "" }, // MD_C_ExprKind_Mod - {MD_C_ExprKindGroup_Binary, +6, "==", "", "" }, // MD_C_ExprKind_IsEqual - {MD_C_ExprKindGroup_Binary, +6, "!=", "", "" }, // MD_C_ExprKind_IsNotEqual - {MD_C_ExprKindGroup_Binary, +7, "<", "", "" }, // MD_C_ExprKind_LessThan - {MD_C_ExprKindGroup_Binary, +7, ">", "", "" }, // MD_C_ExprKind_GreaterThan - {MD_C_ExprKindGroup_Binary, +7, "<=", "", "" }, // MD_C_ExprKind_LessThanEqualTo - {MD_C_ExprKindGroup_Binary, +7, ">=", "", "" }, // MD_C_ExprKind_GreaterThanEqualTo - {MD_C_ExprKindGroup_Binary, +2, "&&", "", "" }, // MD_C_ExprKind_BoolAnd - {MD_C_ExprKindGroup_Binary, +1, "||", "", "" }, // MD_C_ExprKind_BoolOr - {MD_C_ExprKindGroup_PreUnary, +12, "", "!", "" }, // MD_C_ExprKind_BoolNot - {MD_C_ExprKindGroup_Binary, +5, "&", "", "" }, // MD_C_ExprKind_BitAnd - {MD_C_ExprKindGroup_Binary, +3, "|", "", "" }, // MD_C_ExprKind_BitOr - {MD_C_ExprKindGroup_PreUnary, +12, "", "~", "" }, // MD_C_ExprKind_BitNot - {MD_C_ExprKindGroup_Binary, +4, "^", "", "" }, // MD_C_ExprKind_BitXor - {MD_C_ExprKindGroup_Binary, +8, "<<", "", "" }, // MD_C_ExprKind_LeftShift - {MD_C_ExprKindGroup_Binary, +8, ">>", "", "" }, // MD_C_ExprKind_RightShift - {MD_C_ExprKindGroup_PreUnary, +12, "", "-", "" }, // MD_C_ExprKind_Negative - {MD_C_ExprKindGroup_Type, +0, "*", "", "" }, // MD_C_ExprKind_Pointer - {MD_C_ExprKindGroup_Type, +0, "", "[", "]"}, // MD_C_ExprKind_Array - }; - return &metadata[kind]; -} - -MD_FUNCTION_IMPL MD_C_ExprKind -MD_C_PostUnaryExprKindFromNode(MD_Node *node) -{ - MD_C_ExprKind kind = MD_C_ExprKind_Nil; - if(!MD_NodeIsNil(node->first_child)) - { - if(node->flags & MD_NodeFlag_HasParenLeft && - node->flags & MD_NodeFlag_HasParenRight) - { - kind = MD_C_ExprKind_Call; - } - else if(node->flags & MD_NodeFlag_HasBracketLeft && - node->flags & MD_NodeFlag_HasBracketRight) - { - kind = MD_C_ExprKind_Subscript; - } - } - return kind; -} - -MD_FUNCTION_IMPL MD_C_ExprKind -MD_C_PreUnaryExprKindFromNode(MD_Node *node) -{ - MD_C_ExprKind kind = MD_C_ExprKind_Nil; - // NOTE(rjf): Special-cases for calls/subscripts. - if(!MD_NodeIsNil(node->first_child)) - { - if(node->flags & MD_NodeFlag_HasParenLeft && - node->flags & MD_NodeFlag_HasParenRight) - { - kind = MD_C_ExprKind_Call; - } - else if(node->flags & MD_NodeFlag_HasBracketLeft && - node->flags & MD_NodeFlag_HasBracketRight) - { - kind = MD_C_ExprKind_Subscript; - } - } - else - { - for(MD_C_ExprKind kind_it = (MD_C_ExprKind)0; kind_it < MD_C_ExprKind_MAX; - kind_it = (MD_C_ExprKind)((int)kind_it + 1)) - { - _MD_C_ExprKindMetadata *metadata = _MD_MetadataFromExprKind(kind_it); - if(metadata->group == MD_C_ExprKindGroup_PreUnary) - { - if(MD_S8Match(node->string, MD_S8CString(metadata->pre_symbol), 0)) - { - kind = kind_it; - break; - } - } - } - } - return kind; -} - -MD_FUNCTION_IMPL MD_C_ExprKind -MD_C_BinaryExprKindFromNode(MD_Node *node) -{ - MD_C_ExprKind kind = MD_C_ExprKind_Nil; - if(node->kind == MD_NodeKind_Main && MD_NodeIsNil(node->first_child)) - { - for(MD_C_ExprKind kind_it = (MD_C_ExprKind)0; kind_it < MD_C_ExprKind_MAX; - kind_it = (MD_C_ExprKind)((int)kind_it + 1)) - { - _MD_C_ExprKindMetadata *metadata = _MD_MetadataFromExprKind(kind_it); - if(metadata->group == MD_C_ExprKindGroup_Binary) - { - if(MD_S8Match(node->string, MD_S8CString(metadata->symbol), 0)) - { - kind = kind_it; - break; - } - } - } - } - return kind; -} - -MD_FUNCTION_IMPL MD_C_ExprPrec -MD_C_ExprPrecFromExprKind(MD_C_ExprKind kind) -{ - _MD_C_ExprKindMetadata *metadata = _MD_MetadataFromExprKind(kind); - return metadata->prec; -} - -MD_FUNCTION_IMPL MD_C_Expr * -MD_C_MakeExpr(MD_Arena *arena, MD_Node *node, MD_C_ExprKind kind, MD_C_Expr *left, - MD_C_Expr *right) -{ - MD_C_Expr *expr = MD_PushArrayZero(arena, MD_C_Expr, 1); - if(left == 0) left = MD_C_NilExpr(); - if(right == 0) right = MD_C_NilExpr(); - expr->node = node; - expr->kind = kind; - expr->sub[0] = left; - expr->sub[1] = right; - return expr; -} - -typedef struct _MD_NodeParseCtx _MD_NodeParseCtx; -struct _MD_NodeParseCtx -{ - MD_Node *at; - MD_Node *one_past_last; -}; - -MD_FUNCTION_IMPL MD_b32 -_MD_NodeParse_ConsumeAtom(_MD_NodeParseCtx *ctx, MD_Node **out) -{ - MD_b32 result = 0; - if(ctx->at->kind == MD_NodeKind_Main && - MD_NodeIsNil(ctx->at->first_child)) - { - result = 1; - if(out) - { - *out = ctx->at; - } - ctx->at = ctx->at->next; - } - return result; -} - -MD_FUNCTION_IMPL MD_b32 -_MD_NodeParse_ConsumeSet(_MD_NodeParseCtx *ctx, MD_Node **out) -{ - MD_b32 result = 0; - if(!MD_NodeIsNil(ctx->at->first_child) || - ctx->at->flags & MD_NodeFlag_HasParenLeft || - ctx->at->flags & MD_NodeFlag_HasParenRight || - ctx->at->flags & MD_NodeFlag_HasBracketLeft || - ctx->at->flags & MD_NodeFlag_HasBracketRight || - ctx->at->flags & MD_NodeFlag_HasBraceLeft || - ctx->at->flags & MD_NodeFlag_HasBraceRight) - { - if(out) - { - *out = ctx->at; - } - ctx->at = ctx->at->next; - result = 1; - } - return result; -} - -MD_FUNCTION_IMPL MD_b32 -_MD_NodeParse_ConsumeLiteral(_MD_NodeParseCtx *ctx, MD_Node **out) -{ - MD_b32 result = 0; - if(ctx->at->flags & MD_NodeFlag_StringLiteral || - ctx->at->flags & MD_NodeFlag_Numeric) - { - result = 1; - if(out) - { - *out = ctx->at; - } - ctx->at = ctx->at->next; - } - return result; -} - -MD_FUNCTION_IMPL MD_b32 -_MD_NodeParse_Consume(_MD_NodeParseCtx *ctx, MD_String8 string, MD_Node **out) -{ - MD_b32 result = 0; - if(MD_S8Match(ctx->at->string, string, 0)) - { - result = 1; - if(out) - { - *out = ctx->at; - } - ctx->at = ctx->at->next; - } - return result; -} - -MD_FUNCTION_IMPL void -_MD_NodeParse_Next(_MD_NodeParseCtx *ctx) -{ - ctx->at = ctx->at->next; -} - -MD_FUNCTION MD_C_Expr * _MD_ParseExpr_(MD_Arena *arena, _MD_NodeParseCtx *ctx, - int precedence_in); -MD_FUNCTION MD_C_Expr * _MD_ParseExpr(MD_Arena *arena, _MD_NodeParseCtx *ctx); - -MD_FUNCTION_IMPL MD_C_Expr * -_MD_ParseUnaryExpr(MD_Arena *arena, _MD_NodeParseCtx *ctx) -{ - MD_C_Expr *result = MD_C_NilExpr(); - MD_Node *set = 0; - MD_Node *node = 0; - - // NOTE(rjf): Sub-expression - if(_MD_NodeParse_ConsumeSet(ctx, &set)) - { - result = MD_C_ParseAsExpr(arena, set->first_child, set->last_child); - } - - // NOTE(rjf): Literal - else if(_MD_NodeParse_ConsumeLiteral(ctx, &node)) - { - result = MD_C_MakeExpr(arena, node, MD_C_ExprKind_Atom, 0, 0); - } - - // NOTE(rjf): Literal - else if(_MD_NodeParse_ConsumeAtom(ctx, &node)) - { - result = MD_C_MakeExpr(arena, node, MD_C_ExprKind_Atom, 0, 0); - } - - // NOTE(rjf): Negative - else if(_MD_NodeParse_Consume(ctx, MD_S8Lit("-"), &node)) - { - result = MD_C_MakeExpr(arena, node, MD_C_ExprKind_Negative, 0, _MD_ParseExpr(arena, ctx)); - } - - // NOTE(rjf): Bitwise Negate - else if(_MD_NodeParse_Consume(ctx, MD_S8Lit("~"), &node)) - { - result = MD_C_MakeExpr(arena, node, MD_C_ExprKind_BitNot, 0, _MD_ParseExpr(arena, ctx)); - } - - // NOTE(rjf): Boolean Negate - else if(_MD_NodeParse_Consume(ctx, MD_S8Lit("!"), &node)) - { - result = MD_C_MakeExpr(arena, node, MD_C_ExprKind_BoolNot, 0, _MD_ParseExpr(arena, ctx)); - } - - // NOTE(rjf): Post-Unary Sets (calls and subscripts) - if(_MD_NodeParse_ConsumeSet(ctx, &set)) - { - if(set->flags & MD_NodeFlag_HasParenLeft && set->flags & MD_NodeFlag_HasParenRight) - { - result = MD_C_MakeExpr(arena, set, MD_C_ExprKind_Call, result, 0); - } - else if(set->flags & MD_NodeFlag_HasBracketLeft && set->flags & MD_NodeFlag_HasBracketRight) - { - result = MD_C_MakeExpr(arena, set, MD_C_ExprKind_Subscript, result, MD_C_ParseAsExpr(arena, set->first_child, set->last_child)); - } - } - - return result; -} - -MD_FUNCTION_IMPL MD_C_Expr * -_MD_ParseExpr_(MD_Arena *arena, _MD_NodeParseCtx *ctx, int precedence_in) -{ - MD_C_Expr *expr = _MD_ParseUnaryExpr(arena, ctx); - MD_C_ExprKind expr_kind; - if(MD_C_ExprIsNil(expr)) - { - goto end_parse; - } - - expr_kind = MD_C_BinaryExprKindFromNode(ctx->at); - if(expr_kind != MD_C_ExprKind_Nil) - { - for(int precedence = MD_C_ExprPrecFromExprKind(expr_kind); - precedence >= precedence_in; - precedence -= 1) - { - for(;;) - { - MD_Node *op_node = ctx->at; - expr_kind = MD_C_BinaryExprKindFromNode(ctx->at); - int operator_precedence = MD_C_ExprPrecFromExprKind(expr_kind); - if(operator_precedence != precedence) - { - break; - } - - if(expr_kind == MD_C_ExprKind_Nil) - { - break; - } - - _MD_NodeParse_Next(ctx); - - MD_C_Expr *right = _MD_ParseExpr_(arena, ctx, precedence+1); - if(MD_C_ExprIsNil(right)) - { - // TODO(rjf): Error: "Expected right-hand-side of binary expression." - goto end_parse; - } - - MD_C_Expr *left = expr; - expr = MD_C_MakeExpr(arena, op_node, expr_kind, left, right); - expr->sub[0] = left; - expr->sub[1] = right; - } - } - } - - end_parse:; - return expr; -} - -MD_FUNCTION_IMPL MD_C_Expr * -_MD_ParseExpr(MD_Arena *arena, _MD_NodeParseCtx *ctx) -{ - return _MD_ParseExpr_(arena, ctx, 1); -} - -MD_FUNCTION_IMPL MD_C_Expr * -MD_C_ParseAsExpr(MD_Arena *arena, MD_Node *first, MD_Node *opl) -{ - _MD_NodeParseCtx ctx_ = { first, opl }; - _MD_NodeParseCtx *ctx = &ctx_; - return _MD_ParseExpr(arena, ctx); -} - -MD_FUNCTION_IMPL MD_C_Expr * -MD_C_ParseAsType(MD_Arena *arena, MD_Node *first, MD_Node *opl) -{ - MD_C_Expr *expr = MD_C_NilExpr(); - MD_C_Expr *last_expr = expr; - _MD_NodeParseCtx ctx_ = { first, opl }; - _MD_NodeParseCtx *ctx = &ctx_; -#define _MD_PushType(x) if(MD_C_ExprIsNil(last_expr)) { expr = last_expr = x; } else { last_expr = last_expr->sub[0] = x; } - MD_Node *set = 0; - MD_Node *ptr = 0; - MD_Node *base_type = 0; - MD_Node *node = 0; - for(;;) - { - if(_MD_NodeParse_Consume(ctx, MD_S8Lit("*"), &ptr)) - { - MD_C_Expr *t = MD_C_MakeExpr(arena, ptr, MD_C_ExprKind_Pointer, - MD_C_NilExpr(), MD_C_NilExpr()); - _MD_PushType(t); - } - else if(_MD_NodeParse_Consume(ctx, MD_S8Lit("volatile"), &node)) - { - MD_C_Expr *t = MD_C_MakeExpr(arena, node, MD_C_ExprKind_Volatile, MD_C_NilExpr(), MD_C_NilExpr()); - _MD_PushType(t); - } - else if(_MD_NodeParse_Consume(ctx, MD_S8Lit("const"), &node)) - { - MD_C_Expr *t = MD_C_MakeExpr(arena, node, MD_C_ExprKind_Const, MD_C_NilExpr(), MD_C_NilExpr()); - _MD_PushType(t); - } - else if(_MD_NodeParse_ConsumeSet(ctx, &set)) - { - MD_C_Expr *t = MD_C_MakeExpr(arena, set, MD_C_ExprKind_Array, MD_C_NilExpr(), MD_C_NilExpr()); - t->sub[1] = MD_C_ParseAsExpr(arena, set->first_child, set->last_child); - _MD_PushType(t); - } - else if(_MD_NodeParse_ConsumeAtom(ctx, &base_type)) - { - MD_C_Expr *t = MD_C_MakeExpr(arena, base_type, MD_C_ExprKind_Atom, MD_C_NilExpr(), MD_C_NilExpr()); - _MD_PushType(t); - } - else - { - break; - } - } -#undef _MD_PushType - return expr; -} - -MD_FUNCTION_IMPL MD_i64 -MD_C_EvaluateExpr_I64(MD_C_Expr *expr) -{ - MD_i64 result = 0; - switch(expr->kind) - { -#define _MD_BinaryOp(name, op) case MD_C_ExprKind_##name: { result = MD_C_EvaluateExpr_I64(expr->sub[0]) op MD_C_EvaluateExpr_I64(expr->sub[1]); }break - _MD_BinaryOp(Add, +); - _MD_BinaryOp(Subtract, -); - _MD_BinaryOp(Multiply, *); - _MD_BinaryOp(Divide, /); -#undef _MD_BinaryOp - case MD_C_ExprKind_Atom: { - result = MD_CStyleIntFromString(expr->node->string); - }break; - default: break; - } - return result; -} - -MD_FUNCTION_IMPL MD_f64 -MD_C_EvaluateExpr_F64(MD_C_Expr *expr) -{ - MD_f64 result = 0; - switch(expr->kind) - { -#define _MD_BinaryOp(name, op) case MD_C_ExprKind_##name: { result = MD_C_EvaluateExpr_F64(expr->sub[0]) op MD_C_EvaluateExpr_F64(expr->sub[1]); }break - _MD_BinaryOp(Add, +); - _MD_BinaryOp(Subtract, -); - _MD_BinaryOp(Multiply, *); - _MD_BinaryOp(Divide, /); -#undef _MD_BinaryOp - case MD_C_ExprKind_Atom: { result = MD_F64FromString(expr->node->string); }break; - default: break; - } - return result; -} - -MD_FUNCTION_IMPL MD_b32 -MD_C_ExprMatch(MD_C_Expr *a, MD_C_Expr *b, MD_MatchFlags flags) -{ - MD_b32 result = 0; - if(a->kind == b->kind) - { - result = 1; - if(a->kind == MD_C_ExprKind_Atom) - { - result = MD_S8Match(a->node->string, b->node->string, flags); - } - } - return result; -} - -MD_FUNCTION_IMPL MD_b32 -MD_C_ExprDeepMatch(MD_C_Expr *a, MD_C_Expr *b, MD_MatchFlags flags) -{ - MD_b32 result = MD_C_ExprMatch(a, b, flags); - if(result && !MD_C_ExprIsNil(a) && !MD_C_ExprIsNil(b)) - { - result = (MD_C_ExprDeepMatch(a->sub[0], b->sub[0], flags) && - MD_C_ExprDeepMatch(a->sub[1], b->sub[1], flags)); - } - return result; -} - -//~ C Language Generation - -MD_FUNCTION_IMPL void -MD_C_Generate_String(FILE *file, MD_Node *node) -{ - fprintf(file, "\""); - for(MD_u64 i = 0; i < node->string.size; i += 1) - { - if(node->string.str[i] == '\n') - { - fprintf(file, "\\n\"\n\""); - } - else if(node->string.str[i] == '\r' && - i+1 < node->string.size && node->string.str[i+1] == '\n') - { - // NOTE(mal): Step over CR when quoting CRLF newlines - } - else - { - fprintf(file, "%c", node->string.str[i]); - } - } - fprintf(file, "\""); -} - -MD_FUNCTION_IMPL void -MD_C_Generate_Struct(FILE *file, MD_Node *node) -{ - if(node) - { - fprintf(file, "typedef struct %.*s %.*s;\n", - MD_S8VArg(node->string), - MD_S8VArg(node->string)); - fprintf(file, "struct %.*s\n{\n", MD_S8VArg(node->string)); - for(MD_Node *child = node->first_child; !MD_NodeIsNil(child); child = child->next) - { - MD_C_Generate_Decl(file, child); - fprintf(file, ";\n"); - } - fprintf(file, "};\n\n"); - } -} - -MD_FUNCTION_IMPL void -MD_C_Generate_Expr(FILE *file, MD_C_Expr *expr) -{ - if(!MD_NodeIsNil(expr->node)) - { - _MD_C_ExprKindMetadata *metadata = _MD_MetadataFromExprKind(expr->kind); - switch(metadata->group) - { - case MD_C_ExprKindGroup_Atom: - { - fprintf(file, "%.*s", MD_S8VArg(expr->node->string)); - }break; - - case MD_C_ExprKindGroup_Binary: - { - fprintf(file, "("); - MD_C_Generate_Expr(file, expr->sub[0]); - fprintf(file, " %s ", metadata->symbol); - MD_C_Generate_Expr(file, expr->sub[1]); - fprintf(file, ")"); - }break; - - case MD_C_ExprKindGroup_PreUnary: - { - fprintf(file, "%s", metadata->pre_symbol); - fprintf(file, "("); - MD_C_Generate_Expr(file, expr->sub[0]); - fprintf(file, ")"); - }break; - - case MD_C_ExprKindGroup_PostUnary: - { - fprintf(file, "("); - MD_C_Generate_Expr(file, expr->sub[0]); - fprintf(file, ")"); - fprintf(file, "%s", metadata->post_symbol); - }break; - - default: break; - } - } -} - -MD_FUNCTION_IMPL MD_b32 -_MD_OutputType_C_NeedsParens(MD_C_Expr *type) -{ - MD_b32 result = 0; - if (type->kind == MD_C_ExprKind_Pointer && - type->sub[0]->kind == MD_C_ExprKind_Array) - { - result = 1; - } - return(result); -} - -MD_FUNCTION_IMPL void -MD_C_Generate_TypeLHS(FILE *file, MD_C_Expr *type) -{ - switch (type->kind) - { - case MD_C_ExprKind_Atom: - { - MD_Node *node = type->node; - fprintf(file, "%.*s", MD_S8VArg(node->raw_string)); - }break; - - case MD_C_ExprKind_Pointer: - { - MD_C_Generate_TypeLHS(file, type->sub[0]); - if (_MD_OutputType_C_NeedsParens(type)) - { - fprintf(file, "("); - } - fprintf(file, "*"); - }break; - - case MD_C_ExprKind_Array: - { - MD_C_Generate_TypeLHS(file, type->sub[0]); - if (_MD_OutputType_C_NeedsParens(type)) - { - fprintf(file, "("); - } - }break; - - case MD_C_ExprKind_Volatile: { fprintf(file, "volatile "); }break; - case MD_C_ExprKind_Const: { fprintf(file, "const "); }break; - - default: - { - fprintf(file, "{ unexpected MD_C_ExprKind (%i) in type info for node \"%.*s\" }", - type->kind, - MD_S8VArg(type->node->raw_string)); - }break; - } -} - -MD_FUNCTION_IMPL void -MD_C_Generate_TypeRHS(FILE *file, MD_C_Expr *type) -{ - switch (type->kind) - { - case MD_C_ExprKind_Atom: - {}break; - - case MD_C_ExprKind_Pointer: - { - if (_MD_OutputType_C_NeedsParens(type)) - { - fprintf(file, ")"); - } - MD_C_Generate_TypeRHS(file, type->sub[0]); - }break; - - case MD_C_ExprKind_Array: - { - if (_MD_OutputType_C_NeedsParens(type)) - { - fprintf(file, ")"); - } - fprintf(file, "["); - MD_C_Generate_Expr(file, type->sub[1]); - fprintf(file, "]"); - MD_C_Generate_TypeRHS(file, type->sub[0]); - }break; - - case MD_C_ExprKind_Volatile: { fprintf(file, "volatile "); }break; - case MD_C_ExprKind_Const: { fprintf(file, "const "); }break; - - default: - {}break; - } -} - -MD_FUNCTION_IMPL void -MD_C_Generate_DeclByNameAndType(FILE *file, MD_String8 name, MD_C_Expr *type) -{ - MD_C_Generate_TypeLHS(file, type); - fprintf(file, " %.*s", MD_S8VArg(name)); - MD_C_Generate_TypeRHS(file, type); -} - -MD_FUNCTION_IMPL void -MD_C_Generate_Decl(FILE *file, MD_Node *node) -{ - if(node) - { - MD_ArenaTemp scratch = MD_GetScratch(0, 0); - MD_C_Expr *type = MD_C_ParseAsType(scratch.arena, node->first_child, node->last_child); - MD_C_Generate_DeclByNameAndType(file, node->string, type); - MD_ReleaseScratch(scratch); - } -} - -/* -Copyright 2021 Dion Systems LLC - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ diff --git a/source/md_c_helpers.h b/source/md_c_helpers.h deleted file mode 100644 index d766415..0000000 --- a/source/md_c_helpers.h +++ /dev/null @@ -1,131 +0,0 @@ -/* date = April 16th 2021 6:05 pm */ - -#error foo - -#ifndef MD_C_HELPERS_H -#define MD_C_HELPERS_H - -//~ Expression and Type-Expression parser helper types. - -typedef enum MD_C_ExprKind -{ - // VERY_IMPORTANT_NOTE(rjf): If this enum is ever changed, ensure that - // it is kept in-sync with the _MD_MetadataFromExprKind function. - - MD_C_ExprKind_Nil, - - // NOTE(rjf): Atom - MD_C_ExprKind_Atom, - - // NOTE(rjf): Access - MD_C_ExprKind_Dot, - MD_C_ExprKind_Arrow, - MD_C_ExprKind_Call, - MD_C_ExprKind_Subscript, - MD_C_ExprKind_Dereference, - MD_C_ExprKind_Reference, - - // NOTE(rjf): Arithmetic - MD_C_ExprKind_Add, - MD_C_ExprKind_Subtract, - MD_C_ExprKind_Multiply, - MD_C_ExprKind_Divide, - MD_C_ExprKind_Mod, - - // NOTE(rjf): Comparison - MD_C_ExprKind_IsEqual, - MD_C_ExprKind_IsNotEqual, - MD_C_ExprKind_LessThan, - MD_C_ExprKind_GreaterThan, - MD_C_ExprKind_LessThanEqualTo, - MD_C_ExprKind_GreaterThanEqualTo, - - // NOTE(rjf): Bools - MD_C_ExprKind_BoolAnd, - MD_C_ExprKind_BoolOr, - MD_C_ExprKind_BoolNot, - // NOTE(rjf): Bitwise - MD_C_ExprKind_BitAnd, - MD_C_ExprKind_BitOr, - MD_C_ExprKind_BitNot, - MD_C_ExprKind_BitXor, - MD_C_ExprKind_LeftShift, - MD_C_ExprKind_RightShift, - - // NOTE(rjf): Unary numeric - MD_C_ExprKind_Negative, - - // NOTE(rjf): Type - MD_C_ExprKind_Pointer, - MD_C_ExprKind_Array, - MD_C_ExprKind_Volatile, - MD_C_ExprKind_Const, - - MD_C_ExprKind_MAX, -} -MD_C_ExprKind; - -typedef enum MD_C_ExprKindGroup -{ - MD_C_ExprKindGroup_Nil, - MD_C_ExprKindGroup_Atom, - MD_C_ExprKindGroup_Binary, - MD_C_ExprKindGroup_PreUnary, - MD_C_ExprKindGroup_PostUnary, - MD_C_ExprKindGroup_Type, -} -MD_C_ExprKindGroup; - -typedef MD_i32 MD_C_ExprPrec; - -typedef struct MD_C_Expr MD_C_Expr; -struct MD_C_Expr -{ - MD_Node *node; - MD_C_ExprKind kind; - MD_C_Expr *parent; - MD_C_Expr *sub[2]; -}; - -//~ C_Expression and Type-C_Expression Helper - -MD_FUNCTION MD_C_Expr * MD_C_NilExpr(void); -MD_FUNCTION MD_b32 MD_C_ExprIsNil(MD_C_Expr *expr); -MD_FUNCTION MD_C_ExprKind MD_C_PreUnaryExprKindFromNode(MD_Node *node); -MD_FUNCTION MD_C_ExprKind MD_C_BinaryExprKindFromNode(MD_Node *node); -MD_FUNCTION MD_C_ExprPrec MD_C_ExprPrecFromExprKind(MD_C_ExprKind kind); -MD_FUNCTION MD_C_Expr * MD_C_MakeExpr(MD_Arena *arena, MD_Node *node, MD_C_ExprKind kind, - MD_C_Expr *left, MD_C_Expr *right); -MD_FUNCTION MD_C_Expr * MD_C_ParseAsExpr(MD_Arena *arena, MD_Node *first, MD_Node *opl); -MD_FUNCTION MD_C_Expr * MD_C_ParseAsType(MD_Arena *arena, MD_Node *first, MD_Node *opl); -MD_FUNCTION MD_i64 MD_C_EvaluateExpr_I64(MD_C_Expr *expr); -MD_FUNCTION MD_f64 MD_C_EvaluateExpr_F64(MD_C_Expr *expr); -MD_FUNCTION MD_b32 MD_C_ExprMatch(MD_C_Expr *a, MD_C_Expr *b, MD_MatchFlags flags); -MD_FUNCTION MD_b32 MD_C_ExprDeepMatch(MD_C_Expr *a, MD_C_Expr *b, MD_MatchFlags flags); - -//~ C Language Generation - -// TODO(allen): cleanup these printers -#include - -MD_FUNCTION void MD_C_Generate_String(FILE *file, MD_Node *node); -MD_FUNCTION void MD_C_Generate_Struct(FILE *file, MD_Node *node); - -MD_FUNCTION void MD_C_Generate_Expr(FILE *file, MD_C_Expr *expr); -MD_FUNCTION void MD_C_Generate_TypeLHS(FILE *file, MD_C_Expr *type); -MD_FUNCTION void MD_C_Generate_TypeRHS(FILE *file, MD_C_Expr *type); - -MD_FUNCTION void MD_C_Generate_DeclByNameAndType(FILE *file, MD_String8 name, MD_C_Expr *type); -MD_FUNCTION void MD_C_Generate_Decl(FILE *file, MD_Node *node); - -#endif //MD_C_HELPERS_H - -/* -Copyright 2021 Dion Systems LLC - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ From fb53369f3041b7bf219f36d5a7635015e4823460 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 8 Oct 2021 15:56:16 -0700 Subject: [PATCH 04/28] [examples] finish commentary in overrides.c --- examples/integration/overrides.c | 39 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/examples/integration/overrides.c b/examples/integration/overrides.c index b845b25..c48c038 100644 --- a/examples/integration/overrides.c +++ b/examples/integration/overrides.c @@ -53,13 +53,16 @@ void examp_free(ExampleAllocator *a, void *ptr); // override memory to use malloc/free // @notes A common practice in setting up allocator overrides is to use a pass -// through opaque user context pointer. We took a look at that for *long time* -// and just really don't like the amount of baggage it adds to the entire API. -// Instead we recommend handling the context pointer with a global for single -// threaded use cases, or with a pointer in thread local storage for -// multi-threaded cases. +// through opaque user context pointer. Metadesk does something different. +// We recommend passing the context pointer with a global in single threaded +// cases, and with a pointer in thread local storage in multi-threaded cases. ExampleAllocator* md_example_allocator = 0; +// @notes In this example the allocator only provides alloc & free, but the +// Metadesk override group we want to plug into has reserve commit, decommit & +// release. This is okay though, we can turn commit & decommit into no-ops and +// reserve & release as equivalent to alloc & free. + void* md_reserve_by_example_allocator(unsigned long long size); void md_release_by_example_allocator(void *ptr, unsigned long long ignore); @@ -68,12 +71,26 @@ void md_release_by_example_allocator(void *ptr, unsigned long long ignore); #define MD_IMPL_Decommit(p,z) ((void)0) #define MD_IMPL_Release md_release_by_example_allocator +// @notes Since we are turning commit & decommit into no-ops it doesn't make +// sense for the Metadesk arena to have a reserve size larger than it's +// commit size anymore. The default for reserve size is 64 megabytes, which +// is usually too large of an alloc block size, and the default commit size +// is 64 kilabytes, which is usually too small. So we set both to 1 megabyte. +// +// Pro-Tip: (N << 20) is a nice shorthand for N megabytes, and +// (N << 10) is N kilabytes. + #define MD_DEFAULT_ARENA_RES_SIZE (1 << 20) #define MD_DEFAULT_ARENA_CMT_SIZE (1 << 20) // override file loading +// @notes We'll also demonstrate another override, this time one that relies +// on Metadesk-provided types. The actual override here is pointless, as it's +// just another implementation of "LoadEntireFile" on stdio.h, which is what +// the default provided by the library is as well. + MD_String8 md_load_entire_file_by_stdio(MD_Arena *arena, MD_String8 filename); #define MD_IMPL_LoadEntireFile md_load_entire_file_by_stdio @@ -197,3 +214,15 @@ examp_free(ExampleAllocator *a, void *ptr) free(node); } +//~ final notes /////////////////////////////////////////////////////////////// + +// @notes The Metadesk override system uses macro overriding. This means it +// does not provide a mechanism for dynamically dispatching to different +// implementations of the overridable operations. But you can always plug in +// your own dynamic dispatch into the macro if you really need it. We +// recommend against trying to instantiate the library twice in the same +// program with different overrides, because that will lead to separate +// instances of our thread local context variables which will make Metadesk +// more resource intensive than it needs to be, and may lead to surprising +// behavior. + From 367b5272295d63997e4ab6c88daebd456e69e97c Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 8 Oct 2021 21:01:04 -0700 Subject: [PATCH 05/28] [examples] start type metadata commentary --- examples/examples_directory.txt | 2 + examples/type_metadata/type_info.h | 5 +-- .../type_metadata/type_info_final_program.c | 2 +- examples/type_metadata/type_metadata.c | 39 ++++++++++++++++++- examples/type_metadata/types.mdesk | 4 +- 5 files changed, 46 insertions(+), 6 deletions(-) diff --git a/examples/examples_directory.txt b/examples/examples_directory.txt index d1a4c2f..bed4b02 100644 --- a/examples/examples_directory.txt +++ b/examples/examples_directory.txt @@ -34,6 +34,8 @@ EXAMPLES: 6. "type metadata" type_metadata/* + (Start exploring this example at type_metadata/type_metadata.c) + A common use case for a metaprogramming system in C is to mark up type information with metadata. This example shows the "Metadesk way" of implementing that use case. This is the biggest example included and closely diff --git a/examples/type_metadata/type_info.h b/examples/type_metadata/type_info.h index 271d0e1..f13b317 100644 --- a/examples/type_metadata/type_info.h +++ b/examples/type_metadata/type_info.h @@ -1,9 +1,8 @@ /* ** Example: type metadata ** -** This is a hand written header to be included into the final program to -** define types that will be used to layout the metadata tables created by the -** generator. +** This is a hand written header to be included into the final program. It +** defines the types used to layout the generated tables of metadata. ** ** This file *does not* get included into the generator itself. ** diff --git a/examples/type_metadata/type_info_final_program.c b/examples/type_metadata/type_info_final_program.c index ac00754..2ccfafc 100644 --- a/examples/type_metadata/type_info_final_program.c +++ b/examples/type_metadata/type_info_final_program.c @@ -31,7 +31,7 @@ struct V2F32 int main(int argc, char **argv) { - + // TODO(allen): write example final program return(0); } diff --git a/examples/type_metadata/type_metadata.c b/examples/type_metadata/type_metadata.c index 6799984..f3ae63a 100644 --- a/examples/type_metadata/type_metadata.c +++ b/examples/type_metadata/type_metadata.c @@ -1,7 +1,44 @@ /* ** Example: type metadata ** -** TODO full commentary +** This example is full of commentary on strategies for setting up Metadesk +** programs. There are many many ways a system like this could be built, +** this example only demonstrates one way. Everything here should be taken as +** our best attempt to pass on insights about using Metadesk, and not as the +** "one true way". +** +** This example shows what a typical Metadesk based metaprogram for generating +** type metadata looks like. Metadesk can be used for all sorts of other +** purposes, managing data tables, authoring content, configuration systems, +** and more, but a very common reason that a C programmer reaches for a +** metaprogramming system is to make up for C's lack of type metadata. +** +** The "Metadesk way" of solving this problem is to move the hand written type +** definitions into metadesk files, and to generate the C structs AND the +** tables for type metadata from the metaprogram. +** +** A more common approach is to mark up C types and create a custom parser +** for the modified C hybdrid. That approach requires a lot of heuristics to +** deal with the complex grammar of C code, and is especially hard to make +** both robust and reusable. The Metadesk way makes the problem a lot more +** simple, if you are comfortable with generating C types instead of writing +** them by hand in C. +** +** +** Files: +** type_metadata.h - This example is big enough that we pulled out the types +** and function declarations from the metaprogram to keep +** things organized. +** type_info.h - This header is included in the "final program" and +** defines the types that will form the tables of metdata. +** type_info_final_program.c - In addition to the metaprogram this example +** includes a "final program", i.e. one that depends on +** generated code. There is more commentary there. +** generated/* - This is the output folder where the generated files +** should be saved by the metaprogram. +** types.mdesk - Sample input for this metaprogram. The code generated +** from this metadesk file will be necessary to make +** type_info_final_program.c compile. ** */ diff --git a/examples/type_metadata/types.mdesk b/examples/type_metadata/types.mdesk index f47ed17..554788f 100644 --- a/examples/type_metadata/types.mdesk +++ b/examples/type_metadata/types.mdesk @@ -1,5 +1,7 @@ /* -** Setup as input to the type metadata example +** Example: type metadata +** +** Input for the type definition system setup by type_metadata.c */ @type(basic) U32: 4; From a5436aec0eec1373bcae62533008af8e31549522 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 8 Oct 2021 22:13:45 -0700 Subject: [PATCH 06/28] [examples] more type metadata commentary; write some usage code for the generated type info --- ...fo_example.sh => type_metadata_example.sh} | 18 +- examples/type_metadata/generated/meta_types.c | 8 +- examples/type_metadata/generated/meta_types.h | 6 +- .../type_metadata/type_info_final_program.c | 180 +++++++++++++++++- examples/type_metadata/type_metadata.c | 2 +- examples/type_metadata/type_metadata.h | 3 +- project.4coder | 10 +- 7 files changed, 205 insertions(+), 22 deletions(-) rename bin/{type_info_example.sh => type_metadata_example.sh} (80%) diff --git a/bin/type_info_example.sh b/bin/type_metadata_example.sh similarity index 80% rename from bin/type_info_example.sh rename to bin/type_metadata_example.sh index 84dc9f9..0be198c 100644 --- a/bin/type_info_example.sh +++ b/bin/type_metadata_example.sh @@ -30,12 +30,20 @@ $build_path/type_metadata.exe $examps_path/type_metadata/types.mdesk echo if [ -f $examps_path/type_metadata/generated/meta_types.h ]; then -echo ~~~ Building Program ~~~ -$bin_path/bld_core.sh unit type_info $examps/type_metadata/type_info_final_program.c -else -echo !!! Skipping Program !!! -fi +echo ~~~ Building Final Program ~~~ +$bin_path/bld_core.sh unit type_info_final $examps/type_metadata/type_info_final_program.c echo +echo ~~~ Running Final Program ~~~ +$build_path/type_info_final +echo + +else + +echo !!! Skipping Final Program !!! +echo + +fi + ###### Restore Path ########################################################### cd $og_path diff --git a/examples/type_metadata/generated/meta_types.c b/examples/type_metadata/generated/meta_types.c index 80ae53a..f0b52fa 100644 --- a/examples/type_metadata/generated/meta_types.c +++ b/examples/type_metadata/generated/meta_types.c @@ -1,4 +1,4 @@ -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:837 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:874 TypeInfoMember Circle_members[2] = { {"r", 1, -1, &F32_type_info}, {"pos", 3, -1, &V2F32_type_info}, @@ -14,14 +14,14 @@ TypeInfoMember RoundedPolygon_members[3] = { {"p", 1, 1, &V2F32_type_info}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:876 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:913 TypeInfoEnumerant Shape_members[3] = { {"Circle", 6, 1}, {"Segment", 7, 2}, {"Polygon", 7, 3}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:912 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:949 TypeInfo U32_type_info = {TypeKind_Basic, "U32", 3, 4, 0, 0}; TypeInfo F32_type_info = {TypeKind_Basic, "F32", 3, 4, 0, 0}; TypeInfo V2F32_type_info = {TypeKind_Basic, "V2F32", 5, 8, 0, 0}; @@ -30,7 +30,7 @@ TypeInfo RoundedSegment_type_info = {TypeKind_Struct, "RoundedSegment", 14, 3, R TypeInfo RoundedPolygon_type_info = {TypeKind_Struct, "RoundedPolygon", 14, 3, RoundedPolygon_members, 0}; TypeInfo Shape_type_info = {TypeKind_Enum, "Shape", 5, 3, Shape_members, &U32_type_info}; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:972 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1009 TypeInfo* type_info_from_shape(Shape v) { diff --git a/examples/type_metadata/generated/meta_types.h b/examples/type_metadata/generated/meta_types.h index 74939a4..5fe38c9 100644 --- a/examples/type_metadata/generated/meta_types.h +++ b/examples/type_metadata/generated/meta_types.h @@ -1,6 +1,6 @@ #if !defined(META_TYPES_H) #define META_TYPES_H -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:714 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:751 typedef struct Circle Circle; struct Circle { @@ -29,11 +29,11 @@ Shape_Segment = 2, Shape_Polygon = 3, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:798 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:835 TypeInfo* type_info_from_shape(Shape v); U32 max_slot_from_shape(Shape v); -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:821 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:858 extern TypeInfo U32_type_info; extern TypeInfo F32_type_info; extern TypeInfo V2F32_type_info; diff --git a/examples/type_metadata/type_info_final_program.c b/examples/type_metadata/type_info_final_program.c index 2ccfafc..886d8c5 100644 --- a/examples/type_metadata/type_info_final_program.c +++ b/examples/type_metadata/type_info_final_program.c @@ -1,13 +1,15 @@ /* ** Example: type metadata ** -** This file shows including the generated type information into a final -** program and using that type info to unpack a buffer of data. +** This file shows including the generated types and metadata in the a final +** program and using it to unpack a buffer of data. ** */ //~ setup base types ////////////////////////////////////////////////////////// #include +typedef uint8_t U8; +typedef uint16_t U16; typedef uint32_t U32; typedef float F32; typedef struct V2F32 V2F32; @@ -26,12 +28,184 @@ struct V2F32 //~ main ////////////////////////////////////////////////////////////////////// +U8 raw_buffer[] = { + 0x03,0x00,0x00,0x00,0x00,0x00,0x90,0x40,0x06,0x00,0x00,0x00, + 0x00,0x00,0x80,0x3F,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x3F, + 0x2D,0xB2,0x5D,0x3F,0x00,0x00,0x00,0xBF,0x2D,0xB2,0x5D,0x3F, + 0x00,0x00,0x80,0xBF,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xBF, + 0x2D,0xB2,0x5D,0xBF,0x00,0x00,0x00,0x3F,0x2D,0xB2,0x5D,0xBF, + 0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xC0, + 0x00,0x00,0x00,0xC0,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0xC0, + 0x02,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40, + 0x00,0x00,0x00,0xC0,0x00,0x00,0x80,0x40,0x00,0x00,0x00,0x00, +}; + #include +#include + +int +print_data_from_type_info(U8 **ptr_inout, U8 *opl, TypeInfo *type_info, int indent){ + static char spaces[] = + " "; + + int result = 1; + + switch (type_info->kind){ + case TypeKind_Basic: + { + // special case each basic type + + if (type_info == &U32_type_info) + { + if (*ptr_inout + sizeof(U32) > opl) + { + result = 0; + goto finish; + } + U32 x = *(U32*)*ptr_inout; + *ptr_inout += sizeof(x); + fprintf(stdout, "%u\n", x); + } + + if (type_info == &F32_type_info) + { + if (*ptr_inout + sizeof(F32) > opl) + { + result = 0; + goto finish; + } + F32 x = *(F32*)*ptr_inout; + *ptr_inout += sizeof(x); + fprintf(stdout, "%f\n", x); + } + + if (type_info == &V2F32_type_info) + { + if (*ptr_inout + sizeof(V2F32) > opl) + { + result = 0; + goto finish; + } + V2F32 x = *(V2F32*)*ptr_inout; + *ptr_inout += sizeof(x); + fprintf(stdout, "(%f,%f)\n", x.x, x.y); + } + }break; + + case TypeKind_Struct: + { + U32 member_value_memory[20] = {0}; + + // iterate members + TypeInfoMember *first_member = (TypeInfoMember*)type_info->children; + TypeInfoMember *member = first_member; + int child_count = type_info->child_count; + for (int i = 0; i < child_count; i += 1, member += 1) + { + assert(i < 20); + TypeInfo *member_type_info = member->type; + + // decode single members + if (member->array_count_member_index == -1) + { + // save the offset of this member for later + U8 *ptr_og = *ptr_inout; + // decode this member + fprintf(stdout, "%.*s%.*s: ", indent, spaces, member->name_length, member->name); + if (!print_data_from_type_info(ptr_inout, opl, member_type_info, indent + 1)) + { + result = 0; + goto finish; + } + // if we see a U32 we want to remember it in case it's an array count + if (member_type_info == &U32_type_info) + { + member_value_memory[i] = *(U32*)ptr_og; + } + } + + // decode arrays + else + { + assert(member->array_count_member_index < i); + U32 count = member_value_memory[member->array_count_member_index]; + fprintf(stdout, "%.*s%.*s:\n", indent, spaces, member->name_length, member->name); + indent += 1; + for (U32 j = 0; j < count; j += 1) + { + fprintf(stdout, "%.*s[%d]: ", indent, spaces, j); + if (!print_data_from_type_info(ptr_inout, opl, member_type_info, indent + 1)) + { + result = 0; + goto finish; + } + } + indent -= 1; + } + } + }break; + + case TypeKind_Enum: + { + // rely on the underlying type (default to U32) to print enums + TypeInfo *underlying_type = type_info->underlying_type; + if (underlying_type == 0) + { + underlying_type = &U32_type_info; + } + fprintf(stdout, "(%.*s)", type_info->name_length, type_info->name); + if (!print_data_from_type_info(ptr_inout, opl, underlying_type, indent)) + { + result = 0; + goto finish; + } + }break; + + default: + { + result = 0; + goto finish; + }break; + } + + finish:; + + return(result); +} int main(int argc, char **argv) { - // TODO(allen): write example final program + // decode the raw buffer + U8 *ptr = raw_buffer; + U8 *opl = ptr + sizeof(raw_buffer); + for (;ptr < opl;) + { + // decode a shape discriminator + if (ptr + sizeof(Shape) > opl){ + fprintf(stdout, "Could not decode shape discriminator\n"); + break; + } + Shape shape = *(Shape*)ptr; + ptr += sizeof(shape); + + // get type info + TypeInfo *type_info = type_info_from_shape(shape); + if (type_info == 0){ + fprintf(stdout, "Unrecognized shape discriminator\n"); + break; + } + + // print type name + fprintf(stdout, "%.*s:\n", type_info->name_length, type_info->name); + + // print contents + if (!print_data_from_type_info(&ptr, opl, type_info, 1)) + { + fprintf(stdout, "Decoding error\n"); + break; + } + } return(0); } diff --git a/examples/type_metadata/type_metadata.c b/examples/type_metadata/type_metadata.c index f3ae63a..ae6aa1f 100644 --- a/examples/type_metadata/type_metadata.c +++ b/examples/type_metadata/type_metadata.c @@ -34,7 +34,7 @@ ** type_info_final_program.c - In addition to the metaprogram this example ** includes a "final program", i.e. one that depends on ** generated code. There is more commentary there. -** generated/* - This is the output folder where the generated files +** generated/ * - This is the output folder where the generated files ** should be saved by the metaprogram. ** types.mdesk - Sample input for this metaprogram. The code generated ** from this metadesk file will be necessary to make diff --git a/examples/type_metadata/type_metadata.h b/examples/type_metadata/type_metadata.h index 204e52c..a2a3f79 100644 --- a/examples/type_metadata/type_metadata.h +++ b/examples/type_metadata/type_metadata.h @@ -1,7 +1,8 @@ /* ** Example: type metadata ** -** TODO full commentary +** The header file for the metaprogram's main file. There isn't much commentary +** here, this file is just to help us keep the example organized. ** */ diff --git a/project.4coder b/project.4coder index ad07c0f..1dbe6ba 100644 --- a/project.4coder +++ b/project.4coder @@ -94,16 +94,16 @@ command_list = }, }, { - .name = "type_info_example", + .name = "type_metadata_example", .out = "*run*", .footer_panel = false, .save_dirty_files = true, .cursor_at_end = false, .cmd = { - { "git_bash bin\\type_info_example.sh", .os = "win" }, - { "bin/type_info_example.sh", .os = "linux" }, - { "bin/type_info_example.sh", .os = "mac" }, + { "git_bash bin\\type_metadata_example.sh", .os = "win" }, + { "bin/type_metadata_example.sh", .os = "linux" }, + { "bin/type_metadata_example.sh", .os = "mac" }, }, }, }; @@ -112,4 +112,4 @@ fkey_command[1] = "build_tests"; fkey_command[2] = "run_tests"; fkey_command[3] = "build_examples"; fkey_command[4] = "run_examples"; -fkey_command[5] = "type_info_example"; +fkey_command[5] = "type_metadata_example"; From c8c5763d0a197d2c611fe5d28f05f20637f48adb Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Fri, 8 Oct 2021 23:12:32 -0700 Subject: [PATCH 07/28] [examples] progress on type metadata commentary --- examples/type_metadata/generated/meta_types.c | 8 +-- examples/type_metadata/generated/meta_types.h | 6 +- .../type_metadata/type_info_final_program.c | 23 ++++---- examples/type_metadata/type_metadata.c | 59 +++++++++++++++++-- examples/type_metadata/type_metadata.h | 2 + 5 files changed, 74 insertions(+), 24 deletions(-) diff --git a/examples/type_metadata/generated/meta_types.c b/examples/type_metadata/generated/meta_types.c index f0b52fa..a5eb56c 100644 --- a/examples/type_metadata/generated/meta_types.c +++ b/examples/type_metadata/generated/meta_types.c @@ -1,4 +1,4 @@ -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:874 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:914 TypeInfoMember Circle_members[2] = { {"r", 1, -1, &F32_type_info}, {"pos", 3, -1, &V2F32_type_info}, @@ -14,14 +14,14 @@ TypeInfoMember RoundedPolygon_members[3] = { {"p", 1, 1, &V2F32_type_info}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:913 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:953 TypeInfoEnumerant Shape_members[3] = { {"Circle", 6, 1}, {"Segment", 7, 2}, {"Polygon", 7, 3}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:949 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:989 TypeInfo U32_type_info = {TypeKind_Basic, "U32", 3, 4, 0, 0}; TypeInfo F32_type_info = {TypeKind_Basic, "F32", 3, 4, 0, 0}; TypeInfo V2F32_type_info = {TypeKind_Basic, "V2F32", 5, 8, 0, 0}; @@ -30,7 +30,7 @@ TypeInfo RoundedSegment_type_info = {TypeKind_Struct, "RoundedSegment", 14, 3, R TypeInfo RoundedPolygon_type_info = {TypeKind_Struct, "RoundedPolygon", 14, 3, RoundedPolygon_members, 0}; TypeInfo Shape_type_info = {TypeKind_Enum, "Shape", 5, 3, Shape_members, &U32_type_info}; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1009 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1049 TypeInfo* type_info_from_shape(Shape v) { diff --git a/examples/type_metadata/generated/meta_types.h b/examples/type_metadata/generated/meta_types.h index 5fe38c9..b969a2c 100644 --- a/examples/type_metadata/generated/meta_types.h +++ b/examples/type_metadata/generated/meta_types.h @@ -1,6 +1,6 @@ #if !defined(META_TYPES_H) #define META_TYPES_H -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:751 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:791 typedef struct Circle Circle; struct Circle { @@ -29,11 +29,11 @@ Shape_Segment = 2, Shape_Polygon = 3, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:835 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:875 TypeInfo* type_info_from_shape(Shape v); U32 max_slot_from_shape(Shape v); -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:858 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:898 extern TypeInfo U32_type_info; extern TypeInfo F32_type_info; extern TypeInfo V2F32_type_info; diff --git a/examples/type_metadata/type_info_final_program.c b/examples/type_metadata/type_info_final_program.c index 886d8c5..4875493 100644 --- a/examples/type_metadata/type_info_final_program.c +++ b/examples/type_metadata/type_info_final_program.c @@ -44,7 +44,8 @@ U8 raw_buffer[] = { #include int -print_data_from_type_info(U8 **ptr_inout, U8 *opl, TypeInfo *type_info, int indent){ +print_data_from_type_info(U8 **ptr_inout, U8 *opl, TypeInfo *type_info, int indent) +{ static char spaces[] = " "; @@ -108,20 +109,16 @@ print_data_from_type_info(U8 **ptr_inout, U8 *opl, TypeInfo *type_info, int inde // decode single members if (member->array_count_member_index == -1) { - // save the offset of this member for later - U8 *ptr_og = *ptr_inout; - // decode this member + if (member_type_info == &U32_type_info && *ptr_inout + 4 <= opl) + { + member_value_memory[i] = *(U32*)*ptr_inout; + } fprintf(stdout, "%.*s%.*s: ", indent, spaces, member->name_length, member->name); if (!print_data_from_type_info(ptr_inout, opl, member_type_info, indent + 1)) { result = 0; goto finish; } - // if we see a U32 we want to remember it in case it's an array count - if (member_type_info == &U32_type_info) - { - member_value_memory[i] = *(U32*)ptr_og; - } } // decode arrays @@ -176,13 +173,14 @@ print_data_from_type_info(U8 **ptr_inout, U8 *opl, TypeInfo *type_info, int inde int main(int argc, char **argv) { - // decode the raw buffer + // decode the raw buffer of shape data U8 *ptr = raw_buffer; U8 *opl = ptr + sizeof(raw_buffer); for (;ptr < opl;) { // decode a shape discriminator - if (ptr + sizeof(Shape) > opl){ + if (ptr + sizeof(Shape) > opl) + { fprintf(stdout, "Could not decode shape discriminator\n"); break; } @@ -191,7 +189,8 @@ main(int argc, char **argv) // get type info TypeInfo *type_info = type_info_from_shape(shape); - if (type_info == 0){ + if (type_info == 0) + { fprintf(stdout, "Unrecognized shape discriminator\n"); break; } diff --git a/examples/type_metadata/type_metadata.c b/examples/type_metadata/type_metadata.c index ae6aa1f..53321e3 100644 --- a/examples/type_metadata/type_metadata.c +++ b/examples/type_metadata/type_metadata.c @@ -54,6 +54,18 @@ static MD_Arena *arena = 0; FILE *error_file = 0; // node maps + +// @notes As we analyze the Metadesk tree we create more data. In this example +// the new data is stored in two major types GEN_TypeInfo and GEN_MapInfo. +// We form a list for each type of these "processed" types holds a pointer +// back to the original MD_Node that generated it, and room for information +// that will be equiped to the "processed" type in later stages of analysis. +// We also use the MD_Map helper to create a string -> pointer mapping so that +// we can look up the "processed" info pointers by name after the initial +// gather stage. +// +// TODO notes on managing output files. + GEN_TypeInfo *first_type = 0; GEN_TypeInfo *last_type = 0; MD_Map type_map = {0}; @@ -77,6 +89,12 @@ GEN_TypeInfo* gen_resolve_type_info_from_string(MD_String8 name) { GEN_TypeInfo *result = 0; + // @notes The MD_Map helper is a "flexibly" typed hash table. It's keys can + // be a mix of strings and pointers. Here MD_MapKeyStr(name) is making the + // `name` string into a key for the map. The lookup function returns a + // "map slot" because the map is not restricted to storing just one value + // per key, if we were using it that way we could use MD_MapScan to + // iterate through the map slots. MD_MapSlot *slot = MD_MapLookup(&type_map, MD_MapKeyStr(name)); if (slot != 0) { @@ -136,6 +154,15 @@ gen_map_case_from_enumerant(GEN_MapInfo *map, GEN_TypeEnumerant *enumerant) //~ analyzers ///////////////////////////////////////////////////////////////// +// @notes The first stage of processing is to loop over the top level nodes +// from each parse. We are using the tags `@type` and `@map` to mark the nodes +// that this generator will process. Whenever we see one of those tags we +// create a GEN_TypeInfo or GEN_MapInfo to gather up information from the +// stages of analysis, and we insert the new info pointer into the appropriate +// map. On the types we do a little bit of the analysis right in this function +// to figure out which "type kind" it is, this lets us avoid ever having info +// where the kind field is not one of the expected values. + void gen_gather_types_and_maps(MD_Node *list) { @@ -149,6 +176,8 @@ gen_gather_types_and_maps(MD_Node *list) if (!MD_NodeIsNil(type_tag)) { + // TODO(allen): check for duplicates + GEN_TypeKind kind = GEN_TypeKind_Null; MD_Node *tag_arg_node = type_tag->first_child; MD_String8 tag_arg_str = tag_arg_node->string; @@ -185,6 +214,8 @@ gen_gather_types_and_maps(MD_Node *list) // gather map if (MD_NodeHasTag(node, MD_S8Lit("map"), 0)) { + // TODO(allen): check for duplicates + GEN_MapInfo *map_info = MD_PushArrayZero(arena, GEN_MapInfo, 1); map_info->node = node; MD_QueuePush(first_map, last_map, map_info); @@ -226,6 +257,10 @@ gen_check_duplicate_member_names(void) } } +// @notes In the next few stages of analysis we 'equip' the info nodes we +// gathered with further information by examining the sub-trees rooted at the +// metadesk nodes we saw durring the gather phase. + void gen_equip_basic_type_size(void) { @@ -745,9 +780,14 @@ gen_check_complete_map_cases(void) //~ generators //////////////////////////////////////////////////////////////// +// @notes Each generator function does all of the generation for one of the +// generated features we want. TODO keep going here. + void gen_type_definitions_from_types(FILE *out) { + // @notes This Metadesk helper generates a comment that points back here. + // Generating a comment like this can help a lot to with issues later. MD_PrintGenNoteCComment(out); for (GEN_TypeInfo *type = first_type; @@ -1163,6 +1203,13 @@ main(int argc, char **argv) gen_check_duplicate_cases(); gen_check_complete_map_cases(); + // @notes Here we explicitly use one block to generate each output file. + // This approach makes it a lot easier to understand where the contents + // of generated files come from, because it's all layed out it one place. + // However if the number of output files grows, this can get out of hand + // and the situation may start calling for more automation of the output + // files. There is a large amount of judgement calling in this part! + // generate header file { FILE *h = fopen("meta_types.h", "wb"); @@ -1177,20 +1224,22 @@ main(int argc, char **argv) // generate definitions file { - // open output file FILE *c = fopen("meta_types.c", "wb"); - gen_struct_member_tables_from_types(c); gen_enum_member_tables_from_types(c); gen_type_info_definitions_from_types(c); gen_function_definitions_from_maps(c); - - // close output file fclose(c); } + + // @notes The generated code doesn't go straight to stdout, and has a lot + // of transforms applied. When writing the analyzers, it's often useful to + // have a way to directly dump the results of analysis right to stdout to + // see what's going on. + // print diagnostics of the parse analysis -#if 0 +#if 1 for (GEN_TypeInfo *type = first_type; type != 0; type = type->next) diff --git a/examples/type_metadata/type_metadata.h b/examples/type_metadata/type_metadata.h index a2a3f79..353526b 100644 --- a/examples/type_metadata/type_metadata.h +++ b/examples/type_metadata/type_metadata.h @@ -120,6 +120,8 @@ void gen_equip_map_cases(void); void gen_check_duplicate_cases(void); void gen_check_complete_map_cases(void); +// TODO: check array member indexes point to members from before the array + //~ generators //////////////////////////////////////////////////////////////// void gen_type_definitions_from_types(FILE *out); void gen_function_declarations_from_maps(FILE *out); From 2a6bf65f136310189b01a27463192dde8bd16209 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 11:44:07 -0700 Subject: [PATCH 08/28] [examples] finish type metadata commentary --- examples/type_metadata/type_metadata.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/examples/type_metadata/type_metadata.c b/examples/type_metadata/type_metadata.c index 53321e3..4cc32bf 100644 --- a/examples/type_metadata/type_metadata.c +++ b/examples/type_metadata/type_metadata.c @@ -780,8 +780,21 @@ gen_check_complete_map_cases(void) //~ generators //////////////////////////////////////////////////////////////// -// @notes Each generator function does all of the generation for one of the -// generated features we want. TODO keep going here. +// @notes Each generator function handles generating every instance of a +// particular function. This means that there is only one place where the +// generator gets called and only one place where the generated code gets +// written. This keeps things simple but may be quite limiting depending on +// the use case. +// +// For instance, to have multiple groups of types and metadata generated to +// different .h/.c file pairs, we could make the output file name a parameter +// on the command line, and run the generator multiple times with different +// metadesk files. +// +// If we wanted to support references to entities accross files we would have +// to handle all the files in one run of the generator, and the generator +// functions would need to be called with different 'entities' for different +// output files. void gen_type_definitions_from_types(FILE *out) @@ -801,7 +814,8 @@ gen_type_definitions_from_types(FILE *out) case GEN_TypeKind_Struct: { MD_String8 struct_name = type->node->string; - fprintf(out, "typedef struct %.*s %.*s;\n", MD_S8VArg(struct_name), MD_S8VArg(struct_name)); + fprintf(out, "typedef struct %.*s %.*s;\n", + MD_S8VArg(struct_name), MD_S8VArg(struct_name)); fprintf(out, "struct %.*s\n", MD_S8VArg(struct_name)); fprintf(out, "{\n"); for (GEN_TypeMember *member = type->first_member; From 5a6ad0a770c58e50c30a28162291ae57ea3af50e Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 11:59:06 -0700 Subject: [PATCH 09/28] [examples] more robust array index checker in the type example --- examples/type_metadata/generated/meta_types.c | 8 ++--- examples/type_metadata/generated/meta_types.h | 6 ++-- examples/type_metadata/type_metadata.c | 35 +++++++++++++++---- examples/type_metadata/type_metadata.h | 6 ++-- 4 files changed, 38 insertions(+), 17 deletions(-) diff --git a/examples/type_metadata/generated/meta_types.c b/examples/type_metadata/generated/meta_types.c index a5eb56c..6b477b7 100644 --- a/examples/type_metadata/generated/meta_types.c +++ b/examples/type_metadata/generated/meta_types.c @@ -1,4 +1,4 @@ -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:914 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:949 TypeInfoMember Circle_members[2] = { {"r", 1, -1, &F32_type_info}, {"pos", 3, -1, &V2F32_type_info}, @@ -14,14 +14,14 @@ TypeInfoMember RoundedPolygon_members[3] = { {"p", 1, 1, &V2F32_type_info}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:953 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:988 TypeInfoEnumerant Shape_members[3] = { {"Circle", 6, 1}, {"Segment", 7, 2}, {"Polygon", 7, 3}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:989 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1024 TypeInfo U32_type_info = {TypeKind_Basic, "U32", 3, 4, 0, 0}; TypeInfo F32_type_info = {TypeKind_Basic, "F32", 3, 4, 0, 0}; TypeInfo V2F32_type_info = {TypeKind_Basic, "V2F32", 5, 8, 0, 0}; @@ -30,7 +30,7 @@ TypeInfo RoundedSegment_type_info = {TypeKind_Struct, "RoundedSegment", 14, 3, R TypeInfo RoundedPolygon_type_info = {TypeKind_Struct, "RoundedPolygon", 14, 3, RoundedPolygon_members, 0}; TypeInfo Shape_type_info = {TypeKind_Enum, "Shape", 5, 3, Shape_members, &U32_type_info}; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1049 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1084 TypeInfo* type_info_from_shape(Shape v) { diff --git a/examples/type_metadata/generated/meta_types.h b/examples/type_metadata/generated/meta_types.h index b969a2c..d032e68 100644 --- a/examples/type_metadata/generated/meta_types.h +++ b/examples/type_metadata/generated/meta_types.h @@ -1,6 +1,6 @@ #if !defined(META_TYPES_H) #define META_TYPES_H -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:791 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:826 typedef struct Circle Circle; struct Circle { @@ -29,11 +29,11 @@ Shape_Segment = 2, Shape_Polygon = 3, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:875 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:910 TypeInfo* type_info_from_shape(Shape v); U32 max_slot_from_shape(Shape v); -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:898 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:933 extern TypeInfo U32_type_info; extern TypeInfo F32_type_info; extern TypeInfo V2F32_type_info; diff --git a/examples/type_metadata/type_metadata.c b/examples/type_metadata/type_metadata.c index 4cc32bf..4ce43df 100644 --- a/examples/type_metadata/type_metadata.c +++ b/examples/type_metadata/type_metadata.c @@ -349,7 +349,7 @@ gen_equip_struct_members(void) // resolved type: if (got_list) { - MD_Node *array_count = MD_NilNode(); + GEN_TypeMember *array_count = 0; MD_Node *array_tag = MD_TagFromString(type_name_node, MD_S8Lit("array"), 0); if (!MD_NodeIsNil(array_tag)) { @@ -362,14 +362,35 @@ gen_equip_struct_members(void) } else { - array_count = MD_ChildFromString(type_root_node, array_count_referencer->string, 0); - if (MD_NodeIsNil(array_count)) + MD_Node *array_count_member_node = + MD_ChildFromString(type_root_node, array_count_referencer->string, 0); + if (MD_NodeIsNil(array_count_member_node)) { MD_CodeLoc loc = MD_CodeLocFromNode(array_count_referencer); MD_PrintMessageFmt(error_file, loc, MD_MessageKind_Error, "'%.*s' is not a member of %.*s", MD_S8VArg(array_count_referencer->string), MD_S8VArg(type_name)); } + else + { + for (GEN_TypeMember *member_it = first_member; + member_it != 0; + member_it = member_it->next) + { + if (member_it->node == array_count_member_node) + { + array_count = member_it; + break; + } + } + if (array_count == 0) + { + MD_CodeLoc loc = MD_CodeLocFromNode(array_count_referencer); + MD_PrintMessageFmt(error_file, loc, MD_MessageKind_Error, + "'%.*s' comes after this array", + MD_S8VArg(array_count_referencer->string), MD_S8VArg(type_name)); + } + } } } @@ -377,6 +398,7 @@ gen_equip_struct_members(void) member->node = member_node; member->type = type_info; member->array_count = array_count; + member->member_index = member_count; MD_QueuePush(first_member, last_member, member); member_count += 1; } @@ -824,8 +846,7 @@ gen_type_definitions_from_types(FILE *out) { MD_String8 type_name = member->type->node->string; MD_String8 member_name = member->node->string; - int is_array = (!MD_NodeIsNil(member->array_count)); - if (is_array) + if (member->array_count != 0) { fprintf(out, "%.*s *%.*s;\n", MD_S8VArg(type_name), MD_S8VArg(member_name)); } @@ -945,9 +966,9 @@ gen_struct_member_tables_from_types(FILE *out) MD_String8 member_name = member->node->string; MD_String8 member_type_name = member->type->node->string; int array_count_member_index = -1; - if (!MD_NodeIsNil(member->array_count)) + if (member->array_count != 0) { - array_count_member_index = MD_IndexFromNode(member->array_count); + array_count_member_index = member->array_count->member_index; } fprintf(out, "{\"%.*s\", %d, %d, &%.*s_type_info},\n", MD_S8VArg(member_name), (int)member_name.size, diff --git a/examples/type_metadata/type_metadata.h b/examples/type_metadata/type_metadata.h index 353526b..5cfbc9c 100644 --- a/examples/type_metadata/type_metadata.h +++ b/examples/type_metadata/type_metadata.h @@ -47,7 +47,9 @@ struct GEN_TypeMember GEN_TypeMember *next; MD_Node *node; GEN_TypeInfo *type; - MD_Node *array_count; + struct GEN_TypeMember *array_count; + //MD_Node *array_count; + int member_index; }; typedef struct GEN_TypeEnumerant GEN_TypeEnumerant; @@ -120,8 +122,6 @@ void gen_equip_map_cases(void); void gen_check_duplicate_cases(void); void gen_check_complete_map_cases(void); -// TODO: check array member indexes point to members from before the array - //~ generators //////////////////////////////////////////////////////////////// void gen_type_definitions_from_types(FILE *out); void gen_function_declarations_from_maps(FILE *out); From ee3adcfb60f29ec2bb1c4ff7b218484f9215568a Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 16:08:24 -0700 Subject: [PATCH 10/28] [examples] input for types example that generates errors --- examples/type_metadata/bad_types.mdesk | 42 ++++++++++ examples/type_metadata/generated/meta_types.c | 79 +++---------------- examples/type_metadata/generated/meta_types.h | 40 +++------- examples/type_metadata/type_metadata.c | 7 +- examples_todo.txt | 6 +- 5 files changed, 72 insertions(+), 102 deletions(-) create mode 100644 examples/type_metadata/bad_types.mdesk diff --git a/examples/type_metadata/bad_types.mdesk b/examples/type_metadata/bad_types.mdesk new file mode 100644 index 0000000..3df46d6 --- /dev/null +++ b/examples/type_metadata/bad_types.mdesk @@ -0,0 +1,42 @@ +/* +** Example: type metadata +** +** This input for the type_metadata.c metaprogram is here to exercise some of +** the error checks in the example. +*/ + + +@type(basic) U32: 4; + + +// What is a type? +@type JustAType; + + +// Really promising: infinite precision 2D points. Just need to figure +// out how to implement the MagicInt now. +@type(struct) +MagicalPoint2D: +{ + x: MagicInt; + y: MagicInt; +} + + +// If we serialize the count of an array after the array, how do we find the +// count? +@type(struct) +BackwardsArray: +{ + vals: @array(count) U32; + count: U32; +} + + +// These two just won't play nice together. +@type(basic) TwinType: 4; +@type(struct) +TwinType: +{ + val: U32; +} diff --git a/examples/type_metadata/generated/meta_types.c b/examples/type_metadata/generated/meta_types.c index 6b477b7..f49ad16 100644 --- a/examples/type_metadata/generated/meta_types.c +++ b/examples/type_metadata/generated/meta_types.c @@ -1,81 +1,22 @@ // generated by W:/metadesk/examples/type_metadata/type_metadata.c:949 -TypeInfoMember Circle_members[2] = { -{"r", 1, -1, &F32_type_info}, -{"pos", 3, -1, &V2F32_type_info}, +TypeInfoMember MagicalPoint2D_members[0] = { }; -TypeInfoMember RoundedSegment_members[3] = { -{"r", 1, -1, &F32_type_info}, -{"p1", 2, -1, &V2F32_type_info}, -{"p2", 2, -1, &V2F32_type_info}, -}; -TypeInfoMember RoundedPolygon_members[3] = { -{"r", 1, -1, &F32_type_info}, +TypeInfoMember BackwardsArray_members[2] = { +{"vals", 4, -1, &U32_type_info}, {"count", 5, -1, &U32_type_info}, -{"p", 1, 1, &V2F32_type_info}, +}; +TypeInfoMember TwinType_members[1] = { +{"val", 3, -1, &U32_type_info}, }; // generated by W:/metadesk/examples/type_metadata/type_metadata.c:988 -TypeInfoEnumerant Shape_members[3] = { -{"Circle", 6, 1}, -{"Segment", 7, 2}, -{"Polygon", 7, 3}, -}; // generated by W:/metadesk/examples/type_metadata/type_metadata.c:1024 TypeInfo U32_type_info = {TypeKind_Basic, "U32", 3, 4, 0, 0}; -TypeInfo F32_type_info = {TypeKind_Basic, "F32", 3, 4, 0, 0}; -TypeInfo V2F32_type_info = {TypeKind_Basic, "V2F32", 5, 8, 0, 0}; -TypeInfo Circle_type_info = {TypeKind_Struct, "Circle", 6, 2, Circle_members, 0}; -TypeInfo RoundedSegment_type_info = {TypeKind_Struct, "RoundedSegment", 14, 3, RoundedSegment_members, 0}; -TypeInfo RoundedPolygon_type_info = {TypeKind_Struct, "RoundedPolygon", 14, 3, RoundedPolygon_members, 0}; -TypeInfo Shape_type_info = {TypeKind_Enum, "Shape", 5, 3, Shape_members, &U32_type_info}; +TypeInfo MagicalPoint2D_type_info = {TypeKind_Struct, "MagicalPoint2D", 14, 0, MagicalPoint2D_members, 0}; +TypeInfo BackwardsArray_type_info = {TypeKind_Struct, "BackwardsArray", 14, 2, BackwardsArray_members, 0}; +TypeInfo TwinType_type_info = {TypeKind_Basic, "TwinType", 8, 4, 0, 0}; +TypeInfo TwinType_type_info = {TypeKind_Struct, "TwinType", 8, 1, TwinType_members, 0}; // generated by W:/metadesk/examples/type_metadata/type_metadata.c:1084 -TypeInfo* -type_info_from_shape(Shape v) -{ -TypeInfo* result; -switch (v) -{ -default: -{ -result = 0; -}break; -case Shape_Circle: -{ -result = &Circle_type_info; -}break; -case Shape_Segment: -{ -result = &RoundedSegment_type_info; -}break; -case Shape_Polygon: -{ -result = &RoundedPolygon_type_info; -}break; -} -return(result); -} -U32 -max_slot_from_shape(Shape v) -{ -U32 result; -switch (v) -{ -default: -{ -result = 0; -}break; -case Shape_Circle: -case Shape_Segment: -{ -result = 64; -}break; -case Shape_Polygon: -{ -result = 12; -}break; -} -return(result); -} diff --git a/examples/type_metadata/generated/meta_types.h b/examples/type_metadata/generated/meta_types.h index d032e68..665118f 100644 --- a/examples/type_metadata/generated/meta_types.h +++ b/examples/type_metadata/generated/meta_types.h @@ -1,45 +1,29 @@ #if !defined(META_TYPES_H) #define META_TYPES_H // generated by W:/metadesk/examples/type_metadata/type_metadata.c:826 -typedef struct Circle Circle; -struct Circle +typedef struct MagicalPoint2D MagicalPoint2D; +struct MagicalPoint2D { -F32 r; -V2F32 pos; }; -typedef struct RoundedSegment RoundedSegment; -struct RoundedSegment +typedef struct BackwardsArray BackwardsArray; +struct BackwardsArray { -F32 r; -V2F32 p1; -V2F32 p2; -}; -typedef struct RoundedPolygon RoundedPolygon; -struct RoundedPolygon -{ -F32 r; +U32 vals; U32 count; -V2F32 *p; }; -typedef U32 Shape; -enum +typedef struct TwinType TwinType; +struct TwinType { -Shape_Circle = 1, -Shape_Segment = 2, -Shape_Polygon = 3, +U32 val; }; // generated by W:/metadesk/examples/type_metadata/type_metadata.c:910 -TypeInfo* type_info_from_shape(Shape v); -U32 max_slot_from_shape(Shape v); // generated by W:/metadesk/examples/type_metadata/type_metadata.c:933 extern TypeInfo U32_type_info; -extern TypeInfo F32_type_info; -extern TypeInfo V2F32_type_info; -extern TypeInfo Circle_type_info; -extern TypeInfo RoundedSegment_type_info; -extern TypeInfo RoundedPolygon_type_info; -extern TypeInfo Shape_type_info; +extern TypeInfo MagicalPoint2D_type_info; +extern TypeInfo BackwardsArray_type_info; +extern TypeInfo TwinType_type_info; +extern TypeInfo TwinType_type_info; #endif // META_TYPES_H diff --git a/examples/type_metadata/type_metadata.c b/examples/type_metadata/type_metadata.c index 4ce43df..ae795a3 100644 --- a/examples/type_metadata/type_metadata.c +++ b/examples/type_metadata/type_metadata.c @@ -39,6 +39,7 @@ ** types.mdesk - Sample input for this metaprogram. The code generated ** from this metadesk file will be necessary to make ** type_info_final_program.c compile. +** bad_types.mdesk - Sample input for seeing some of the error checking work. ** */ @@ -198,7 +199,7 @@ gen_gather_types_and_maps(MD_Node *list) { MD_CodeLoc loc = MD_CodeLocFromNode(node); MD_PrintMessageFmt(error_file, loc, MD_MessageKind_Error, - "Unrecognized type kind '%.*s'\n", + "Unrecognized type kind '%.*s'", MD_S8VArg(tag_arg_str)); } else @@ -1188,12 +1189,14 @@ gen_function_definitions_from_maps(FILE *out) int main(int argc, char **argv) { +#if 1 char *argv_dummy[2] = { 0, - "W:/metadesk/examples/type_metadata/types.mdesk" + "W:/metadesk/examples/type_metadata/bad_types.mdesk" }; argc = 2; argv = argv_dummy; +#endif // setup the global arena arena = MD_ArenaAlloc(); diff --git a/examples_todo.txt b/examples_todo.txt index 202b81e..7c8e76c 100644 --- a/examples_todo.txt +++ b/examples_todo.txt @@ -3,10 +3,10 @@ Example Programs: [x] Metadesk parse checker [x] User error checking [x] Data Desk like setup -[ ] Example type metadata +[x] Example type metadata [x] Cleanup & Simplification Pass - [ ] Commentary - [ ] Example Type File With Errors + [x] Commentary + [x] Example Type File With Errors [ ] Example of simple expression parser [ ] Example of C-like expression parser (with value and type expressions) [ ] Example(s) of using overrides From 670f5592f7335118ea690e9cfb3def074fa874ba Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 19:13:27 -0700 Subject: [PATCH 11/28] [examples] final pass over todos in the type metaprogram --- examples/type_metadata/generated/meta_types.c | 87 ++++++++++++++++--- examples/type_metadata/generated/meta_types.h | 46 ++++++---- examples/type_metadata/type_metadata.c | 74 +++++++++++----- examples/type_metadata/type_metadata.h | 8 +- 4 files changed, 164 insertions(+), 51 deletions(-) diff --git a/examples/type_metadata/generated/meta_types.c b/examples/type_metadata/generated/meta_types.c index f49ad16..67ac7c7 100644 --- a/examples/type_metadata/generated/meta_types.c +++ b/examples/type_metadata/generated/meta_types.c @@ -1,22 +1,81 @@ -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:949 -TypeInfoMember MagicalPoint2D_members[0] = { +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:993 +TypeInfoMember Circle_members[2] = { +{"r", 1, -1, &F32_type_info}, +{"pos", 3, -1, &V2F32_type_info}, }; -TypeInfoMember BackwardsArray_members[2] = { -{"vals", 4, -1, &U32_type_info}, +TypeInfoMember RoundedSegment_members[3] = { +{"r", 1, -1, &F32_type_info}, +{"p1", 2, -1, &V2F32_type_info}, +{"p2", 2, -1, &V2F32_type_info}, +}; +TypeInfoMember RoundedPolygon_members[3] = { +{"r", 1, -1, &F32_type_info}, {"count", 5, -1, &U32_type_info}, -}; -TypeInfoMember TwinType_members[1] = { -{"val", 3, -1, &U32_type_info}, +{"p", 1, 1, &V2F32_type_info}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:988 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1032 +TypeInfoEnumerant Shape_members[3] = { +{"Circle", 6, 1}, +{"Segment", 7, 2}, +{"Polygon", 7, 3}, +}; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1024 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1068 TypeInfo U32_type_info = {TypeKind_Basic, "U32", 3, 4, 0, 0}; -TypeInfo MagicalPoint2D_type_info = {TypeKind_Struct, "MagicalPoint2D", 14, 0, MagicalPoint2D_members, 0}; -TypeInfo BackwardsArray_type_info = {TypeKind_Struct, "BackwardsArray", 14, 2, BackwardsArray_members, 0}; -TypeInfo TwinType_type_info = {TypeKind_Basic, "TwinType", 8, 4, 0, 0}; -TypeInfo TwinType_type_info = {TypeKind_Struct, "TwinType", 8, 1, TwinType_members, 0}; +TypeInfo F32_type_info = {TypeKind_Basic, "F32", 3, 4, 0, 0}; +TypeInfo V2F32_type_info = {TypeKind_Basic, "V2F32", 5, 8, 0, 0}; +TypeInfo Circle_type_info = {TypeKind_Struct, "Circle", 6, 2, Circle_members, 0}; +TypeInfo RoundedSegment_type_info = {TypeKind_Struct, "RoundedSegment", 14, 3, RoundedSegment_members, 0}; +TypeInfo RoundedPolygon_type_info = {TypeKind_Struct, "RoundedPolygon", 14, 3, RoundedPolygon_members, 0}; +TypeInfo Shape_type_info = {TypeKind_Enum, "Shape", 5, 3, Shape_members, &U32_type_info}; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1084 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1128 +TypeInfo* +type_info_from_shape(Shape v) +{ +TypeInfo* result; +switch (v) +{ +default: +{ +result = 0; +}break; +case Shape_Circle: +{ +result = &Circle_type_info; +}break; +case Shape_Segment: +{ +result = &RoundedSegment_type_info; +}break; +case Shape_Polygon: +{ +result = &RoundedPolygon_type_info; +}break; +} +return(result); +} +U32 +max_slot_from_shape(Shape v) +{ +U32 result; +switch (v) +{ +default: +{ +result = 0; +}break; +case Shape_Circle: +case Shape_Segment: +{ +result = 64; +}break; +case Shape_Polygon: +{ +result = 12; +}break; +} +return(result); +} diff --git a/examples/type_metadata/generated/meta_types.h b/examples/type_metadata/generated/meta_types.h index 665118f..8251a53 100644 --- a/examples/type_metadata/generated/meta_types.h +++ b/examples/type_metadata/generated/meta_types.h @@ -1,29 +1,45 @@ #if !defined(META_TYPES_H) #define META_TYPES_H -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:826 -typedef struct MagicalPoint2D MagicalPoint2D; -struct MagicalPoint2D +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:870 +typedef struct Circle Circle; +struct Circle { +F32 r; +V2F32 pos; }; -typedef struct BackwardsArray BackwardsArray; -struct BackwardsArray +typedef struct RoundedSegment RoundedSegment; +struct RoundedSegment { -U32 vals; +F32 r; +V2F32 p1; +V2F32 p2; +}; +typedef struct RoundedPolygon RoundedPolygon; +struct RoundedPolygon +{ +F32 r; U32 count; +V2F32 *p; }; -typedef struct TwinType TwinType; -struct TwinType +typedef U32 Shape; +enum { -U32 val; +Shape_Circle = 1, +Shape_Segment = 2, +Shape_Polygon = 3, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:910 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:954 +TypeInfo* type_info_from_shape(Shape v); +U32 max_slot_from_shape(Shape v); -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:933 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:977 extern TypeInfo U32_type_info; -extern TypeInfo MagicalPoint2D_type_info; -extern TypeInfo BackwardsArray_type_info; -extern TypeInfo TwinType_type_info; -extern TypeInfo TwinType_type_info; +extern TypeInfo F32_type_info; +extern TypeInfo V2F32_type_info; +extern TypeInfo Circle_type_info; +extern TypeInfo RoundedSegment_type_info; +extern TypeInfo RoundedPolygon_type_info; +extern TypeInfo Shape_type_info; #endif // META_TYPES_H diff --git a/examples/type_metadata/type_metadata.c b/examples/type_metadata/type_metadata.c index ae795a3..56c34f6 100644 --- a/examples/type_metadata/type_metadata.c +++ b/examples/type_metadata/type_metadata.c @@ -64,8 +64,6 @@ FILE *error_file = 0; // We also use the MD_Map helper to create a string -> pointer mapping so that // we can look up the "processed" info pointers by name after the initial // gather stage. -// -// TODO notes on managing output files. GEN_TypeInfo *first_type = 0; GEN_TypeInfo *last_type = 0; @@ -111,14 +109,6 @@ gen_resolve_type_info_from_referencer(MD_Node *reference) return(result); } -void -gen_type_resolve_error(MD_Node *reference) -{ - MD_CodeLoc loc = MD_CodeLocFromNode(reference); - MD_PrintMessageFmt(error_file, loc, MD_MessageKind_Error, - "could not resolve type name '%.*s'", MD_S8VArg(reference->string)); -} - GEN_TypeEnumerant* gen_enumerant_from_name(GEN_TypeInfo *enum_type, MD_String8 name) { @@ -153,6 +143,57 @@ gen_map_case_from_enumerant(GEN_MapInfo *map, GEN_TypeEnumerant *enumerant) return(result); } +MD_Node* +gen_get_symbol_md_node_by_name(MD_String8 name) +{ + MD_Node *result = MD_NilNode(); + MD_MapSlot *type_slot = MD_MapLookup(&type_map, MD_MapKeyStr(name)); + if (type_slot != 0) + { + GEN_TypeInfo *type_info = (GEN_TypeInfo*)type_slot->val; + result = type_info->node; + } + MD_MapSlot *map_slot = MD_MapLookup(&map_map, MD_MapKeyStr(name)); + if (map_slot != 0) + { + GEN_MapInfo *map_info = (GEN_MapInfo*)map_slot->val; + result = map_info->node; + } + return(result); +} + +void +gen_type_resolve_error(MD_Node *reference) +{ + MD_CodeLoc loc = MD_CodeLocFromNode(reference); + MD_PrintMessageFmt(error_file, loc, MD_MessageKind_Error, + "could not resolve type name '%.*s'", MD_S8VArg(reference->string)); +} + +void +gen_duplicate_symbol_error(MD_Node *new_node, MD_Node *existing_node) +{ + MD_CodeLoc loc = MD_CodeLocFromNode(new_node); + MD_PrintMessageFmt(error_file, loc, MD_MessageKind_Error, + "Symbol name '%.*s' is already used", + MD_S8VArg(new_node->string)); + MD_CodeLoc existing_loc = MD_CodeLocFromNode(existing_node); + MD_PrintMessageFmt(error_file, existing_loc, MD_MessageKind_Note, + "See '%.*s' is already used", + MD_S8VArg(existing_node->string)); +} + +void +gen_check_and_do_duplicate_symbol_error(MD_Node *new_node) +{ + MD_Node *existing = gen_get_symbol_md_node_by_name(new_node->string); + if (!MD_NodeIsNil(existing)) + { + gen_duplicate_symbol_error(new_node, existing); + } +} + + //~ analyzers ///////////////////////////////////////////////////////////////// // @notes The first stage of processing is to loop over the top level nodes @@ -177,7 +218,7 @@ gen_gather_types_and_maps(MD_Node *list) if (!MD_NodeIsNil(type_tag)) { - // TODO(allen): check for duplicates + gen_check_and_do_duplicate_symbol_error(node); GEN_TypeKind kind = GEN_TypeKind_Null; MD_Node *tag_arg_node = type_tag->first_child; @@ -215,7 +256,7 @@ gen_gather_types_and_maps(MD_Node *list) // gather map if (MD_NodeHasTag(node, MD_S8Lit("map"), 0)) { - // TODO(allen): check for duplicates + gen_check_and_do_duplicate_symbol_error(node); GEN_MapInfo *map_info = MD_PushArrayZero(arena, GEN_MapInfo, 1); map_info->node = node; @@ -1189,15 +1230,6 @@ gen_function_definitions_from_maps(FILE *out) int main(int argc, char **argv) { -#if 1 - char *argv_dummy[2] = { - 0, - "W:/metadesk/examples/type_metadata/bad_types.mdesk" - }; - argc = 2; - argv = argv_dummy; -#endif - // setup the global arena arena = MD_ArenaAlloc(); diff --git a/examples/type_metadata/type_metadata.h b/examples/type_metadata/type_metadata.h index 5cfbc9c..90f1bf3 100644 --- a/examples/type_metadata/type_metadata.h +++ b/examples/type_metadata/type_metadata.h @@ -104,12 +104,18 @@ MD_Node* gen_get_child_value(MD_Node *parent, MD_String8 child_name); GEN_TypeInfo* gen_resolve_type_info_from_string(MD_String8 name); GEN_TypeInfo* gen_resolve_type_info_from_referencer(MD_Node *reference); -void gen_type_resolve_error(MD_Node *reference); GEN_TypeEnumerant* gen_enumerant_from_name(GEN_TypeInfo *enum_type, MD_String8 name); GEN_MapCase* gen_map_case_from_enumerant(GEN_MapInfo *map, GEN_TypeEnumerant *enumerant); +MD_Node* gen_get_symbol_md_node_by_name(MD_String8 name); + +void gen_type_resolve_error(MD_Node *reference); +void gen_duplicate_symbol_error(MD_Node *new_node, MD_Node *existing_node); + +void gen_check_and_do_duplicate_symbol_error(MD_Node *new_node); + //~ analyzers ///////////////////////////////////////////////////////////////// void gen_gather_types_and_maps(MD_Node *list); void gen_check_duplicate_member_names(void); From 838283c70654d5eb31a6a488e2719888f784fb83 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 20:14:07 -0700 Subject: [PATCH 12/28] [examples] start setting up expression parser intro --- examples/expr/expr_intro.c | 92 ++++++++++++++++++++++++++++++++++ examples/expr/expr_intro.mdesk | 14 ++++++ source/md.h | 6 ++- 3 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 examples/expr/expr_intro.c create mode 100644 examples/expr/expr_intro.mdesk diff --git a/examples/expr/expr_intro.c b/examples/expr/expr_intro.c new file mode 100644 index 0000000..70b33af --- /dev/null +++ b/examples/expr/expr_intro.c @@ -0,0 +1,92 @@ +/* +** Example: expressions intro +** +** TODO +** +*/ + +//~ includes and globals ////////////////////////////////////////////////////// + +#include "md.h" +#include "md.c" + +static MD_Arena *arena = 0; + +//~ expression setup and helpers ////////////////////////////////////////////// + +void +print_expression(FILE *out, MD_Expr *expr) +{ + // TODO(allen): finish +} + + +//~ main ////////////////////////////////////////////////////////////////////// + +int main(int argc, char **argv) +{ + // setup the global arena + arena = MD_ArenaAlloc(); + + // parse all files passed to the command line + MD_Node *list = MD_MakeList(arena); + for (int i = 1; i < argc; i += 1) + { + + // parse the file + MD_String8 file_name = MD_S8CString(argv[i]); + MD_ParseResult parse_result = MD_ParseWholeFile(arena, file_name); + + // print metadesk errors + for (MD_Message *message = parse_result.errors.first; + message != 0; + message = message->next) + { + MD_CodeLoc code_loc = MD_CodeLocFromNode(message->node); + MD_PrintMessage(stdout, code_loc, message->kind, message->string); + } + + // save to parse results list + if (parse_result.errors.max_message_kind < MD_MessageKind_Error) + { + MD_PushNewReference(arena, list, parse_result.node); + } + } + + // setup the expression system + MD_ExprOprTable table = {0}; + { + MD_ExprOprList list = {0}; + + MD_ExprOprPush(arena, &list, MD_ExprOprKind_Binary, 1, MD_S8Lit("+"), OpAdd, 0); + MD_ExprOprPush(arena, &list, MD_ExprOprKind_Binary, 2, MD_S8Lit("*"), OpMul, 0); + + table = MD_ExprBakeOperatorTableFromList(arena, &list); + } + + // print the verbose parse results + for (MD_EachNode(root_it, list->first_child)) + { + MD_Node *root = MD_ResolveNodeFromReference(root_it); + for (MD_EachNode(node, root->first_child)) + { + MD_ExprParseResult parse = MD_ExprParse(arena, &table, node->first_child, MD_NilNode()); + + // print errors + for (MD_Message *message = parse.errors.first; + message != 0; + message = message->next) + { + MD_CodeLoc code_loc = MD_CodeLocFromNode(message->node); + MD_PrintMessage(stdout, code_loc, message->kind, message->string); + } + + // print the expression + fprintf(stdout, "%s = "); + print_expression(stdout, parse.expr); + fprintf(stdout, ";\n"); + } + } + + return 0; +} diff --git a/examples/expr/expr_intro.mdesk b/examples/expr/expr_intro.mdesk new file mode 100644 index 0000000..90cc1fc --- /dev/null +++ b/examples/expr/expr_intro.mdesk @@ -0,0 +1,14 @@ +/* +** Example: expressions intro +** +** TODO +*/ + +a: 1; +b: 2; +c: 3; + +w: 100; +x: a*w + b; +y: b*w + c; +z: c*w + a; diff --git a/source/md.h b/source/md.h index 970eb32..5c839a6 100644 --- a/source/md.h +++ b/source/md.h @@ -727,6 +727,7 @@ typedef struct MD_MessageList MD_MessageList; struct MD_MessageList { MD_MessageKind max_message_kind; + // TODO(allen): rename MD_u64 node_count; MD_Message *first; MD_Message *last; @@ -780,7 +781,8 @@ struct MD_ExprOprList typedef struct MD_ExprOprTable MD_ExprOprTable; struct MD_ExprOprTable { - MD_ExprOprList table[MD_ExprOprKind_COUNT]; // TODO(mal): Hash? + // TODO(mal): @upgrade_potential Hash? + MD_ExprOprList table[MD_ExprOprKind_COUNT]; MD_MessageList errors; }; @@ -791,6 +793,7 @@ struct MD_Expr struct MD_Expr *left; struct MD_Expr *right; MD_b32 is_op; + // TODO(allen): this should be MD_ExprOpr* MD_u32 op_id; void *op_ptr; MD_Node *md_node; @@ -818,6 +821,7 @@ struct MD_ExprParseCtx } accel; #undef MD_POSTFIX_SETLIKE_OP_COUNT + // TODO(allen): different message list type here MD_MessageList errors; }; From 39e10cd246abf7f3763ca1a0ee961d0c5225aa3e Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 20:30:05 -0700 Subject: [PATCH 13/28] [expression parser] use the operator pointer right inside the expression node --- examples/type_metadata/generated/meta_types.c | 8 ++++---- examples/type_metadata/generated/meta_types.h | 6 +++--- source/md.c | 4 +--- source/md.h | 5 +---- tests/expression_tests.c | 20 +++++++++---------- 5 files changed, 19 insertions(+), 24 deletions(-) diff --git a/examples/type_metadata/generated/meta_types.c b/examples/type_metadata/generated/meta_types.c index 67ac7c7..448dbd5 100644 --- a/examples/type_metadata/generated/meta_types.c +++ b/examples/type_metadata/generated/meta_types.c @@ -1,4 +1,4 @@ -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:993 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:991 TypeInfoMember Circle_members[2] = { {"r", 1, -1, &F32_type_info}, {"pos", 3, -1, &V2F32_type_info}, @@ -14,14 +14,14 @@ TypeInfoMember RoundedPolygon_members[3] = { {"p", 1, 1, &V2F32_type_info}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1032 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1030 TypeInfoEnumerant Shape_members[3] = { {"Circle", 6, 1}, {"Segment", 7, 2}, {"Polygon", 7, 3}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1068 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1066 TypeInfo U32_type_info = {TypeKind_Basic, "U32", 3, 4, 0, 0}; TypeInfo F32_type_info = {TypeKind_Basic, "F32", 3, 4, 0, 0}; TypeInfo V2F32_type_info = {TypeKind_Basic, "V2F32", 5, 8, 0, 0}; @@ -30,7 +30,7 @@ TypeInfo RoundedSegment_type_info = {TypeKind_Struct, "RoundedSegment", 14, 3, R TypeInfo RoundedPolygon_type_info = {TypeKind_Struct, "RoundedPolygon", 14, 3, RoundedPolygon_members, 0}; TypeInfo Shape_type_info = {TypeKind_Enum, "Shape", 5, 3, Shape_members, &U32_type_info}; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1128 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1126 TypeInfo* type_info_from_shape(Shape v) { diff --git a/examples/type_metadata/generated/meta_types.h b/examples/type_metadata/generated/meta_types.h index 8251a53..789fcea 100644 --- a/examples/type_metadata/generated/meta_types.h +++ b/examples/type_metadata/generated/meta_types.h @@ -1,6 +1,6 @@ #if !defined(META_TYPES_H) #define META_TYPES_H -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:870 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:868 typedef struct Circle Circle; struct Circle { @@ -29,11 +29,11 @@ Shape_Segment = 2, Shape_Polygon = 3, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:954 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:952 TypeInfo* type_info_from_shape(Shape v); U32 max_slot_from_shape(Shape v); -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:977 +// generated by W:/metadesk/examples/type_metadata/type_metadata.c:975 extern TypeInfo U32_type_info; extern TypeInfo F32_type_info; extern TypeInfo V2F32_type_info; diff --git a/source/md.c b/source/md.c index 84f1da2..9e17ed8 100644 --- a/source/md.c +++ b/source/md.c @@ -3530,9 +3530,7 @@ MD_FUNCTION MD_Expr* MD_Expr_NewOp(MD_Arena *arena, MD_ExprOpr *op, MD_Node *op_node, MD_Expr *l, MD_Expr *r) { MD_Expr *result = MD_PushArrayZero(arena, MD_Expr, 1); - result->is_op = 1; - result->op_id = op->op_id; - result->op_ptr = op->op_ptr; + result->op = op; result->md_node = op_node; result->parent = 0; result->left = l; diff --git a/source/md.h b/source/md.h index 5c839a6..28cff5e 100644 --- a/source/md.h +++ b/source/md.h @@ -792,10 +792,7 @@ struct MD_Expr struct MD_Expr *parent; struct MD_Expr *left; struct MD_Expr *right; - MD_b32 is_op; - // TODO(allen): this should be MD_ExprOpr* - MD_u32 op_id; - void *op_ptr; + MD_ExprOpr *op; MD_Node *md_node; }; diff --git a/tests/expression_tests.c b/tests/expression_tests.c index 7878d57..c4b7734 100644 --- a/tests/expression_tests.c +++ b/tests/expression_tests.c @@ -86,7 +86,7 @@ X(PostfixParenBracket, "(]", Postfix, 18) \ typedef enum{ Op_Null, OPERATORS - Op_COUNT + Op_COUNT } Op; #undef X @@ -123,14 +123,14 @@ static MD_String8 node_raw_contents(MD_Node *node, MD_b32 exclude_outer) static void parenthesize_exclude_outer(MD_Arena *arena, OperatorDescription *descs, MD_String8List *l, MD_Expr *node, MD_b32 exclude_outer_parens) { - if(node->is_op) + if(node->op != 0) { if(!exclude_outer_parens) { MD_S8ListPush(arena, l, MD_S8Lit("(")); } - MD_ExprOpr *op = &descs[node->op_id].op; + MD_ExprOpr *op = node->op; if(op->kind == MD_ExprOprKind_Binary || op->kind == MD_ExprOprKind_BinaryRightAssociative) { parenthesize_exclude_outer(arena, descs, l, node->left, 0); @@ -154,7 +154,7 @@ static void parenthesize_exclude_outer(MD_Arena *arena, OperatorDescription *des else if(op->kind == MD_ExprOprKind_Postfix) { parenthesize_exclude_outer(arena, descs, l, node->left, 0); - + MD_String8 op_s = descs[op->op_id].s; if(MD_S8Match(op_s, MD_S8Lit("()"), 0) || MD_S8Match(op_s, MD_S8Lit("[]"), 0) || MD_S8Match(op_s, MD_S8Lit("{}"), 0) || MD_S8Match(op_s, MD_S8Lit("[)"), 0) || @@ -199,9 +199,9 @@ static void parenthesize_exclude_outer(MD_Arena *arena, OperatorDescription *des { MD_S8ListPush(arena, l, MD_S8Lit("[")); } - + MD_S8ListPush(arena, l, MD_S8Lit("...")); - + if(node->md_node->flags & MD_NodeFlag_HasParenRight) { MD_S8ListPush(arena, l, MD_S8Lit(")")); @@ -235,7 +235,7 @@ operator_array[Op_##name].op = (MD_ExprOpr){ .op_id = Op_##name, .kind = MD_Expr #undef X arena = MD_ArenaAlloc(); - + /* NOTE: Operator table bake errors */ { MD_ExprOprList operator_list = {0}; @@ -257,7 +257,7 @@ operator_array[Op_##name].op = (MD_ExprOpr){ .op_id = Op_##name, .kind = MD_Expr MD_Assert(op_table.errors.max_message_kind == MD_MessageKind_Warning && op_table.errors.node_count == 1 && op_table.errors.first->user_ptr == plus_node); - + // NOTE: () not as unary postfix operator_list = (MD_ExprOprList){0}; MD_ExprOprPush(arena, &operator_list, MD_ExprOprKind_Prefix, 1, MD_S8Lit("()"), @@ -308,7 +308,7 @@ operator_array[Op_##name].op = (MD_ExprOpr){ .op_id = Op_##name, .kind = MD_Expr MD_Assert(op_table.errors.max_message_kind == MD_MessageKind_Warning && op_table.errors.node_count == 1 && op_table.errors.first->user_ptr == plus_node); - + // NOTE: Wrong token kind operator operator_list = (MD_ExprOprList){0}; MD_ExprOprPush(arena, &operator_list, MD_ExprOprKind_Prefix, 1, MD_S8Lit("123"), @@ -384,7 +384,7 @@ operator_array[Op_##name].op = (MD_ExprOpr){ .op_id = Op_##name, .kind = MD_Expr { .q = "\"a\" + b + c", .a = "(\"a\" + b) + c" }, { .q = "a{b+c}", .a = "a{...}" }, // NOTE(mal): Non-standard postfix set-like operators { .q = "a[b+c)", .a = "a[...)" }, - + { .q = "(a", .a = "", ExpressionErrorKind_MD, 0}, { .q = "a)", .a = "", ExpressionErrorKind_MD, 1}, { .q = "/a", .a = "", ExpressionErrorKind_Expr, 0}, From ac9f415180c86a29fb610565c8f86f3d22bca468 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 20:58:33 -0700 Subject: [PATCH 14/28] [examples] write expressions intro example --- bin/build_examples.sh | 4 ++++ bin/run_examples.sh | 6 ++++++ examples/expr/expr_intro.c | 33 +++++++++++++++++++++++++++++++-- tests/expression_tests.c | 8 +++++--- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/bin/build_examples.sh b/bin/build_examples.sh index 92eee3c..414f54a 100755 --- a/bin/build_examples.sh +++ b/bin/build_examples.sh @@ -11,6 +11,10 @@ bin/bld_core.sh show_ctx examps="examples" +bin/bld_core.sh unit expr_intro $examps/expr/expr_intro.c + +exit + bin/bld_core.sh unit hello_world $examps/intro/hello_world.c bin/bld_core.sh unit parse_check $examps/intro/parse_check.c bin/bld_core.sh unit data_desk_like $examps/intro/data_desk_like_template.c diff --git a/bin/run_examples.sh b/bin/run_examples.sh index 331a8c3..8756617 100755 --- a/bin/run_examples.sh +++ b/bin/run_examples.sh @@ -8,6 +8,12 @@ root_path=$PWD build_path=$root_path/build examps=$root_path/examples +echo ~~~ Running Expression Intro ~~~ +$build_path/expr_intro.exe $examps/expr/expr_intro.mdesk +echo + +exit + echo ~~~ Running Type Metadata Generator Example ~~~ cd $examps/type_metadata/generated $build_path/type_metadata.exe $examps/type_metadata/types.mdesk diff --git a/examples/expr/expr_intro.c b/examples/expr/expr_intro.c index 70b33af..66966e7 100644 --- a/examples/expr/expr_intro.c +++ b/examples/expr/expr_intro.c @@ -14,10 +14,39 @@ static MD_Arena *arena = 0; //~ expression setup and helpers ////////////////////////////////////////////// +enum +{ + OpAdd, + OpMul, +}; + void print_expression(FILE *out, MD_Expr *expr) { - // TODO(allen): finish + MD_ExprOpr *op = expr->op; + if (op == 0) + { + MD_Node *node = expr->md_node; + if (node->raw_string.size != 0 && + MD_NodeIsNil(node->first_child)) + { + fprintf(out, "%.*s", MD_S8VArg(node->raw_string)); + } + else + { + MD_CodeLoc loc = MD_CodeLocFromNode(node); + MD_PrintMessage(stderr, loc, MD_MessageKind_Error, + MD_S8Lit("the expression system does not expect this kind of node")); + } + } + else + { + fprintf(out, "("); + print_expression(out, expr->left); + fprintf(out, " %.*s ", MD_S8VArg(op->string)); + print_expression(out, expr->right); + fprintf(out, ")"); + } } @@ -82,7 +111,7 @@ int main(int argc, char **argv) } // print the expression - fprintf(stdout, "%s = "); + fprintf(stdout, "%.*s = ", MD_S8VArg(node->string)); print_expression(stdout, parse.expr); fprintf(stdout, ";\n"); } diff --git a/tests/expression_tests.c b/tests/expression_tests.c index c4b7734..9de772c 100644 --- a/tests/expression_tests.c +++ b/tests/expression_tests.c @@ -160,9 +160,11 @@ static void parenthesize_exclude_outer(MD_Arena *arena, OperatorDescription *des MD_S8Match(op_s, MD_S8Lit("{}"), 0) || MD_S8Match(op_s, MD_S8Lit("[)"), 0) || MD_S8Match(op_s, MD_S8Lit("(]"), 0)) { - MD_S8ListPush(arena, l, MD_S8Substring(op_s, 0, 1)); - MD_S8ListPush(arena, l, MD_S8Lit("...")); - MD_S8ListPush(arena, l, MD_S8Substring(op_s, 1, 2)); + MD_u8 buf[5]; + buf[0] = op_s.str[0]; + buf[1] = buf[2] = buf[3] = '.'; + buf[4] = op_s.str[1]; + MD_S8ListPush(arena, l, MD_S8(buf, sizeof(buf))); } else { From 126fab22df89d5028dc73c3e78fb184c53ec7b35 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 22:07:52 -0700 Subject: [PATCH 15/28] [examples] c like expressions example --- bin/build_examples.sh | 1 + bin/run_examples.sh | 4 + examples/expr/expr_c_like.c | 259 ++++++++++++++++++++++++++++++++ examples/expr/expr_c_like.mdesk | 18 +++ examples/expr/expr_intro.c | 5 +- examples/expr/expr_intro.mdesk | 6 +- source/md.h | 6 +- tests/expression_tests.c | 26 ++-- 8 files changed, 305 insertions(+), 20 deletions(-) create mode 100644 examples/expr/expr_c_like.c create mode 100644 examples/expr/expr_c_like.mdesk diff --git a/bin/build_examples.sh b/bin/build_examples.sh index 414f54a..2bd4f5a 100755 --- a/bin/build_examples.sh +++ b/bin/build_examples.sh @@ -12,6 +12,7 @@ bin/bld_core.sh show_ctx examps="examples" bin/bld_core.sh unit expr_intro $examps/expr/expr_intro.c +bin/bld_core.sh unit expr_c_like $examps/expr/expr_c_like.c exit diff --git a/bin/run_examples.sh b/bin/run_examples.sh index 8756617..c507880 100755 --- a/bin/run_examples.sh +++ b/bin/run_examples.sh @@ -12,6 +12,10 @@ echo ~~~ Running Expression Intro ~~~ $build_path/expr_intro.exe $examps/expr/expr_intro.mdesk echo +echo ~~~ Running C Like Expression ~~~ +$build_path/expr_c_like.exe $examps/expr/expr_c_like.mdesk +echo + exit echo ~~~ Running Type Metadata Generator Example ~~~ diff --git a/examples/expr/expr_c_like.c b/examples/expr/expr_c_like.c new file mode 100644 index 0000000..44bcd1a --- /dev/null +++ b/examples/expr/expr_c_like.c @@ -0,0 +1,259 @@ +/* +** Example: c like expressions +** +** TODO +** +*/ + +//~ includes and globals ////////////////////////////////////////////////////// + +#include "md.h" +#include "md.c" + +static MD_Arena *arena = 0; + +//~ expression setup and helpers ////////////////////////////////////////////// + +#define C_LIKE_OPS_NO_SIDE_EFFECTS(X) \ +X(ArraySubscript, "[]", Postfix, 18) \ +X(Member, ".", Binary, 18) \ +X(PointerMember, "->", Binary, 18) \ +X(UnaryPlus, "+", Prefix, 17) \ +X(UnaryMinus, "-", Prefix, 17) \ +X(LogicalNot, "!", Prefix, 17) \ +X(BitwiseNot, "~", Prefix, 17) \ +X(Dereference, "*", Prefix, 17) \ +X(AddressOf, "&", Prefix, 17) \ +X(SizeOf, "sizeof", Prefix, 17) \ +X(Multiplication, "*", Binary, 15) \ +X(Division, "/", Binary, 15) \ +X(Modulo, "%", Binary, 15) \ +X(Addition, "+", Binary, 14) \ +X(Subtraction, "-", Binary, 14) \ +X(LeftShift, "<<", Binary, 13) \ +X(RightShift, ">>", Binary, 13) \ +X(LessThan, "<", Binary, 11) \ +X(LessThanOrEqual, "<=", Binary, 11) \ +X(GreaterThan, ">", Binary, 11) \ +X(GreaterThanOrEqual, ">=", Binary, 11) \ +X(Equal, "==", Binary, 10) \ +X(NotEqual, "!=", Binary, 10) \ +X(BitwiseAnd, "&", Binary, 9) \ +X(BitwiseXor, "^", Binary, 8) \ +X(BitwiseOr, "|", Binary, 7) \ +X(LogicalAnd, "&&", Binary, 6) \ +X(LogicalOr, "||", Binary, 5) + +#define C_LIKE_OPS_CALLS(X) \ +X(Call, "()", Postfix, 18) + +#define C_LIKE_OPS_WITH_SIDE_EFFECTS(X) \ +X(PostFixIncrement, "++", Postfix, 18) \ +X(PostFixDecrement, "--", Postfix, 18) \ +X(PreFixIncrement, "++", Prefix, 17) \ +X(PreFixDecrement, "--", Prefix, 17) \ +X(Assign, "=", BinaryRightAssociative, 3) \ +X(AssignAddition, "+=", BinaryRightAssociative, 3) \ +X(AssignSubtraction, "-=", BinaryRightAssociative, 3) \ +X(AssignMultiplication,"*=", BinaryRightAssociative, 3) \ +X(AssignDivision, "/=", BinaryRightAssociative, 3) \ +X(AssignModulo, "%=", BinaryRightAssociative, 3) \ +X(AssignLeftShift, "<<=", BinaryRightAssociative, 3) \ +X(AssignRightShift, ">>=", BinaryRightAssociative, 3) \ +X(AssignBitwiseAnd, "&=", BinaryRightAssociative, 3) \ +X(AssignBitwiseXor, "^=", BinaryRightAssociative, 3) \ +X(AssignBitwiseOr, "|=", BinaryRightAssociative, 3) + +enum Op +{ +#define DEF_ENUM(e,t,k,p) Op##e, + C_LIKE_OPS_NO_SIDE_EFFECTS(DEF_ENUM) + C_LIKE_OPS_CALLS(DEF_ENUM) + C_LIKE_OPS_WITH_SIDE_EFFECTS(DEF_ENUM) +#undef DEF_ENUM +}; + +void +print_expression(FILE *out, MD_Expr *expr) +{ + MD_ExprOpr *op = expr->op; + if (op == 0) + { + MD_Node *node = expr->md_node; + if (node->raw_string.size != 0) + { + fprintf(out, "%.*s", MD_S8VArg(node->raw_string)); + } + else if (!MD_NodeIsNil(node->first_child)) + { + char c1 = 0; + char c2 = 0; + if (node->flags & MD_NodeFlag_HasParenLeft) + { + c1 = '('; + } + if (node->flags & MD_NodeFlag_HasBraceLeft) + { + c1 = '{'; + } + if (node->flags & MD_NodeFlag_HasBracketLeft) + { + c1 = '['; + } + if (node->flags & MD_NodeFlag_HasParenRight) + { + c2 = ')'; + } + if (node->flags & MD_NodeFlag_HasBraceRight) + { + c2 = '}'; + } + if (node->flags & MD_NodeFlag_HasBracketRight) + { + c2 = ']'; + } + fprintf(out, "%c...%c", c1, c2); + } + else + { + MD_CodeLoc loc = MD_CodeLocFromNode(node); + MD_PrintMessage(stderr, loc, MD_MessageKind_Error, + MD_S8Lit("the expression system does not expect this kind of node")); + } + } + else + { + switch (op->kind) + { + default: + { + MD_Node *node = expr->md_node; + MD_CodeLoc loc = MD_CodeLocFromNode(node); + MD_PrintMessage(stderr, loc, MD_MessageKind_FatalError, + MD_S8Lit("this is an unknown kind of operator")); + }break; + + case MD_ExprOprKind_Prefix: + { + fprintf(out, "%.*s", MD_S8VArg(op->string)); + print_expression(out, expr->unary_operand); + }break; + + case MD_ExprOprKind_Postfix: + { + print_expression(out, expr->unary_operand); + MD_String8 op_string = op->string; + // TODO(allen): formalize 'set delimiter mask' into the library + MD_u64 set_delimiter_mask = 0x3F; + if ((expr->md_node->flags & set_delimiter_mask) != 0) + { + fprintf(out, "%c...%c", op_string.str[0], op_string.str[1]); + } + else + { + fprintf(out, "%.*s", MD_S8VArg(op_string)); + } + }break; + + case MD_ExprOprKind_Binary: + case MD_ExprOprKind_BinaryRightAssociative: + { + fprintf(out, "("); + print_expression(out, expr->left); + fprintf(out, " %.*s ", MD_S8VArg(op->string)); + print_expression(out, expr->right); + fprintf(out, ")"); + }break; + } + } +} + + +//~ main ////////////////////////////////////////////////////////////////////// + +int main(int argc, char **argv) +{ +#if 1 + char *argv_dummy[2] = { + 0, + "W:\\metadesk\\examples\\expr\\expr_c_like.mdesk", + }; + argc = 2; + argv = argv_dummy; +#endif + + // setup the global arena + arena = MD_ArenaAlloc(); + + // parse all files passed to the command line + MD_Node *list = MD_MakeList(arena); + for (int i = 1; i < argc; i += 1) + { + + // parse the file + MD_String8 file_name = MD_S8CString(argv[i]); + MD_ParseResult parse_result = MD_ParseWholeFile(arena, file_name); + + // print metadesk errors + for (MD_Message *message = parse_result.errors.first; + message != 0; + message = message->next) + { + MD_CodeLoc code_loc = MD_CodeLocFromNode(message->node); + MD_PrintMessage(stdout, code_loc, message->kind, message->string); + } + + // save to parse results list + if (parse_result.errors.max_message_kind < MD_MessageKind_Error) + { + MD_PushNewReference(arena, list, parse_result.node); + } + } + + // setup the expression system + MD_ExprOprTable table = {0}; + { + MD_ExprOprList list = {0}; + +#define PUSH_OP(e,t,k,p) \ +MD_ExprOprPush(arena, &list, MD_ExprOprKind_##k, p, MD_S8Lit(t), Op##e, 0); + C_LIKE_OPS_NO_SIDE_EFFECTS(PUSH_OP); + C_LIKE_OPS_CALLS(PUSH_OP); + C_LIKE_OPS_WITH_SIDE_EFFECTS(PUSH_OP); +#undef PUSH_OP + + table = MD_ExprBakeOperatorTableFromList(arena, &list); + } + + // print the verbose parse results + for (MD_EachNode(root_it, list->first_child)) + { + MD_Node *root = MD_ResolveNodeFromReference(root_it); + for (MD_EachNode(node, root->first_child)) + { + MD_ExprParseResult parse = MD_ExprParse(arena, &table, node->first_child, MD_NilNode()); + + // print errors + for (MD_Message *message = parse.errors.first; + message != 0; + message = message->next) + { + MD_CodeLoc code_loc = MD_CodeLocFromNode(message->node); + MD_PrintMessage(stdout, code_loc, message->kind, message->string); + } + + // print the expression + if (parse.expr != 0) + { + if (node->string.size != 0) + { + fprintf(stdout, "%.*s = ", MD_S8VArg(node->string)); + } + print_expression(stdout, parse.expr); + fprintf(stdout, ";\n"); + } + } + } + + return 0; +} diff --git a/examples/expr/expr_c_like.mdesk b/examples/expr/expr_c_like.mdesk new file mode 100644 index 0000000..00906d3 --- /dev/null +++ b/examples/expr/expr_c_like.mdesk @@ -0,0 +1,18 @@ +/* +** Example: c like expressions +** +** TODO +*/ + +a: 100; +b: ( array[a] ); +c: sizeof xyz; +d: i*stride + j; +e: 2.71828; +f: ( (H << 16) | (H >> 16) ); +g: ( {0, +1, -1} ); +h: foo.bar; +(i += 1); +(j = 0); +(k = i*stride + j); +l: ( foo(bar, array[k], sizeof c) ); diff --git a/examples/expr/expr_intro.c b/examples/expr/expr_intro.c index 66966e7..a214c1c 100644 --- a/examples/expr/expr_intro.c +++ b/examples/expr/expr_intro.c @@ -111,7 +111,10 @@ int main(int argc, char **argv) } // print the expression - fprintf(stdout, "%.*s = ", MD_S8VArg(node->string)); + if (node->string.size != 0) + { + fprintf(stdout, "%.*s = ", MD_S8VArg(node->string)); + } print_expression(stdout, parse.expr); fprintf(stdout, ";\n"); } diff --git a/examples/expr/expr_intro.mdesk b/examples/expr/expr_intro.mdesk index 90cc1fc..8bebe5f 100644 --- a/examples/expr/expr_intro.mdesk +++ b/examples/expr/expr_intro.mdesk @@ -9,6 +9,6 @@ b: 2; c: 3; w: 100; -x: a*w + b; -y: b*w + c; -z: c*w + a; +x: a; +y: b + w*a; +z: (c + w*(b + w*a)); diff --git a/source/md.h b/source/md.h index 28cff5e..7045ba6 100644 --- a/source/md.h +++ b/source/md.h @@ -790,7 +790,11 @@ typedef struct MD_Expr MD_Expr; struct MD_Expr { struct MD_Expr *parent; - struct MD_Expr *left; + union + { + struct MD_Expr *left; + struct MD_Expr *unary_operand; + }; struct MD_Expr *right; MD_ExprOpr *op; MD_Node *md_node; diff --git a/tests/expression_tests.c b/tests/expression_tests.c index 9de772c..b20bda2 100644 --- a/tests/expression_tests.c +++ b/tests/expression_tests.c @@ -62,26 +62,22 @@ X(BitwiseXor, "^", Binary, 8) \ X(BitwiseOr, "|", Binary, 7) \ X(LogicalAnd, "&&", Binary, 6) \ X(LogicalOr, "||", Binary, 5) \ -X(Assign, "=", BinaryRightAssociative, 4) \ -X(AssignAddition, "+=", BinaryRightAssociative, 4) \ -X(AssignSubtraction, "-=", BinaryRightAssociative, 4) \ -X(AssignMultiplication,"*=", BinaryRightAssociative, 4) \ -X(AssignDivision, "/=", BinaryRightAssociative, 4) \ -X(AssignModulo, "%=", BinaryRightAssociative, 4) \ -X(AssignLeftShift, "<<=", BinaryRightAssociative, 4) \ -X(AssignRightShift, ">>=", BinaryRightAssociative, 4) \ -X(AssignBitwiseAnd, "&=", BinaryRightAssociative, 4) \ -X(AssignBitwiseXor, "^=", BinaryRightAssociative, 4) \ -X(AssignBitwiseOr, "|=", BinaryRightAssociative, 4) \ +X(Assign, "=", BinaryRightAssociative, 3) \ +X(AssignAddition, "+=", BinaryRightAssociative, 3) \ +X(AssignSubtraction, "-=", BinaryRightAssociative, 3) \ +X(AssignMultiplication,"*=", BinaryRightAssociative, 3) \ +X(AssignDivision, "/=", BinaryRightAssociative, 3) \ +X(AssignModulo, "%=", BinaryRightAssociative, 3) \ +X(AssignLeftShift, "<<=", BinaryRightAssociative, 3) \ +X(AssignRightShift, ">>=", BinaryRightAssociative, 3) \ +X(AssignBitwiseAnd, "&=", BinaryRightAssociative, 3) \ +X(AssignBitwiseXor, "^=", BinaryRightAssociative, 3) \ +X(AssignBitwiseOr, "|=", BinaryRightAssociative, 3) \ /* NOTE(mal): These are not in C */ \ X(PostfixBraceBrace, "{}", Postfix, 18) \ X(PostfixBracketParen, "[)", Postfix, 18) \ X(PostfixParenBracket, "(]", Postfix, 18) \ -// TODO(allen): I don't think we want to do this -// X(Cast "()", Prefix, 17) -// X(Comma, ",", Binary, 3) - #define X(name, token, kind, prec) Op_##name, typedef enum{ Op_Null, From b67dc566c665334ba921fbe86a1a89261dda0c2c Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 22:34:41 -0700 Subject: [PATCH 16/28] [examples] tweaks to overrides example and commentary --- bin/build_examples.sh | 8 +++----- bin/run_examples.sh | 18 ++++++++---------- examples/expr/expr_intro.mdesk | 2 +- examples/integration/overrides.c | 26 ++++++++++++++++++++++---- source/md.c | 2 +- 5 files changed, 35 insertions(+), 21 deletions(-) diff --git a/bin/build_examples.sh b/bin/build_examples.sh index 2bd4f5a..9b27674 100755 --- a/bin/build_examples.sh +++ b/bin/build_examples.sh @@ -11,11 +11,6 @@ bin/bld_core.sh show_ctx examps="examples" -bin/bld_core.sh unit expr_intro $examps/expr/expr_intro.c -bin/bld_core.sh unit expr_c_like $examps/expr/expr_c_like.c - -exit - bin/bld_core.sh unit hello_world $examps/intro/hello_world.c bin/bld_core.sh unit parse_check $examps/intro/parse_check.c bin/bld_core.sh unit data_desk_like $examps/intro/data_desk_like_template.c @@ -26,6 +21,9 @@ if [ -d $examps/type_metadata/generated/type_info_meta.h ]; then bin/bld_core.sh unit type_info $examps/type_metadata/type_info_final_program.c fi +bin/bld_core.sh unit expr_intro $examps/expr/expr_intro.c +bin/bld_core.sh unit expr_c_like $examps/expr/expr_c_like.c + bin/bld_core.sh unit overrides $examps/integration/overrides.c echo diff --git a/bin/run_examples.sh b/bin/run_examples.sh index c507880..54c6d51 100755 --- a/bin/run_examples.sh +++ b/bin/run_examples.sh @@ -8,16 +8,6 @@ root_path=$PWD build_path=$root_path/build examps=$root_path/examples -echo ~~~ Running Expression Intro ~~~ -$build_path/expr_intro.exe $examps/expr/expr_intro.mdesk -echo - -echo ~~~ Running C Like Expression ~~~ -$build_path/expr_c_like.exe $examps/expr/expr_c_like.mdesk -echo - -exit - echo ~~~ Running Type Metadata Generator Example ~~~ cd $examps/type_metadata/generated $build_path/type_metadata.exe $examps/type_metadata/types.mdesk @@ -26,5 +16,13 @@ echo echo ~~~ Running Error Generation Example ~~~ $build_path/user_errors.exe $examps/user_errors/user_errors.mdesk +echo ~~~ Running Expression Intro ~~~ +$build_path/expr_intro.exe $examps/expr/expr_intro.mdesk +echo + +echo ~~~ Running C Like Expression ~~~ +$build_path/expr_c_like.exe $examps/expr/expr_c_like.mdesk +echo + ###### Restore Path ########################################################### cd $og_path diff --git a/examples/expr/expr_intro.mdesk b/examples/expr/expr_intro.mdesk index 8bebe5f..f8f4012 100644 --- a/examples/expr/expr_intro.mdesk +++ b/examples/expr/expr_intro.mdesk @@ -11,4 +11,4 @@ c: 3; w: 100; x: a; y: b + w*a; -z: (c + w*(b + w*a)); +z: c + w*b + w*w*a; diff --git a/examples/integration/overrides.c b/examples/integration/overrides.c index c48c038..1068746 100644 --- a/examples/integration/overrides.c +++ b/examples/integration/overrides.c @@ -40,11 +40,24 @@ void examp_free(ExampleAllocator *a, void *ptr); //~ include metadesk header /////////////////////////////////////////////////// +// @notes Disabling print helpers removes any APIs from the library that depend +// on FILE from stdio.h. +#define MD_DISABLE_PRINT_HELPERS 1 + +// @notes Here is also a good place to disable the default implementations of +// anything that is overriden to avoid extra includes. +#define MD_DEFAULT_MEMORY 0 +#define MD_DEFAULT_FILE_LOAD 0 + +// @notes We can also disable default implementations for "optional" parts, +// here we disable the default file iterator without replacing it, which gets +// this example off of direct OS header dependencies. +#define MD_DEFAULT_FILE_ITER 0 + // @notes We include the metadesk header before we define the overrides because // some overrides require that metadesk base types be visible. There are // exceptions to this pattern, in particular overrides for types need to be // defined before including md.h, we aren't going that far here. - #include "md.h" @@ -126,6 +139,8 @@ md_release_by_example_allocator(void *ptr, unsigned long long ignore) // override file loading +#include + MD_String8 md_load_entire_file_by_stdio(MD_Arena *arena, MD_String8 filename) { @@ -151,16 +166,19 @@ md_load_entire_file_by_stdio(MD_Arena *arena, MD_String8 filename) int main(int argc, char **argv) { + // ... where ever program init stuff is happening ... + // initialize the example allocator ExampleAllocator allocator = {0}; - // configure the allocator context + // metadesk allocator context gets setup before a call to MD_ArenaAlloc md_example_allocator = &allocator; - // setup the global arena arena = MD_ArenaAlloc(); - // ... any normal metadesk usage may go here ... + + + // ... any normal metadesk usage may now happen ... return 0; diff --git a/source/md.c b/source/md.c index 9e17ed8..b8754f6 100644 --- a/source/md.c +++ b/source/md.c @@ -127,7 +127,7 @@ MD_CRT_LoadEntireFile(MD_Arena *arena, MD_String8 filename) //////////////////////////////////////////////////////////////////////////////// //- win32 header -#if (MD_DEFAULT_FILE_ITER || MD_DEFAULT_MEMORY) && MD_OS_WINDOWS +#if (MD_DEFAULT_FILE_ITER || MD_2DEFAULT_MEMORY) && MD_OS_WINDOWS # include # pragma comment(lib, "User32.lib") #endif From 8ee69f012452ee140bf1e18cb460cffc6485e9b6 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sat, 9 Oct 2021 23:45:02 -0700 Subject: [PATCH 17/28] [examples] setting up the last two integration examples and slots in the examples directory --- bin/build_examples.sh | 9 ++++++--- bin/run_examples.sh | 9 +++++++++ examples/examples_directory.txt | 7 ++++++- examples/integration/memory_management.c | 18 ++++++++++++++++++ examples/integration/multi_threaded.c | 18 ++++++++++++++++++ examples/intro/hello_world.c | 2 +- 6 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 examples/integration/memory_management.c create mode 100644 examples/integration/multi_threaded.c diff --git a/bin/build_examples.sh b/bin/build_examples.sh index 9b27674..d7bccf3 100755 --- a/bin/build_examples.sh +++ b/bin/build_examples.sh @@ -11,6 +11,9 @@ bin/bld_core.sh show_ctx examps="examples" +bin/bld_core.sh unit memory_management $examps/integration/memory_management.c +bin/bld_core.sh unit multi_threaded $examps/integration/multi_threaded.c + bin/bld_core.sh unit hello_world $examps/intro/hello_world.c bin/bld_core.sh unit parse_check $examps/intro/parse_check.c bin/bld_core.sh unit data_desk_like $examps/intro/data_desk_like_template.c @@ -21,10 +24,10 @@ if [ -d $examps/type_metadata/generated/type_info_meta.h ]; then bin/bld_core.sh unit type_info $examps/type_metadata/type_info_final_program.c fi -bin/bld_core.sh unit expr_intro $examps/expr/expr_intro.c -bin/bld_core.sh unit expr_c_like $examps/expr/expr_c_like.c +bin/bld_core.sh unit expr_intro $examps/expr/expr_intro.c +bin/bld_core.sh unit expr_c_like $examps/expr/expr_c_like.c -bin/bld_core.sh unit overrides $examps/integration/overrides.c +bin/bld_core.sh unit overrides $examps/integration/overrides.c echo diff --git a/bin/run_examples.sh b/bin/run_examples.sh index 54c6d51..0bbd96e 100755 --- a/bin/run_examples.sh +++ b/bin/run_examples.sh @@ -8,6 +8,14 @@ root_path=$PWD build_path=$root_path/build examps=$root_path/examples +echo ~~~ Running Memory Management Example ~~~ +$build_path/memory_management.exe $examps/intro/hello_world.mdesk $examps/intro/labels.mdesk $examps/intro/sets.mdesk +echo + +echo ~~~ Running Multi-Threaded Parse Example ~~~ +$build_path/multi_threaded.exe $examps/intro/hello_world.mdesk $examps/intro/labels.mdesk $examps/intro/sets.mdesk +echo + echo ~~~ Running Type Metadata Generator Example ~~~ cd $examps/type_metadata/generated $build_path/type_metadata.exe $examps/type_metadata/types.mdesk @@ -15,6 +23,7 @@ echo echo ~~~ Running Error Generation Example ~~~ $build_path/user_errors.exe $examps/user_errors/user_errors.mdesk +echo echo ~~~ Running Expression Intro ~~~ $build_path/expr_intro.exe $examps/expr/expr_intro.mdesk diff --git a/examples/examples_directory.txt b/examples/examples_directory.txt index bed4b02..7bdeb2a 100644 --- a/examples/examples_directory.txt +++ b/examples/examples_directory.txt @@ -53,12 +53,17 @@ EXAMPLES: Commentary in this example focuses on strategies for setting up an effective metaprogram. +7. TODO +8. TODO -7. "overrides" integration/overrides.c +9. "overrides" integration/overrides.c When including the Metadesk library into an existing codebase, the overrides system in the library will let you plug in existing implementations you have for many of the basic requirements of the library. This can also be useful if you want to make your program CRT-free, or direct metadesk allocations to your own custom allocator. +10. TODO +11. TODO + diff --git a/examples/integration/memory_management.c b/examples/integration/memory_management.c new file mode 100644 index 0000000..916be1c --- /dev/null +++ b/examples/integration/memory_management.c @@ -0,0 +1,18 @@ +/* +** Example: memory management +** +** TODO +*/ + +//~ includes and globals ////////////////////////////////////////////////////// + +#include "md.h" +#include "md.c" + +//~ main ////////////////////////////////////////////////////////////////////// + +int +main(int argc, char **argv) +{ + // TODO(allen) +} diff --git a/examples/integration/multi_threaded.c b/examples/integration/multi_threaded.c new file mode 100644 index 0000000..a88c287 --- /dev/null +++ b/examples/integration/multi_threaded.c @@ -0,0 +1,18 @@ +/* +** Example: multi threaded parse +** +** TODO +*/ + +//~ includes and globals ////////////////////////////////////////////////////// + +#include "md.h" +#include "md.c" + +//~ main ////////////////////////////////////////////////////////////////////// + +int +main(int argc, char **argv) +{ + // TODO(allen) +} diff --git a/examples/intro/hello_world.c b/examples/intro/hello_world.c index ed88dcb..31b9751 100644 --- a/examples/intro/hello_world.c +++ b/examples/intro/hello_world.c @@ -24,4 +24,4 @@ main(int argc, char **argv) // print the results MD_PrintDebugDumpFromNode(stdout, parse.node, MD_GenerateFlags_All); -} \ No newline at end of file +} From 93ac2b4bae021d31d1d4533f9dfe8b24f95c28bb Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 10:00:03 -0700 Subject: [PATCH 18/28] [examples] get started on the multi threaded parse example --- bin/run_examples.sh | 13 +- examples/integration/multi_threaded.c | 132 +++++++++++++++++- examples/type_metadata/generated/meta_types.c | 8 +- examples/type_metadata/generated/meta_types.h | 6 +- source/md.c | 2 + 5 files changed, 151 insertions(+), 10 deletions(-) diff --git a/bin/run_examples.sh b/bin/run_examples.sh index 0bbd96e..5532d48 100755 --- a/bin/run_examples.sh +++ b/bin/run_examples.sh @@ -8,12 +8,21 @@ root_path=$PWD build_path=$root_path/build examps=$root_path/examples +# Setup a big list of files for a few of the examples +big_list= $examps/intro/hello_world.mdesk +bit_list=$big_list $examps/intro/labels.mdesk +bit_list=$big_list $examps/intro/sets.mdesk +bit_list=$big_list $examps/type_metadata/types.mdesk +bit_list=$big_list $examps/type_metadata/bad_types.mdesk +bit_list=$big_list $examps/expr/expr_intro.mdesk +bit_list=$big_list $examps/expr/expr_c_like.mdesk + echo ~~~ Running Memory Management Example ~~~ -$build_path/memory_management.exe $examps/intro/hello_world.mdesk $examps/intro/labels.mdesk $examps/intro/sets.mdesk +$build_path/memory_management.exe $big_list echo echo ~~~ Running Multi-Threaded Parse Example ~~~ -$build_path/multi_threaded.exe $examps/intro/hello_world.mdesk $examps/intro/labels.mdesk $examps/intro/sets.mdesk +$build_path/multi_threaded.exe $big_list echo echo ~~~ Running Type Metadata Generator Example ~~~ diff --git a/examples/integration/multi_threaded.c b/examples/integration/multi_threaded.c index a88c287..d9acecc 100644 --- a/examples/integration/multi_threaded.c +++ b/examples/integration/multi_threaded.c @@ -9,10 +9,140 @@ #include "md.h" #include "md.c" +#include + +//~ multi-threaded parse setup //////////////////////////////////////////////// + +#if MD_COMPILER_CL +# define atomic_inc_u64(p) InterlockedIncrement64((LONG64*)p) +#else +# error Not implemented for this compiler +#endif + +typedef struct TaskData +{ + MD_u64 task_max; + char **tasks; + + volatile MD_u64 task_counter; + volatile MD_u64 thread_counter; +} TaskData; + +typedef struct ThreadData +{ + TaskData *task; + MD_Arena *arena; + MD_Node *list; + MD_MessageList errors; +} ThreadData; + +void +parse_worker_loop(ThreadData *thread_data) +{ + TaskData *task = thread_data->task; + for (;;) + { + MD_u64 task_index = atomic_inc_u64(&task->task_counter); + if (task_index >= task->task_max) + { + break; + } + MD_String8 file_name = MD_S8CString(task->tasks[task_index]); + MD_ParseResult parse = MD_ParseWholeFile(thread_data->arena, file_name); + if (parse.errors.first != 0) + { + MD_MessageListConcat(&thread_data->errors, &parse.errors); + } + else + { + MD_PushNewReference(thread_data->arena, thread_data->list, parse.node); + } + } + + atomic_inc_u64(&task->thread_counter); +} + +#if MD_OS_WINDOWS +DWORD +parse_worker_win32(LPVOID parameter) +{ + parse_worker_loop((ThreadData*)parameter); + return(0); +} +#else +# error Not implemented for this OS +#endif + + //~ main ////////////////////////////////////////////////////////////////////// int main(int argc, char **argv) { - // TODO(allen) +#if 1 + char *argv_dummy[8] = { + 0, + "W:\\metadesk\\examples\\intro\\hello_world.mdesk", + "W:\\metadesk\\examples\\intro\\labels.mdesk", + "W:\\metadesk\\examples\\intro\\sets.mdesk", + "W:\\metadesk\\examples\\type_metadata\\types.mdesk", + "W:\\metadesk\\examples\\type_metadata\\bad_types.mdesk", + "W:\\metadesk\\examples\\expr\\expr_intro.mdesk", + "W:\\metadesk\\examples\\expr\\expr_c_like.mdesk", + }; + argc = 8; + argv = argv_dummy; +#endif + + // make sure we have something to parse + if (argc <= 1) + { + fprintf(stderr, "pass at least one input file"); + exit(1); + } + + // setup the shared task data + TaskData task = {0}; + task.task_max = argc - 1; + task.tasks = argv + 1; + + // setup the per-thread data +#define THREAD_COUNT 2 + ThreadData threads[THREAD_COUNT] = {0}; + for (int i = 0; i < THREAD_COUNT; i += 1) + { + threads[i].task = &task; + threads[i].arena = MD_ArenaAlloc(); + threads[i].list = MD_MakeList(threads[i].arena); + } + + // launch the worker threads + for (int i = 1; i < THREAD_COUNT; i += 1) + { +#if MD_OS_WINDOWS + HANDLE handle = CreateThread(0, 0, parse_worker_win32, threads + i, 0, 0); + CloseHandle(handle); +#else +# error Not implemented for this OS +#endif + } + + parse_worker_loop(&threads[0]); + + // wait for all threads to be finished + for (;;) + { + MD_u64 thread_counter = task.thread_counter; + if (thread_counter >= THREAD_COUNT) + { + break; + } +#if MD_OS_WINDOWS + Sleep(0); +#else +# error Not implemented for this OS +#endif + } + + // TODO(allen): combine results } diff --git a/examples/type_metadata/generated/meta_types.c b/examples/type_metadata/generated/meta_types.c index 448dbd5..a4b9503 100644 --- a/examples/type_metadata/generated/meta_types.c +++ b/examples/type_metadata/generated/meta_types.c @@ -1,4 +1,4 @@ -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:991 +// generated by W:\metadesk\examples\type_metadata\type_metadata.c:991 TypeInfoMember Circle_members[2] = { {"r", 1, -1, &F32_type_info}, {"pos", 3, -1, &V2F32_type_info}, @@ -14,14 +14,14 @@ TypeInfoMember RoundedPolygon_members[3] = { {"p", 1, 1, &V2F32_type_info}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1030 +// generated by W:\metadesk\examples\type_metadata\type_metadata.c:1030 TypeInfoEnumerant Shape_members[3] = { {"Circle", 6, 1}, {"Segment", 7, 2}, {"Polygon", 7, 3}, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1066 +// generated by W:\metadesk\examples\type_metadata\type_metadata.c:1066 TypeInfo U32_type_info = {TypeKind_Basic, "U32", 3, 4, 0, 0}; TypeInfo F32_type_info = {TypeKind_Basic, "F32", 3, 4, 0, 0}; TypeInfo V2F32_type_info = {TypeKind_Basic, "V2F32", 5, 8, 0, 0}; @@ -30,7 +30,7 @@ TypeInfo RoundedSegment_type_info = {TypeKind_Struct, "RoundedSegment", 14, 3, R TypeInfo RoundedPolygon_type_info = {TypeKind_Struct, "RoundedPolygon", 14, 3, RoundedPolygon_members, 0}; TypeInfo Shape_type_info = {TypeKind_Enum, "Shape", 5, 3, Shape_members, &U32_type_info}; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:1126 +// generated by W:\metadesk\examples\type_metadata\type_metadata.c:1126 TypeInfo* type_info_from_shape(Shape v) { diff --git a/examples/type_metadata/generated/meta_types.h b/examples/type_metadata/generated/meta_types.h index 789fcea..572c62d 100644 --- a/examples/type_metadata/generated/meta_types.h +++ b/examples/type_metadata/generated/meta_types.h @@ -1,6 +1,6 @@ #if !defined(META_TYPES_H) #define META_TYPES_H -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:868 +// generated by W:\metadesk\examples\type_metadata\type_metadata.c:868 typedef struct Circle Circle; struct Circle { @@ -29,11 +29,11 @@ Shape_Segment = 2, Shape_Polygon = 3, }; -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:952 +// generated by W:\metadesk\examples\type_metadata\type_metadata.c:952 TypeInfo* type_info_from_shape(Shape v); U32 max_slot_from_shape(Shape v); -// generated by W:/metadesk/examples/type_metadata/type_metadata.c:975 +// generated by W:\metadesk\examples\type_metadata\type_metadata.c:975 extern TypeInfo U32_type_info; extern TypeInfo F32_type_info; extern TypeInfo V2F32_type_info; diff --git a/source/md.c b/source/md.c index b8754f6..377ed1b 100644 --- a/source/md.c +++ b/source/md.c @@ -597,6 +597,8 @@ MD_ArenaDefaultSetAutoAlign(MD_ArenaDefault *arena, MD_u64 align) arena->align = align; } +// TODO(allen): Arena absorb + #endif //- "arena" implementation checks From f07b74da5fad384e34454e446e0c4ac32dec686b Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 10:53:59 -0700 Subject: [PATCH 19/28] [examples] add output and combining to multi threaded parse example --- examples/integration/multi_threaded.c | 44 ++++++++++++++++++----- source/md.c | 50 ++++++++++++++++++++++----- source/md.h | 1 + 3 files changed, 77 insertions(+), 18 deletions(-) diff --git a/examples/integration/multi_threaded.c b/examples/integration/multi_threaded.c index d9acecc..ff6446e 100644 --- a/examples/integration/multi_threaded.c +++ b/examples/integration/multi_threaded.c @@ -49,14 +49,8 @@ parse_worker_loop(ThreadData *thread_data) } MD_String8 file_name = MD_S8CString(task->tasks[task_index]); MD_ParseResult parse = MD_ParseWholeFile(thread_data->arena, file_name); - if (parse.errors.first != 0) - { - MD_MessageListConcat(&thread_data->errors, &parse.errors); - } - else - { - MD_PushNewReference(thread_data->arena, thread_data->list, parse.node); - } + MD_MessageListConcat(&thread_data->errors, &parse.errors); + MD_PushNewReference(thread_data->arena, thread_data->list, parse.node); } atomic_inc_u64(&task->thread_counter); @@ -144,5 +138,37 @@ main(int argc, char **argv) #endif } - // TODO(allen): combine results + // print results + for (int i = 0; i < THREAD_COUNT; i += 1) + { + fprintf(stdout, "on thread %d:\n", i); + + // print the name of each root + for (MD_EachNode(root_it, threads[i].list->first_child)) + { + MD_Node *root = MD_ResolveNodeFromReference(root_it); + fprintf(stdout, "%.*s\n", MD_S8VArg(root->string)); + } + + // print the errors from this thread + MD_MessageList errors = threads[i].errors; + for (MD_Message *message = errors.first; + message != 0; + message = message->next) + { + MD_CodeLoc loc = MD_CodeLocFromNode(message->node); + MD_PrintMessage(stdout, loc, message->kind, message->string); + } + } + + // combine results + MD_Arena *arena = threads[0].arena; + MD_Node *list = threads[0].list; + MD_MessageList errors = threads[0].errors; + for (int i = 1; i < THREAD_COUNT; i += 1) + { + MD_ArenaDefaultAbsorb(arena, threads[i].arena); + MD_ListConcatInPlace(list, threads[i].list); + MD_MessageListConcat(&errors, &threads[i].errors); + } } diff --git a/source/md.c b/source/md.c index 377ed1b..f468573 100644 --- a/source/md.c +++ b/source/md.c @@ -597,7 +597,20 @@ MD_ArenaDefaultSetAutoAlign(MD_ArenaDefault *arena, MD_u64 align) arena->align = align; } -// TODO(allen): Arena absorb +static void +MD_ArenaDefaultAbsorb(MD_ArenaDefault *arena, MD_ArenaDefault *sub_arena) +{ + MD_ArenaDefault *current = arena->current; + MD_u64 base_pos_shift = current->base_pos + current->cap; + for (MD_ArenaDefault *node = sub_arena->current; + node != 0; + node = node->prev) + { + node->base_pos += base_pos_shift; + } + sub_arena->prev = arena->current; + arena->current = sub_arena->current; +} #endif @@ -2926,9 +2939,9 @@ MD_MessageListPush(MD_MessageList *list, MD_Message *message) MD_FUNCTION void MD_MessageListConcat(MD_MessageList *list, MD_MessageList *to_push) { - if(list->last) + if(to_push->node_count != 0) { - if(to_push->node_count != 0) + if(list->last != 0) { list->last->next = to_push->first; list->last = to_push->last; @@ -2938,12 +2951,12 @@ MD_MessageListConcat(MD_MessageList *list, MD_MessageList *to_push) list->max_message_kind = to_push->max_message_kind; } } + else + { + *list = *to_push; + } + MD_MemoryZeroStruct(to_push); } - else - { - *list = *to_push; - } - MD_MemoryZeroStruct(to_push); } //~ Location Conversions @@ -2996,7 +3009,7 @@ MD_CodeLocFromNode(MD_Node *node) MD_FUNCTION MD_b32 MD_NodeIsNil(MD_Node *node) { - return node == 0 || node == &_md_nil_node || node->kind == MD_NodeKind_Nil; + return(node == 0 || node == &_md_nil_node || node->kind == MD_NodeKind_Nil); } MD_FUNCTION MD_Node * @@ -3045,6 +3058,25 @@ MD_MakeList(MD_Arena *arena) return(result); } +MD_FUNCTION void +MD_ListConcatInPlace(MD_Node *list, MD_Node *to_push) +{ + if (!MD_NodeIsNil(to_push->first_child)) + { + if (!MD_NodeIsNil(list->first_child)) + { + list->last_child->next = to_push->first_child; + list->last_child = to_push->last_child; + } + else + { + list->first_child = to_push->first_child; + list->last_child = to_push->last_child; + } + to_push->first_child = to_push->last_child = MD_NilNode(); + } +} + MD_FUNCTION MD_Node* MD_PushNewReference(MD_Arena *arena, MD_Node *list, MD_Node *target) { diff --git a/source/md.h b/source/md.h index 7045ba6..6b358d6 100644 --- a/source/md.h +++ b/source/md.h @@ -1085,6 +1085,7 @@ MD_FUNCTION void MD_PushChild(MD_Node *parent, MD_Node *new_child); MD_FUNCTION void MD_PushTag(MD_Node *node, MD_Node *tag); MD_FUNCTION MD_Node *MD_MakeList(MD_Arena *arena); +MD_FUNCTION void MD_ListConcatInPlace(MD_Node *list, MD_Node *to_push); MD_FUNCTION MD_Node *MD_PushNewReference(MD_Arena *arena, MD_Node *list, MD_Node *target); //~ Introspection Helpers From c016b8ccccc7e0353403c299f0cb845fc4c5062a Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 11:03:22 -0700 Subject: [PATCH 20/28] [examples] fix indexing bug in multi threaded example; make the intrinsic work with clang too --- examples/integration/multi_threaded.c | 10 ++++++---- examples/intro/sets.mdesk | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/integration/multi_threaded.c b/examples/integration/multi_threaded.c index ff6446e..e78471e 100644 --- a/examples/integration/multi_threaded.c +++ b/examples/integration/multi_threaded.c @@ -13,8 +13,10 @@ //~ multi-threaded parse setup //////////////////////////////////////////////// -#if MD_COMPILER_CL -# define atomic_inc_u64(p) InterlockedIncrement64((LONG64*)p) +// for this intrinsic we're assume pre-increment behavior + +#if MD_OS_WINDOWS +# define atomic_inc_then_eval_u64(p) InterlockedIncrement64((LONG64*)p) #else # error Not implemented for this compiler #endif @@ -42,7 +44,7 @@ parse_worker_loop(ThreadData *thread_data) TaskData *task = thread_data->task; for (;;) { - MD_u64 task_index = atomic_inc_u64(&task->task_counter); + MD_u64 task_index = atomic_inc_then_eval_u64(&task->task_counter) - 1; if (task_index >= task->task_max) { break; @@ -53,7 +55,7 @@ parse_worker_loop(ThreadData *thread_data) MD_PushNewReference(thread_data->arena, thread_data->list, parse.node); } - atomic_inc_u64(&task->thread_counter); + atomic_inc_then_eval_u64(&task->thread_counter); } #if MD_OS_WINDOWS diff --git a/examples/intro/sets.mdesk b/examples/intro/sets.mdesk index b23f6d4..5376e7f 100644 --- a/examples/intro/sets.mdesk +++ b/examples/intro/sets.mdesk @@ -95,6 +95,7 @@ baz: foo_again // undelimited sets may not contain unlabeled delimited sets // the following forms one undelimited set followed by a delimited set // the delimited set is a sibling to foo, not a child to foo. +// this also emits a warning because it's weird. foo: bar (baz) // undelimited sets *may* however contain labeled delimited sets From f26cb332f05707adfae4abb6d8043ba49c24cfd0 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 11:59:58 -0700 Subject: [PATCH 21/28] [examples] tweaks in multi threaded example; fill in the memory management example --- bin/run_examples.sh | 31 +++++---- examples/integration/memory_management.c | 87 +++++++++++++++++++++++- examples/integration/multi_threaded.c | 15 ---- 3 files changed, 102 insertions(+), 31 deletions(-) diff --git a/bin/run_examples.sh b/bin/run_examples.sh index 5532d48..22f2f8a 100755 --- a/bin/run_examples.sh +++ b/bin/run_examples.sh @@ -9,21 +9,14 @@ build_path=$root_path/build examps=$root_path/examples # Setup a big list of files for a few of the examples -big_list= $examps/intro/hello_world.mdesk -bit_list=$big_list $examps/intro/labels.mdesk -bit_list=$big_list $examps/intro/sets.mdesk -bit_list=$big_list $examps/type_metadata/types.mdesk -bit_list=$big_list $examps/type_metadata/bad_types.mdesk -bit_list=$big_list $examps/expr/expr_intro.mdesk -bit_list=$big_list $examps/expr/expr_c_like.mdesk - -echo ~~~ Running Memory Management Example ~~~ -$build_path/memory_management.exe $big_list -echo - -echo ~~~ Running Multi-Threaded Parse Example ~~~ -$build_path/multi_threaded.exe $big_list -echo +big_list=() +big_list+=("$examps/intro/hello_world.mdesk") +big_list+=("$examps/intro/labels.mdesk") +big_list+=("$examps/intro/sets.mdesk") +big_list+=("$examps/type_metadata/types.mdesk") +big_list+=("$examps/type_metadata/bad_types.mdesk") +big_list+=("$examps/expr/expr_intro.mdesk") +big_list+=("$examps/expr/expr_c_like.mdesk") echo ~~~ Running Type Metadata Generator Example ~~~ cd $examps/type_metadata/generated @@ -42,5 +35,13 @@ echo ~~~ Running C Like Expression ~~~ $build_path/expr_c_like.exe $examps/expr/expr_c_like.mdesk echo +echo ~~~ Running Multi-Threaded Parse Example ~~~ +$build_path/multi_threaded.exe ${big_list[@]} +echo + +echo ~~~ Running Memory Management Example ~~~ +$build_path/memory_management.exe ${big_list[@]} +echo + ###### Restore Path ########################################################### cd $og_path diff --git a/examples/integration/memory_management.c b/examples/integration/memory_management.c index 916be1c..e097438 100644 --- a/examples/integration/memory_management.c +++ b/examples/integration/memory_management.c @@ -9,10 +9,95 @@ #include "md.h" #include "md.c" +//~ pretend config file /////////////////////////////////////////////////////// + +typedef struct ConfigFile{ + MD_Arena *arena; + MD_String8 file_name; + MD_String8 contents; + MD_Node *root; + MD_MessageList errors; +} ConfigFile; + +ConfigFile* +new_config_file_from_file_name(char *file_name_cstr) +{ + MD_Arena *arena = MD_ArenaAlloc(); + ConfigFile *result = MD_PushArrayZero(arena, ConfigFile, 1); + + MD_String8 file_name = MD_S8Copy(arena, MD_S8CString(file_name_cstr)); + MD_String8 contents = MD_LoadEntireFile(arena, file_name); + MD_ParseResult parse = MD_ParseWholeString(arena, file_name, contents); + + result->arena = arena; + result->file_name = file_name; + result->contents = contents; + result->root = parse.node; + result->errors = parse.errors; + return(result); +} + +void +release_config_file(ConfigFile *file) +{ + if (file != 0) + { + MD_ArenaRelease(file->arena); + } +} + +//~ just to simulate new config files coming from somewhere /////////////////// + +int in_files_count = 0; +char** in_file_names = 0; +int in_file_iter = 0; + +ConfigFile* +new_config_file(void) +{ + ConfigFile *result = new_config_file_from_file_name(in_file_names[in_file_iter]); + in_file_iter = (in_file_iter + 1)%in_files_count; + return(result); +} + //~ main ////////////////////////////////////////////////////////////////////// int main(int argc, char **argv) { - // TODO(allen) + // make sure we have something to parse + if (argc <= 1) + { + fprintf(stderr, "pass at least one input file"); + exit(1); + } + + // setup the the source of files + in_files_count = argc - 1; + in_file_names = argv + 1; + + + // pretend there are unpredictable lifetimes tied to real-time events + { + // first we get three config files + ConfigFile *files[3]; + files[0] = new_config_file(); + files[1] = new_config_file(); + files[2] = new_config_file(); + + // then we chaotically replace the slots for a while + for (MD_u32 i = 10000; i < 20000; i += 1) + { + MD_u32 x = (i >> (i&3)) ^ (i << (16 + (i&3))); + MD_u32 slot_index = x%3; + + release_config_file(files[slot_index]); + files[slot_index] = new_config_file(); + } + + // then we're done with all the config files + release_config_file(files[0]); + release_config_file(files[1]); + release_config_file(files[2]); + } } diff --git a/examples/integration/multi_threaded.c b/examples/integration/multi_threaded.c index e78471e..5962383 100644 --- a/examples/integration/multi_threaded.c +++ b/examples/integration/multi_threaded.c @@ -75,21 +75,6 @@ parse_worker_win32(LPVOID parameter) int main(int argc, char **argv) { -#if 1 - char *argv_dummy[8] = { - 0, - "W:\\metadesk\\examples\\intro\\hello_world.mdesk", - "W:\\metadesk\\examples\\intro\\labels.mdesk", - "W:\\metadesk\\examples\\intro\\sets.mdesk", - "W:\\metadesk\\examples\\type_metadata\\types.mdesk", - "W:\\metadesk\\examples\\type_metadata\\bad_types.mdesk", - "W:\\metadesk\\examples\\expr\\expr_intro.mdesk", - "W:\\metadesk\\examples\\expr\\expr_c_like.mdesk", - }; - argc = 8; - argv = argv_dummy; -#endif - // make sure we have something to parse if (argc <= 1) { From e21c5b9d6b1cc127fafbf3039e2d01b03593ed94 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 12:51:21 -0700 Subject: [PATCH 22/28] [examples] fill in directory comments for all the latest examples --- bin/build_examples.sh | 6 ++--- examples/examples_directory.txt | 41 +++++++++++++++++++++++++++++---- examples_todo.txt | 12 ++++++++-- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/bin/build_examples.sh b/bin/build_examples.sh index d7bccf3..c1d0bed 100755 --- a/bin/build_examples.sh +++ b/bin/build_examples.sh @@ -11,9 +11,6 @@ bin/bld_core.sh show_ctx examps="examples" -bin/bld_core.sh unit memory_management $examps/integration/memory_management.c -bin/bld_core.sh unit multi_threaded $examps/integration/multi_threaded.c - bin/bld_core.sh unit hello_world $examps/intro/hello_world.c bin/bld_core.sh unit parse_check $examps/intro/parse_check.c bin/bld_core.sh unit data_desk_like $examps/intro/data_desk_like_template.c @@ -29,6 +26,9 @@ bin/bld_core.sh unit expr_c_like $examps/expr/expr_c_like.c bin/bld_core.sh unit overrides $examps/integration/overrides.c +bin/bld_core.sh unit multi_threaded $examps/integration/multi_threaded.c +bin/bld_core.sh unit memory_management $examps/integration/memory_management.c + echo ###### Restore Path ########################################################### diff --git a/examples/examples_directory.txt b/examples/examples_directory.txt index 7bdeb2a..8544db6 100644 --- a/examples/examples_directory.txt +++ b/examples/examples_directory.txt @@ -53,8 +53,27 @@ EXAMPLES: Commentary in this example focuses on strategies for setting up an effective metaprogram. -7. TODO -8. TODO + +7. "expressions intro" expr/expr_intro.c, expr/expr_intro.mdesk + This example introduces the expression parsing feature built into Metadesk. + With it, flat sequences of Metadesk nodes can be parsed as arithmetic + expressions. + + This example shows how to setup an operator table, call the expression parser + and interpret the expression tree that comes back. It discusses some of the + capabilities and limits of the expression system. + + +8. "c like expressions" expr/expr_c_like.c, expr/expr_c_like.mdesk + This example has a pre-built operator table setup for parsing C like + expressions. This can be used as an advanced example of the expression parser + or as a copy-paste template for making an expression parser that matches C. + The only C operator that is not supported in some way is the ternary operator. + + The metadesk file for this example shows how to deal with a few quirks that + arise from the fact that the expression system is layered on top of the + Metadesk grammar. + 9. "overrides" integration/overrides.c When including the Metadesk library into an existing codebase, the overrides @@ -63,7 +82,21 @@ EXAMPLES: you want to make your program CRT-free, or direct metadesk allocations to your own custom allocator. -10. TODO -11. TODO + +10. "memory management" integration/memory_management.c + This example shows how to use Metadesk in situations where the basic + allocate-and-never-free strategy isn't good enough. The example discusses the + MD_Arena, shows the basics on getting started with it, and has a lot of tips + on further ways to use the arenas. +11. "multi threaded parse" integration/multi_threaded.c + If Metadesk is used to encode metadata in something like an asset pipeline + it may become useful to parse multiple Metadesk files in parallel. The library + doesn't have any thread safety "built in" to the APIs, but it is designed + to make it fairly straight forward to manage the thread safety yourself, and + this example shows how. + + This example relies on some of the information explained in the memory + management example, so it may be useful to start there. + diff --git a/examples_todo.txt b/examples_todo.txt index 7c8e76c..a557907 100644 --- a/examples_todo.txt +++ b/examples_todo.txt @@ -8,12 +8,20 @@ Example Programs: [x] Commentary [x] Example Type File With Errors [ ] Example of simple expression parser -[ ] Example of C-like expression parser (with value and type expressions) -[ ] Example(s) of using overrides [x] Write [ ] Commentary +[ ] Example of C-like expression parser (with value and type expressions) + [x] Write + [ ] Commentary +[ ] Example(s) of using overrides + [x] Write + [x] Commentary [ ] Example memory clearing in long-running program + [x] Write + [ ] Commentary [ ] Example multi-threaded parsing + [x] Write + [ ] Commentary Example Metadesk Files: [x] Hello world From 07770d51d1b2e519b21b240109c903c9e0b0a0b0 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 15:11:41 -0700 Subject: [PATCH 23/28] [examples] filled out the expression intro example and added commentary --- examples/expr/expr_intro.c | 161 +++++++++++++++++++++++++++++++-- examples/expr/expr_intro.mdesk | 27 +++++- source/md.h | 9 +- 3 files changed, 185 insertions(+), 12 deletions(-) diff --git a/examples/expr/expr_intro.c b/examples/expr/expr_intro.c index a214c1c..1979945 100644 --- a/examples/expr/expr_intro.c +++ b/examples/expr/expr_intro.c @@ -1,7 +1,9 @@ /* ** Example: expressions intro ** -** TODO +** This example shows how to use expression parsing in Metadesk. There is +** commentary about the setup as well as the features and limits of the +** expression parser. ** */ @@ -14,10 +16,14 @@ static MD_Arena *arena = 0; //~ expression setup and helpers ////////////////////////////////////////////// +// @notes A common easy setup is to give each operator a statically known +// integer code. We can switch on these integer codes later to interpret the +// operator nodes in expressions. enum { OpAdd, OpMul, + OpIllegal, }; void @@ -26,9 +32,16 @@ print_expression(FILE *out, MD_Expr *expr) MD_ExprOpr *op = expr->op; if (op == 0) { + // @notes Any MD_Expr that doesn't have an operator attached must be a + // leaf of the expression. In this example we don't want to recognize + // any nodes with set delimiters as leaves. + // + // Every expression node (MD_Expr) has an `md_node` regardless of + // whether it is a leaf or an operator. This node gives us a way to + // see information about this leaf, and also gives a way to create an + // MD_CodeLoc for error messages. The same works on operator nodes. MD_Node *node = expr->md_node; - if (node->raw_string.size != 0 && - MD_NodeIsNil(node->first_child)) + if ((node->flags & MD_NodeFlag_MaskSetDelimiters) == 0) { fprintf(out, "%.*s", MD_S8VArg(node->raw_string)); } @@ -41,19 +54,88 @@ print_expression(FILE *out, MD_Expr *expr) } else { + // @notes Any MD_Expr that does have an operator attached is an + // internal node of the expression. In this example we only setup + // binary operators, so we don't bother looking at what kind of + // operator this is. fprintf(out, "("); print_expression(out, expr->left); fprintf(out, " %.*s ", MD_S8VArg(op->string)); print_expression(out, expr->right); fprintf(out, ")"); + + if (op->op_id == OpIllegal) + { + MD_CodeLoc loc = MD_CodeLocFromNode(expr->md_node); + MD_PrintMessage(stderr, loc, MD_MessageKind_Error, + MD_S8Lit("this operator is not actually legal")); + } } } +// @notes Commonly a useful thing to do with an expression system is to +// evaluate the expressions. Here's a quick sketch of what that might look +// like. + +MD_Map eval_map = {0}; + +int +eval_expression(MD_Expr *expr) +{ + int result = 0; + MD_ExprOpr *op = expr->op; + if (op == 0) + { + MD_Node *node = expr->md_node; + if (node->flags & MD_NodeFlag_Numeric) + { + result = MD_CStyleIntFromString(node->string); + } + else if (node->flags & MD_NodeFlag_Identifier) + { + MD_MapSlot *slot = MD_MapLookup(&eval_map, MD_MapKeyStr(node->string)); + if (slot != 0) + { + result = (int)(MD_u64)slot->val; + } + } + } + else + { + int l = eval_expression(expr->left); + int r = eval_expression(expr->right); + + // @notes The `op_id` on this op pointer is carried to use from the + // operator setup where we used the OpAdd and OpMul enum to assign + // assign static integers to each operator. + switch (op->op_id) + { + case OpAdd: + { + result = l + r; + }break; + case OpMul: + { + result = l * r; + }break; + } + } + return(result); +} //~ main ////////////////////////////////////////////////////////////////////// int main(int argc, char **argv) { +#if 1 + char *argv_dummy[2] = { + 0, + "W:\\metadesk\\examples\\expr\\expr_intro.mdesk", + }; + argc = 2; + argv = argv_dummy; +#endif + // setup the global arena arena = MD_ArenaAlloc(); @@ -85,20 +167,72 @@ int main(int argc, char **argv) // setup the expression system MD_ExprOprTable table = {0}; { - MD_ExprOprList list = {0}; + // @notes To start using the expression system we have to decide what + // the expression operator table is going to look like. To do this we + // build up a list of operators and then bake that list into an + // optimized operator table for the parser to use. + // + // An operator string can be any string that parses as exactly one + // main node in metadesk. So it must count as a single token, and it + // cannot be a tag "@", set delimiter "()[]{}", or separator ",;". + // + // The system does have one special case for operator strings. A + // postfix operator may be created with "()", "[]", "{}", "[)" or "(]" + // these get specially interpreted to mean that a set with those + // delimiters may be used as a postfix operator. This can be used to + // parse things like array indexers and function calls. + // + // Here we just use symbol tokens as operators, but identifiers as + // operators are also allowed. + // + // Here we can attach user data in two forms (the last two parameters) + // The first is intended for static integers like enum values here. + // The second is intended for non-static user data like a pointer to + // another data structure. Both are optional. + // + // The bake function converts the list into a table optimized for + // parsing, but first it also checks the operator list. These checks + // include checking the names as described above, making sure + // there are no ambiguities from colliding operators, and making sure + // there are no mismatches of precedence levels and operator kinds + // that cannot be resolved. + // + // The memory used in the list is also used by the table, so they + // should be on the same arena, or at the very least the list's arena + // should not be cleared while the table is still in use. + MD_ExprOprList list = {0}; MD_ExprOprPush(arena, &list, MD_ExprOprKind_Binary, 1, MD_S8Lit("+"), OpAdd, 0); MD_ExprOprPush(arena, &list, MD_ExprOprKind_Binary, 2, MD_S8Lit("*"), OpMul, 0); + MD_ExprOprPush(arena, &list, MD_ExprOprKind_Binary, 3, MD_S8Lit("&"), OpIllegal, 0); table = MD_ExprBakeOperatorTableFromList(arena, &list); } - // print the verbose parse results + // apply expression parsing to each top level node for (MD_EachNode(root_it, list->first_child)) { + // init eval map + eval_map = MD_MapMake(arena); + MD_Node *root = MD_ResolveNodeFromReference(root_it); for (MD_EachNode(node, root->first_child)) { + // @notes An expression parse is an extra stage of analysis on top + // of the initial Metadesk parse. It takes in a range of Metadesk + // nodes specified as (first, one-past-last). Here we want to + // parse all of the children of the top-level node as a single + // expression, so we use the node's `first_child` as the first and + // nil as the one-past-last. + // + // The parser can return a list of new error messages, and an + // expression tree. + // + // The tree holds pointers back to the original operator data from + // the setup, so the tree should be on the same arena, or on at + // at least the operator memory should outlive the tree. + + // run the expression parse MD_ExprParseResult parse = MD_ExprParse(arena, &table, node->first_child, MD_NilNode()); // print errors @@ -110,13 +244,20 @@ int main(int argc, char **argv) MD_PrintMessage(stdout, code_loc, message->kind, message->string); } - // print the expression - if (node->string.size != 0) + if (parse.expr != 0) { - fprintf(stdout, "%.*s = ", MD_S8VArg(node->string)); + // evaluate the expression + int eval_result = eval_expression(parse.expr); + MD_MapInsert(arena, &eval_map, MD_MapKeyStr(node->string), (void*)(MD_u64)eval_result); + + // print the expression + if (node->string.size != 0) + { + fprintf(stdout, "%.*s = ", MD_S8VArg(node->string)); + } + print_expression(stdout, parse.expr); + fprintf(stdout, "; (%d)\n", eval_result); } - print_expression(stdout, parse.expr); - fprintf(stdout, ";\n"); } } diff --git a/examples/expr/expr_intro.mdesk b/examples/expr/expr_intro.mdesk index f8f4012..8933cbe 100644 --- a/examples/expr/expr_intro.mdesk +++ b/examples/expr/expr_intro.mdesk @@ -8,7 +8,32 @@ a: 1; b: 2; c: 3; -w: 100; +// @notes See expr_c_like.mdesk for an explanation of why these have to be +// wrapped in parentheses in some cases. +l: 5*5 + 1; +m: (5*(5 + 1)); +n: ((5 + 1)*(5 + 1)); + +w: 0x100; x: a; y: b + w*a; z: c + w*b + w*w*a; + +// @notes The Metadesk expression parser will automatically accept any set with +// delimiters as a leaf. In this example we are doing an extra custom check to +// disallow these cases. +range: ([0,64)); +origin: ({50,100}); +p: (origin + {0, -20}); + +// @notes We can also generate errors that point at operators just as easily: +`no_&`: 5000 & 0xFF; + +// @notes The expression parser will produce an error if the series of nodes +// that form the expression cannot form a complete expression. +bad_1: a b c; +bad_2: z % b; + +// @notes Surprising things might count as expressions! Anything can be a leaf +// so long as it is not recognized as an operator. +`good?`: ! + ? * ==; diff --git a/source/md.h b/source/md.h index 6b358d6..f2afa8b 100644 --- a/source/md.h +++ b/source/md.h @@ -561,21 +561,28 @@ enum MD_NodeFlag_HasBraceLeft = (1<<4), MD_NodeFlag_HasBraceRight = (1<<5), + MD_NodeFlag_MaskSetDelimiters = (0x3F<<0), + MD_NodeFlag_IsBeforeSemicolon = (1<<6), MD_NodeFlag_IsAfterSemicolon = (1<<7), - MD_NodeFlag_IsBeforeComma = (1<<8), MD_NodeFlag_IsAfterComma = (1<<9), + MD_NodeFlag_MaskSeperators = (0xF<<6), + MD_NodeFlag_StringSingleQuote = (1<<10), MD_NodeFlag_StringDoubleQuote = (1<<11), MD_NodeFlag_StringTick = (1<<12), MD_NodeFlag_StringTriplet = (1<<13), + MD_NodeFlag_MaskStringDelimiters = (0xF<<10), + MD_NodeFlag_Numeric = (1<<14), MD_NodeFlag_Identifier = (1<<15), MD_NodeFlag_StringLiteral = (1<<16), MD_NodeFlag_Symbol = (1<<17), + + MD_NodeFlag_MaskLabelKind = (0xF<<14), }; typedef struct MD_Node MD_Node; From c968dd4329fc99b316dfc43360600c8f2c553935 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 16:52:09 -0700 Subject: [PATCH 24/28] [examples] C like expressions comments --- examples/expr/expr_c_like.c | 19 ++++++----- examples/expr/expr_c_like.mdesk | 56 ++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/examples/expr/expr_c_like.c b/examples/expr/expr_c_like.c index 44bcd1a..36713e6 100644 --- a/examples/expr/expr_c_like.c +++ b/examples/expr/expr_c_like.c @@ -1,7 +1,11 @@ /* ** Example: c like expressions ** -** TODO +** This example has the setup for parsing C expressions with the Metadesk +** expression parser. +** +** In the accompanying expr_c_like.mdesk there are more comments about using +** the expression system from the side of the Metadesk files. ** */ @@ -135,23 +139,24 @@ print_expression(FILE *out, MD_Expr *expr) case MD_ExprOprKind_Prefix: { - fprintf(out, "%.*s", MD_S8VArg(op->string)); + MD_Node *node = expr->md_node; + fprintf(out, "%.*s(", MD_S8VArg(op->string)); print_expression(out, expr->unary_operand); + fprintf(out, ")"); }break; case MD_ExprOprKind_Postfix: { + fprintf(out, "("); print_expression(out, expr->unary_operand); MD_String8 op_string = op->string; - // TODO(allen): formalize 'set delimiter mask' into the library - MD_u64 set_delimiter_mask = 0x3F; - if ((expr->md_node->flags & set_delimiter_mask) != 0) + if ((expr->md_node->flags & MD_NodeFlag_MaskSetDelimiters) != 0) { - fprintf(out, "%c...%c", op_string.str[0], op_string.str[1]); + fprintf(out, ")%c...%c", op_string.str[0], op_string.str[1]); } else { - fprintf(out, "%.*s", MD_S8VArg(op_string)); + fprintf(out, ")%.*s", MD_S8VArg(op_string)); } }break; diff --git a/examples/expr/expr_c_like.mdesk b/examples/expr/expr_c_like.mdesk index 00906d3..4f9dfe6 100644 --- a/examples/expr/expr_c_like.mdesk +++ b/examples/expr/expr_c_like.mdesk @@ -1,18 +1,72 @@ /* ** Example: c like expressions ** -** TODO +** Input for the expression parser in expr_c_like.c +** +** Many of the expressions in this example have to be expressed in slightly +** unusual ways (from the perspective of C) to make them work in the Metadesk +** grammar before the expression parser deals with them. Each of those cases in +** this file have comments explaining it. - The rule of thumb is that sometimes +** an expression needs to be placed in parentheses if it gets complicated. +** +** It may be useful to review intro/sets.mdesk to understand some the cases. */ a: 100; + +// @notes If we just have `b: array[a];` here the Metadesk parser will emit a +// warning and the tree won't have the shape we actually intend. We want: +// b: (array [a]) +// But we would actually get +// b: array +// [a] +// This happens because `b` is an "implicitly delimited" set, which _CANNOT_ +// contain explicitly delimited sets like `[a]` +// Adding the parentheses makes this into an "explicitly delimited" set which +// _CAN_ contain `[a]`. b: ( array[a] ); + + c: sizeof xyz; d: i*stride + j; e: 2.71828; + +// @notes The Metadesk expression parser uses parentheses like any common +// expression parser, but we have to remember that before the expression +// parser sees those parentheses, they actually get handled by the main +// Metadesk parser and turned into set nodes. +// +// If we just have `f: (H << 16) | (H >> 16);` here, then the tree we would +// get from the Metadesk parser is: +// f: (H << 16) +// | +// (H >> 16) +// +// Again the solution is to use an extra set of parentheses to tell the +// Metadesk parser that `(H << 16) | (H >> 16)` are all children of `f`. f: ( (H << 16) | (H >> 16) ); + +// @notes This time the problem is that we want the set delimiters `{}` to be +// visible to the expression parser. If we just have `g: {0, +1, -1}` then +// the shape of the tree becomes: +// g: (0, + 1, - 1) +// +// If we want to treat `{...}` as a leaf in an expression in this case we +// need an extra set of parentheses so that `{...}` is a child of g and not +// just the delimiters of the g node itself. g: ( {0, +1, -1} ); h: foo.bar; + +// @notes In these cases we want a nameless node that just contains an +// expression. We need parentheses so that these form sets. An alternative in +// this case is to have the C code do more work to use the `;` separators to +// find the beginning and ending of expressions like these. What we have here +// keeps things very simple. (i += 1); (j = 0); (k = i*stride + j); + +// @notes This one is pretty much the same as the first one, postfix index and +// postfix call operators both introduce an explicitly delimited set, which +// can't be contained in an implicitly delimited set. l: ( foo(bar, array[k], sizeof c) ); From 392ea138ccd6875e8ecbc40250e1bac58a9a40a0 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 19:04:55 -0700 Subject: [PATCH 25/28] [examples] comments in memory management example --- examples/integration/memory_management.c | 83 ++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/examples/integration/memory_management.c b/examples/integration/memory_management.c index e097438..ac60832 100644 --- a/examples/integration/memory_management.c +++ b/examples/integration/memory_management.c @@ -1,16 +1,62 @@ /* ** Example: memory management ** -** TODO +** This example shows how to use Metadesk in a program where memory management +** is important. For example if Metadesk is used as the basis for a config file +** system in an application, you may need to do things like reloading a config, +** or managing multiple configs with different arbitrary lifetimes. +** +** In the case of the simple metaprogram we manage memory by just having one +** global arena, and we never free anything. In this case we'll start using +** multiple arenas to create distinct lifetime "buckets" or "groups". +** +** Comments in this example explains a little about how Metadesk arenas work, + ** and tips for using them effectively. +** */ //~ includes and globals ////////////////////////////////////////////////////// +// @notes In this example we're just using the default implementations of +// the low level memory allocators. There's nothing wrong with doing it this +// way but in a codebase where something like a config file matters there +// may very well already be some custom allocators. Check out the overrides +// example for details on how to plug in custom allocators. +// +// Everything shown here about the Metadesk arena remains true either way, +// because arenas are implemented on top of the low level memory allocator. +// Basically you can think of the arena as performing allocation batching with +// whatever low level memory allocator is available. + #include "md.h" #include "md.c" + //~ pretend config file /////////////////////////////////////////////////////// +// @notes In this example we'll pretend we have a config file system, but we +// only show the part up to finishing the Metadesk parse. Each ConfigFile will +// carry a Metadesk arena, which handles the memory, and every version of the +// file data at each stage of processing. +// +// In a real system there would likely be at least one more stage of +// processing where a more processed version of the config that is made from +// analyzing the Metadesk tree. +// +// Here we can release a ConfigFile by simply releasing the arena because we +// have followed the simple rule that everything in the ConfigFile is +// *allocated on the arena*. If we were doing the analysis phase, we could +// keep this working by just writing the analyzer to allocate on the arena +// too. +// +// An alternative approach here that could save memory is to use the arena +// as an allocator for temporary intermediates. In this approach during the +// analysis stage the final data structure would be allocated outside the +// arena used for parsing, and everything that the final structure needs would +// be copied out. Then the parse could be thrown away. This approach saves +// memory at the cost of making problems harder to trouble shoot, and +// sometimes more time in analysis to copy things out of the temporary arena. + typedef struct ConfigFile{ MD_Arena *arena; MD_String8 file_name; @@ -23,12 +69,37 @@ ConfigFile* new_config_file_from_file_name(char *file_name_cstr) { MD_Arena *arena = MD_ArenaAlloc(); + + // @notes MD_PushArray and MD_PushArrayZero are the fundamental allocation + // operations to do with a Metadesk arena. Metadesk APIs that take an + // MD_Arena* parameter are APIs that need to do allocation. When we pass + // an arena in one of these APIs the returned data is allocated using that + // arena and we say that the data is "allocated on the arena". ConfigFile *result = MD_PushArrayZero(arena, ConfigFile, 1); + // @notes We explicitly copy the file name onto the arena so that we can + // be totally sure that it has the same lifetime as everything else in + // the ConfigFile. MD_String8 file_name = MD_S8Copy(arena, MD_S8CString(file_name_cstr)); + + // @notes Here we break down MD_ParseWholeFile into it's two stages + // explicitly so that we can save the contents and the parse in the + // ConfigFile. MD_String8 contents = MD_LoadEntireFile(arena, file_name); MD_ParseResult parse = MD_ParseWholeString(arena, file_name, contents); + // @notes This part can be a little bit subtle. First we allocated the + // arena with MD_ArenaAlloc. Then we used the arena to allocate a + // ConfigFile. Now we are storing a pointer to the arena in the config. + // The subtle part is that if you're used to thinking in terms of + // 'ownership' it seems like the config owns the arena, but if we release + // the arena, we also release the config. + // + // A different way to think about it is that the arena is a handle that + // manages allocation lifetimes. The ConfigFile and all of the data it + // holds share the same lifetime, so they are all allocated on the same + // "lifetime handle" (i.e. the same arena). The ConfigFile is the root of + // all that data so it also holds the handle for releasing later. result->arena = arena; result->file_name = file_name; result->contents = contents; @@ -40,12 +111,10 @@ new_config_file_from_file_name(char *file_name_cstr) void release_config_file(ConfigFile *file) { - if (file != 0) - { - MD_ArenaRelease(file->arena); - } + MD_ArenaRelease(file->arena); } + //~ just to simulate new config files coming from somewhere /////////////////// int in_files_count = 0; @@ -77,6 +146,10 @@ main(int argc, char **argv) in_file_names = argv + 1; + // @notes The idea here is to simulate a situation where an allocate and + // never free strategy would lead to growing memory usage over time + // (i.e. a memory leak). + // pretend there are unpredictable lifetimes tied to real-time events { // first we get three config files From f30295fb5cd419396dbfa55338aa2f5920a892e3 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 19:16:36 -0700 Subject: [PATCH 26/28] [examples] notes on the scratch pool --- examples/integration/memory_management.c | 38 ++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/examples/integration/memory_management.c b/examples/integration/memory_management.c index ac60832..aa15390 100644 --- a/examples/integration/memory_management.c +++ b/examples/integration/memory_management.c @@ -174,3 +174,41 @@ main(int argc, char **argv) release_config_file(files[2]); } } + +// @notes A final note on "scratch arenas". +// +// Often it is useful to use an arena for a temporary allocation that will be +// thrown away by the end of the current scope. One could create a new arena +// with MD_ArenaAlloc and later release it with MD_ArenaRelease, but this +// sort of case is perfect for using the thread-local scratch pool. +// +// To get an arena for scratch work from the pool one uses: +// MD_ArenaTemp scratch = MD_GetScratch(0, 0); +// To allocate with it: +// MD_WhateverArenaApi(scratch.arena, ...); +// And to release it: +// MD_ReleaseScratch(scratch); +// +// If an arena is being used for allocating something to return to the caller +// it is important that it not also be the scratch, or else when the scratch +// release happens, all of the memory that was supposed to stay allocated +// for the caller to see will also be released (or worse, marked as released +// but still valid looking for some time). +// +// To avoid this when getting a scratch arena the API MD_GetScratch allows you +// to specify arenas you are already using, to force it to pick one you are +// not using. In 99.99% of cases a call to this API either looks like: +// MD_GetScratch(0, 0); +// Or +// MD_GetScratch(&arena, 1); +// +// If there are more than one arena already in use when a new scratch is +// needed they can all be specified by packing them into an array: +// MD_Arena *arena_conflicts[2] = {arena1, arena2}; +// MD_GetScratch(arena_conflicts, 2); +// +// Watch out! If the scratch pool doesn't have a non-conflicting arena then +// it will return a null handle, likely leading to a crash. This can be +// avoided by defining a higher value for #define MD_IMPL_ScratchCount. +// But it's generally possible and a lot better to avoid this path. + From e0df4e5bed260db58df6065e33c76c0f267c2546 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 19:42:00 -0700 Subject: [PATCH 27/28] [examples] comments in multi-threaded parse example --- examples/integration/multi_threaded.c | 58 +++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/examples/integration/multi_threaded.c b/examples/integration/multi_threaded.c index 5962383..c7fc981 100644 --- a/examples/integration/multi_threaded.c +++ b/examples/integration/multi_threaded.c @@ -1,7 +1,17 @@ /* ** Example: multi threaded parse ** -** TODO +** This example shows how to arrange a multi-threaded Metadesk parser. The +** strategy used is to make each *.mdesk file into an independent task. +** +** The goal in this example is to have all of the files parsed and visible at +** the same time by the end. Another conceivable way to make a multi-threaded +** Metadesk parser would be to use each parse as a temporary and throw them +** away after extracting out the important parts. +** +** This example depends directly on OS headers, intrinsics, and functions, +** so you'll have to re-interpret those parts to whatever OS you are targeting. +** */ //~ includes and globals ////////////////////////////////////////////////////// @@ -9,7 +19,11 @@ #include "md.h" #include "md.c" -#include +#if MD_OS_WINDOWS +# include +#else +# error Not implemented for this OS +#endif //~ multi-threaded parse setup //////////////////////////////////////////////// @@ -18,21 +32,35 @@ #if MD_OS_WINDOWS # define atomic_inc_then_eval_u64(p) InterlockedIncrement64((LONG64*)p) #else -# error Not implemented for this compiler +# error Not implemented for this OS #endif +// @notes We use one structure that describes the whole multi-threaded parse +// work load, and carries the shared state values for synchronizing the +// worker threads. typedef struct TaskData { + // the set of tasks MD_u64 task_max; char **tasks; + // synchronization volatile MD_u64 task_counter; volatile MD_u64 thread_counter; } TaskData; +// @notes Each thread gets it's own thread data. +// +// The library doesn't make any of it's data structures thread safe but all of +// the calls are thread safe so long as different threads are operating on +// different data structures. So we arrange for each worker thread to get it's +// own arena, and set of lists for collecting parse results. typedef struct ThreadData { + // shared TaskData *task; + + // unique-to-thread MD_Arena *arena; MD_Node *list; MD_MessageList errors; @@ -44,17 +72,21 @@ parse_worker_loop(ThreadData *thread_data) TaskData *task = thread_data->task; for (;;) { + // atomically get the next unhandled task index MD_u64 task_index = atomic_inc_then_eval_u64(&task->task_counter) - 1; if (task_index >= task->task_max) { break; } + + // load and parse the file specified by this task. MD_String8 file_name = MD_S8CString(task->tasks[task_index]); MD_ParseResult parse = MD_ParseWholeFile(thread_data->arena, file_name); MD_MessageListConcat(&thread_data->errors, &parse.errors); MD_PushNewReference(thread_data->arena, thread_data->list, parse.node); } + // atomically count the threads as they finish atomic_inc_then_eval_u64(&task->thread_counter); } @@ -98,6 +130,7 @@ main(int argc, char **argv) } // launch the worker threads + // (no worker thread 0) for (int i = 1; i < THREAD_COUNT; i += 1) { #if MD_OS_WINDOWS @@ -108,6 +141,7 @@ main(int argc, char **argv) #endif } + // let the main thread act as thread 0 parse_worker_loop(&threads[0]); // wait for all threads to be finished @@ -148,6 +182,24 @@ main(int argc, char **argv) } } + // @notes In this example we are done, but in some cases it might be useful + // to merge the results of a multi-threaded parse to make it as if it was + // a single threaded parse, and it turns out this is quite easy to do from + // here. + // + // We merge all of the arenas together so that we can handle all of the + // memory with a single arena moving forward. This relies on the default + // arena implementation which has an 'absorb' operation. If you plug in + // your own arena via oerrides it's up to that implementation what your + // options are for this part. + // + // The list of roots and the list of messages can be concatenated. + // + // All of these operations (absorb and the concat operations) move + // information out of the right hand operand and into the left hand + // operand, leaving the individual pieces from the worker threads invalid + // after the merge. + // combine results MD_Arena *arena = threads[0].arena; MD_Node *list = threads[0].list; From 5516a221eeb0ba9e6ef92ee2c0fcd6bf6d6f699e Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Sun, 10 Oct 2021 20:30:31 -0700 Subject: [PATCH 28/28] [examples] fix typo in build script --- bin/build_examples.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build_examples.sh b/bin/build_examples.sh index c1d0bed..ca7c95b 100755 --- a/bin/build_examples.sh +++ b/bin/build_examples.sh @@ -6,7 +6,7 @@ cd "$(dirname "$0")" cd .. ###### Script ################################################################# -echo "~~~ Build All Exampes ~~~" +echo "~~~ Build All Examples ~~~" bin/bld_core.sh show_ctx examps="examples"