#include "md.h" #include "md.c" #include // NOTE: https://www.pcg-random.org/download.html typedef struct RandomSeries{ MD_u64 state; // NOTE: RNG state. All values are possible. MD_u64 inc; // Controls which RNG sequence (stream) is selected. Odd at all times. } RandomSeries; static MD_u32 rand_U32(RandomSeries* rng) { MD_u64 oldstate = rng->state; rng->state = oldstate * 6364136223846793005ULL + rng->inc; MD_u32 xorshifted = ((oldstate >> 18u) ^ oldstate) >> 27u; MD_u32 rot = oldstate >> 59u; return (xorshifted >> rot) | (xorshifted << ((-rot) & 31)); } static RandomSeries rand_seed(MD_u64 initstate, MD_u64 initseq) { RandomSeries result = {0}; result.state = 0U; result.inc = (initseq << 1u) | 1u; // NOTE: inc must be odd rand_U32(&result); result.state += initstate; rand_U32(&result); return result; } MD_GLOBAL struct { RandomSeries *random_series; MD_NodeTable *production_table; } globals = {0}; static void TagSquareBracketSetsAsOptional(MD_Node *node, MD_Node *optional_tag) { if(node->kind == MD_NodeKind_Label && (node->flags & MD_NodeFlag_BracketLeft) && (node->flags & MD_NodeFlag_BracketRight)) { if(node->string.size) // NOTE(mal): Production. Tag belongs to the first child { MD_Assert(!MD_NodeIsNil(node->first_child)); MD_PushTag(node->first_child, optional_tag); } else { MD_PushTag(node, optional_tag); } } for(MD_EachNode(child, node->first_child)) { TagSquareBracketSetsAsOptional(child, optional_tag); } } static MD_Node * NewChildLabel(MD_Node *parent, MD_String8 label) { MD_Node *result = 0; result = MD_MakeNodeFromString(MD_NodeKind_Label, MD_S8Lit(""), 0, 0, label); if(parent) { MD_PushChild(parent, result); } return result; } static MD_Node * NewChild(MD_Node *parent) { MD_Node *result = NewChildLabel(parent, MD_S8Lit("")); return result; } #define OPTIONAL_TAG "optional" static void PrintRule(MD_Node *rule) { MD_b32 is_literal_char = rule->flags & MD_NodeFlag_CharLiteral; MD_b32 optional = MD_NodeHasTag(rule, MD_S8Lit(OPTIONAL_TAG)); if(optional) { printf("["); } for(MD_EachNode(tag, rule->first_tag)) { if(!MD_StringMatch(tag->string, MD_S8Lit(OPTIONAL_TAG), 0)) { printf("@%.*s ", MD_StringExpand(tag->string)); } } if(is_literal_char) { printf("'"); } MD_b32 has_children = !MD_NodeIsNil(rule->first_child); if(has_children) { MD_Assert(rule->string.size == 0); for(MD_EachNode(rule_element, rule->first_child)) { PrintRule(rule_element); if(!MD_NodeIsNil(rule_element->next)) { printf(" "); } } } else { MD_Assert(rule->string.size > 0); printf("%.*s", MD_StringExpand(rule->string)); } if(is_literal_char) { printf("'"); } if(optional) { printf("]"); } } typedef enum AllowedOperationFlags AllowedOperationFlags; enum AllowedOperationFlags { AllowedOperationFlag_Fill = 1<<0, AllowedOperationFlag_Tag = 1<<1, }; static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *cur_node, AllowedOperationFlags allowed); static void Extend(MD_String8 *s, char c) { *s = MD_PushStringF("%.*s%c", MD_StringExpand(*s), c); } static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_node, AllowedOperationFlags allowed) { for(MD_EachNode(rule_element, rule->first_child)) { MD_b32 expand = 1; if(MD_NodeHasTag(rule_element, MD_S8Lit(OPTIONAL_TAG))) { expand = rand_U32(globals.random_series)%2; } if(expand) { MD_Node *node_to_tag = 0; MD_b32 is_markup = 0; AllowedOperationFlags new_flags = 0; for(MD_EachNode(tag_node, rule_element->first_tag)){ if(MD_StringMatch(tag_node->string, MD_S8Lit("child"), 0)) { cur_node = NewChild(cur_node); allowed &= ~AllowedOperationFlag_Tag; // NOTE(mal): Tag parameters are not tags } else if(MD_StringMatch(tag_node->string, MD_S8Lit("sibling"), 0)) { cur_node = NewChild(cur_node->parent); } else if(MD_StringMatch(tag_node->string, MD_S8Lit("fill"), 0)) { new_flags |= AllowedOperationFlag_Fill; } else if(MD_StringMatch(tag_node->string, MD_S8Lit("tag"), 0)) { new_flags |= AllowedOperationFlag_Tag; node_to_tag = cur_node; cur_node = NewChild(0); cur_node->kind = MD_NodeKind_Tag; } else if(MD_StringMatch(tag_node->string, MD_S8Lit(OPTIONAL_TAG), 0)) { } else if(MD_StringMatch(tag_node->string, MD_S8Lit("markup"), 0)) { is_markup = 1; } else { MD_Assert(!"Not implemented"); } } allowed |= new_flags; MD_b32 has_children = !MD_NodeIsNil(rule_element->first_child); if(has_children) { ExpandRule(rule_element, out_strings, cur_node, allowed); } else { if(rule_element->flags & MD_NodeFlag_CharLiteral) // NOTE(mal): Terminal production { char c = 0; if(rule_element->string.size == 2 && rule_element->string.str[0] == '\\') { switch(rule_element->string.str[1]){ case '\\': c = '\\'; break; case '\'': c = '\''; break; case '"': c = '\"'; break; case 'n': c = '\n'; break; } } else { c = rule_element->string.str[0]; } MD_String8 character = MD_PushStringF("%c", c); MD_PushStringToList(out_strings, character); if(allowed & (AllowedOperationFlag_Fill|AllowedOperationFlag_Tag)) { Extend(&cur_node->whole_string, c); if(!is_markup) { Extend(&cur_node->string, c); } } } else // NOTE(mal): Non-terminal production { MD_Node * production = MD_NodeTable_Lookup(globals.production_table, rule_element->string)->node; MD_Assert(production); ExpandProduction(production, out_strings, cur_node, allowed); } } if(node_to_tag) { MD_PushTag(node_to_tag, cur_node); cur_node = node_to_tag; } allowed &= ~new_flags; } } } static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *cur_node, AllowedOperationFlags allowed) { MD_i64 rule_count = MD_ChildCountFromNode(production); int rule_number = rand_U32(globals.random_series)%rule_count; MD_Node *rule = MD_ChildFromIndex(production, rule_number); ExpandRule(rule, out, cur_node, allowed); } static MD_Node * FindNonTerminalProduction(MD_Node *node, MD_NodeTable *visited) { MD_Node *result = 0; MD_b32 should_visit = 1; if(!MD_NodeIsNil(node->first_child) && node->string.size) { if(MD_NodeTable_Lookup(visited, node->string)) { should_visit = 0; } else { MD_b32 inserted = MD_NodeTable_Insert(visited, MD_NodeTableCollisionRule_Overwrite, node->string, node); MD_Assert(inserted); } } if(should_visit) { if(MD_NodeIsNil(node->first_child)) { if(node->flags & MD_NodeFlag_CharLiteral) { } else { MD_NodeTableSlot *slot = MD_NodeTable_Lookup(globals.production_table, node->string); if(slot) { MD_Node *production = slot->node; result = FindNonTerminalProduction(production, visited); } else { result = node; } } } else { for(MD_EachNode(child, node->first_child)) { result = FindNonTerminalProduction(child, visited); if(result) { break; } } } } return result; } static MD_b32 EqualTrees(MD_Node *a, MD_Node *b); static MD_b32 EqualList(MD_Node *a, MD_Node *b) { MD_b32 result = 1; while(!MD_NodeIsNil(a) && !MD_NodeIsNil(b)) { if(!EqualTrees(a, b)) { result = 0; } a = a->next; b = b->next; } return result; } static MD_b32 EqualTrees(MD_Node *a, MD_Node *b) { MD_b32 result = (a->kind == b->kind && MD_StringMatch(a->string, b->string, 0) && MD_StringMatch(a->string, b->string, 0)); result &= EqualList(a->first_tag, b->first_tag); result &= EqualList(a->first_child, b->first_child); return result; } typedef struct Test Test; struct Test { MD_Node *file_control_node; MD_String8List expanded_list; }; int TestCompare(const void *a_, const void *b_) { int result = 1; Test *a = (Test *) a_; Test *b = (Test *) b_; if(a->expanded_list.total_size < b->expanded_list.total_size) { result = -1; } return result; } int main(int argument_count, char **arguments) { MD_Node *grammar = MD_ParseWholeFile(MD_S8Lit("tests/grammar.md")); // NOTE(mal): In order to get a BNF-like syntax, I feed the MD output through two transformations: // 1) Tag []-style sets as optional MD_Node *optional_tag = 0; { MD_ParseResult parse_result = MD_ParseOneNode(MD_S8Lit(""), MD_S8Lit("@"OPTIONAL_TAG" is_a_tag")); optional_tag = parse_result.node->first_tag; } TagSquareBracketSetsAsOptional(grammar, optional_tag); // NOTE(mal): 2) Divide rules around '|' operators and transplant them into a new tree structure // productions > production > rules > rule > rule_element MD_Node *productions = NewChild(0); for(MD_EachNode(symbol, grammar->first_child)) { MD_Node *production = NewChildLabel(productions, symbol->string); MD_Node *rule = NewChild(production); for(MD_Node *rule_element = symbol->first_child; !MD_NodeIsNil(rule_element); ) { MD_Node *next = rule_element->next; MD_b32 has_children = !MD_NodeIsNil(rule_element->first_child); if(has_children) { MD_PushChild(rule, rule_element); } else { if(MD_StringMatch(rule_element->string, MD_S8Lit("|"), 0) && !(rule_element->flags & MD_NodeFlag_CharLiteral)) { rule = NewChild(production); } else { MD_PushChild(rule, rule_element); } } rule_element = next; } } // NOTE(mal): Debug rules after transformations #if 0 for(MD_EachNode(production, productions->first_child)) { printf("%.*s: \n", MD_StringExpand(production->string)); for(MD_EachNode(rule, production->first_child)) { printf(" "); PrintRule(rule); printf("\n"); } } #endif // NOTE(mal): Build production hash table MD_NodeTable production_table_ = {0}; globals.production_table = &production_table_; for(MD_EachNode(production, productions->first_child)) { MD_b32 inserted = MD_NodeTable_Insert(globals.production_table, MD_NodeTableCollisionRule_Overwrite, production->string, production); MD_Assert(inserted); } // NOTE(mal): Check for root production MD_Node* file_production = 0; { MD_NodeTableSlot *file_production_slot = MD_NodeTable_Lookup(globals.production_table, MD_S8Lit("file")); if(!file_production_slot) { fprintf(stderr, "Error: Grammar file does not specify \"file\" production\n"); goto error; } file_production = file_production_slot->node; } // NOTE(mal): Check that all branches lead to terminal nodes MD_NodeTable visited_productions = {0}; MD_Node *non_terminal_production = FindNonTerminalProduction(file_production, &visited_productions); if(non_terminal_production) { fprintf(stderr, "Error: Non-terminal production \"%.*s\"\n", MD_StringExpand(non_terminal_production->string)); goto error; } // NOTE(mal): Check that all productions are reachable for(MD_EachNode(production, productions->first_child)) { MD_NodeTableSlot *slot = MD_NodeTable_Lookup(&visited_productions, production->string); if(!slot) { fprintf(stderr, "Warning: Unreachable production \"%.*s\"\n", MD_StringExpand(production->string)); } } RandomSeries random_series = rand_seed(0, 0); // NOTE(mal): Reproduceable globals.random_series = &random_series; MD_Node* node = MD_NodeTable_Lookup(globals.production_table, MD_S8Lit("file"))->node; MD_u32 test_count = 1000; Test *tests = (Test *) calloc(test_count, sizeof(Test)); for(MD_u32 i = 0; i < test_count;) { tests[i].file_control_node = NewChild(0); tests[i].file_control_node->kind = MD_NodeKind_File; // NOTE(mal): Generate a random MD file ExpandProduction(node, &tests[i].expanded_list, tests[i].file_control_node, 0); // NOTE(mal): Accept any non-empty file if(tests[i].expanded_list.total_size) { ++i; } } // NOTE(mal): Sort tests based on length qsort(tests, test_count, sizeof(Test), TestCompare); MD_u32 i_test = 0; for(int i = 0; i < test_count; ++i) { if(tests[i].expanded_list.total_size > 0) { MD_String8 expanded = MD_JoinStringList(tests[i].expanded_list); MD_Node *file_node = MD_ParseWholeString(MD_S8Lit(""), expanded); file_node->string = file_node->whole_string = (MD_String8){0}; // printf("> %.*s <\n", MD_StringExpand(expanded)); if(!EqualTrees(file_node, tests[i].file_control_node)) { printf("\nFailed test %d\n", i_test); printf("> %.*s <\n", MD_StringExpand(expanded)); printf("MD:\n"); MD_OutputTree(stdout, file_node); printf("Grammar:\n"); MD_OutputTree(stdout, tests[i].file_control_node); printf("\n"); return -1; } ++i_test; } } printf("Passed %d tests\n", test_count); error: return 0; }