From 71d5235f81aed3b6feabf9b169c45ed89f941062 Mon Sep 17 00:00:00 2001 From: Miguel Lechon Date: Mon, 22 Feb 2021 14:55:39 +0100 Subject: [PATCH] [grammar test] Fairly complete test grammars. --- tests/grammar.c | 74 +++++++++-------- tests/grammar.md | 206 +++++++++++++++++++---------------------------- 2 files changed, 124 insertions(+), 156 deletions(-) diff --git a/tests/grammar.c b/tests/grammar.c index bcae6a6..c878d87 100644 --- a/tests/grammar.c +++ b/tests/grammar.c @@ -141,22 +141,23 @@ static void PrintRule(MD_Node *rule) } } -typedef enum AllowedOperationFlags AllowedOperationFlags; -enum AllowedOperationFlags +typedef enum OperationFlags OperationFlags; +enum OperationFlags { - AllowedOperationFlag_Fill = 1<<0, - AllowedOperationFlag_Tag = 1<<1, + OperationFlag_Fill = 1<<0, + OperationFlag_Markup = 1<<1, + OperationFlag_Tag = 1<<2, }; -static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *cur_node, - AllowedOperationFlags allowed, MD_u32 max_depth, MD_u32 depth); - static void Extend(MD_String8 *s, char c) { *s = MD_PushStringF("%.*s%c", MD_StringExpand(*s), c); } -static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_node, AllowedOperationFlags allowed, +static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *cur_node, + OperationFlags op_flags, MD_u32 max_depth, MD_u32 depth); + +static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_node, OperationFlags op_flags, MD_u32 max_depth, MD_u32 depth) { for(MD_EachNode(rule_element, rule->first_child)) @@ -178,12 +179,12 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_ if(expand) { MD_Node *node_to_tag = 0; - AllowedOperationFlags new_flags = 0; + OperationFlags old_op_flags = op_flags; for(MD_EachNode(tag_node, rule_element->first_tag)){ if(MD_StringMatch(tag_node->string, MD_S8Lit("child"), 0)) { cur_node = NewChild(cur_node); - allowed &= ~AllowedOperationFlag_Tag; // NOTE(mal): Tag parameters are not tags + op_flags &= ~OperationFlag_Tag; // NOTE(mal): Tag parameters are not tags } else if(MD_StringMatch(tag_node->string, MD_S8Lit("sibling"), 0)) { @@ -191,15 +192,19 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_ } else if(MD_StringMatch(tag_node->string, MD_S8Lit("fill"), 0)) { - new_flags |= AllowedOperationFlag_Fill; + op_flags |= OperationFlag_Fill; } else if(MD_StringMatch(tag_node->string, MD_S8Lit("tag"), 0)) { - new_flags |= AllowedOperationFlag_Tag; + op_flags |= OperationFlag_Tag; node_to_tag = cur_node; cur_node = NewChild(0); cur_node->kind = MD_NodeKind_Tag; } + else if(MD_StringMatch(tag_node->string, MD_S8Lit("markup"), 0)) + { + op_flags |= OperationFlag_Markup; + } else if(MD_StringMatch(tag_node->string, MD_S8Lit(OPTIONAL_TAG), 0)) { } @@ -209,12 +214,10 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_ } } - allowed |= new_flags; - MD_b32 has_children = !MD_NodeIsNil(rule_element->first_child); if(has_children) { - ExpandRule(rule_element, out_strings, cur_node, allowed, max_depth, depth+1); + ExpandRule(rule_element, out_strings, cur_node, op_flags, max_depth, depth+1); } else { @@ -238,17 +241,20 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_ MD_String8 character = MD_PushStringF("%c", c); MD_PushStringToList(out_strings, character); - if(allowed & (AllowedOperationFlag_Fill)) + if(op_flags & OperationFlag_Fill) { Extend(&cur_node->whole_string, c); - Extend(&cur_node->string, c); + if(!(op_flags & OperationFlag_Markup)) + { + Extend(&cur_node->string, c); + } } } else // NOTE(mal): Non-terminal production { MD_Node * production = MD_NodeTable_Lookup(globals.production_table, rule_element->string)->node; MD_Assert(production); - ExpandProduction(production, out_strings, cur_node, allowed, max_depth, depth+1); + ExpandProduction(production, out_strings, cur_node, op_flags, max_depth, depth+1); } } @@ -258,13 +264,13 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_ cur_node = node_to_tag; } - allowed &= ~new_flags; + op_flags = old_op_flags; } } } static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *cur_node, - AllowedOperationFlags allowed, MD_u32 max_depth, MD_u32 depth) + OperationFlags op_flags, MD_u32 max_depth, MD_u32 depth) { MD_i64 rule_count = MD_ChildCountFromNode(production); @@ -276,7 +282,7 @@ static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node * rule = MD_ChildFromIndex(production, rule_number); }while(GET_DEPTH(rule)+depth > max_depth); - ExpandRule(rule, out, cur_node, allowed, max_depth, depth); + ExpandRule(rule, out, cur_node, op_flags, max_depth, depth); } static MD_Node * FindNonTerminalProduction(MD_Node *node, MD_NodeTable *visited) @@ -334,28 +340,27 @@ static MD_Node * FindNonTerminalProduction(MD_Node *node, MD_NodeTable *visited) return result; } -// TODO: use MD_NodeDeepMatch instead static MD_b32 EqualTrees(MD_Node *a, MD_Node *b); static MD_b32 EqualList(MD_Node *a, MD_Node *b) { MD_b32 result = 1; - while(!MD_NodeIsNil(a) || !MD_NodeIsNil(b)) { if(!EqualTrees(a, b)) { result = 0; + break; } a = a->next; b = b->next; } - return result; } static MD_b32 EqualTrees(MD_Node *a, MD_Node *b) { - MD_b32 result = (a->kind == b->kind && MD_StringMatch(a->string, b->string, 0) && - MD_StringMatch(a->string, b->string, 0)); + MD_b32 result = (a->kind == b->kind && + MD_StringMatch(a->string, b->string, 0) && + MD_StringMatch(a->whole_string, b->whole_string, 0)); result &= EqualList(a->first_tag, b->first_tag); result &= EqualList(a->first_child, b->first_child); return result; @@ -435,7 +440,6 @@ static void ComputeElementDepth(MD_Node *re) SET_DEPTH(re, result); } - // NOTE(mal): Compares first by size then alphabetically static int StringCompare(MD_String8 a, MD_String8 b) { @@ -537,11 +541,15 @@ int main(int argument_count, char **arguments) // NOTE(mal): Check that all branches lead to terminal nodes MD_NodeTable visited_productions = {0}; - MD_Node *non_terminal_production = FindNonTerminalProduction(file_production, &visited_productions); - if(non_terminal_production) + + for(MD_EachNode(production, productions->first_child)) { - fprintf(stderr, "Error: Non-terminal production \"%.*s\"\n", MD_StringExpand(non_terminal_production->string)); - goto error; + MD_Node *non_terminal_production = FindNonTerminalProduction(production, &visited_productions); + if(non_terminal_production) + { + fprintf(stderr, "Error: Non-terminal production \"%.*s\"\n", MD_StringExpand(non_terminal_production->string)); + goto error; + } } // NOTE(mal): Check that all productions are reachable @@ -644,13 +652,10 @@ int main(int argument_count, char **arguments) test.expected_output = NewChild(0); test.expected_output->kind = MD_NodeKind_File; - // static int loop = 0; ++loop; if(loop == 893) BP; - MD_String8List string_list = {0}; // NOTE(mal): Generate a random MD file ExpandProduction(file_production_node, &string_list, test.expected_output, 0, max_production_depth, 0); test.input = MD_JoinStringList(string_list); - // if(MD_StringMatch(test.input, MD_S8Lit("A:A:A A\n"), 0)) BP; Test *prev = 0; for(Test *cur = first_test; cur; cur = cur->next) @@ -699,7 +704,6 @@ int main(int argument_count, char **arguments) MD_OutputTree(stdout, test->expected_output); printf("\n"); return -1; } - ++i_test; } diff --git a/tests/grammar.md b/tests/grammar.md index 08ad129..aab017a 100644 --- a/tests/grammar.md +++ b/tests/grammar.md @@ -1,60 +1,3 @@ -// Arbitrarily deep tree, possibly empty -// file : [@child set] -// set : '{' [@child set] '}' | set [' ' @sibling set] - -// Labeled leaves -// file : [@child set] -// set : @fill 'A' | '{' [@child set] '}' | set [' ' @sibling set] - -// Labels also on internal nodes ('iset' is an implicitly separated set) -// file : [@child set] -// set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] curly_set | set [' ' @sibling set] -// iset /* implicit separator */ : @fill 'A' [':' @child iset] | @fill 'A'[':' curly_set] | @fill 'A' [' ' @sibling iset] | @fill 'A' [' ' @sibling curly_set] -// curly_set : '{' [@child set] '}' - -// Labels also on internal nodes (v2) ('iset' is an implicitly separated set) -// file : [@child set] -// set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] curly_set | set [' ' @sibling set] -// iset /* implicit separator */ : @fill 'A' [iset_tail] -// iset_tail : ':' @child iset | ' ' @sibling iset | ':' curly_set | ' ' @sibling curly_set -// curly_set : '{' [@child set] '}' - -// Tags -// file : [@child set] -// set : {[tag_list] untagged_set} -// iset : {[tag_list] untagged_iset} -// tag_list : '@' @tag tag ' ' [tag_list] -// tag : 'T' -// untagged_set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] curly_set | set [' ' @sibling set] -// untagged_iset : @fill 'A' [iset_tail] -// iset_tail : ':' @child iset | ' ' @sibling iset | ':' curly_set | ' ' @sibling curly_set -// curly_set : '{' [@child set] '}' - -// Tag parameters -file : [@child set] -set : {[tag_list] untagged_set} -iset : {[tag_list] untagged_iset} -tag_list : '@' @tag tag ' ' [tag_list] -tag : @fill 'T'['(' [@child set] ')'] -untagged_set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] curly_set | set [' ' @sibling set] -untagged_iset : @fill 'A' [iset_tail] -iset_tail : ':' @child iset | ' ' @sibling iset | ':' curly_set | ' ' @sibling curly_set -curly_set : '{' [@child set] '}' - - -// top-level and set separators -// file : [@child file_set_list] -// file_set_list : set [file_set_separator @sibling file_set_list] -// file_set_separator : ' ' | '\n' -// set : '{' [@child set] '}' | set [separator @sibling set] -// separator : ' ' | ',' - - - -/////////////////////////////////////////// REFERENCE /////////////////////////////////////////////////////////// - -/* - /* MetaDesk grammar with semantic annotations * * Each line represents a BNF-esque production: @@ -66,73 +9,94 @@ curly_set : '{' [@child set] '}' * and miscellaneous semantics (@fill, @markup) */ -file : [@child file_set_list] -file_set_list : { [tag_list] set [file_set_separator @sibling file_set_list] } -file_set_separator : ' ' | '\n' +//// Arbitrarily deep tree, possibly empty +// file : [@child set] +// set : '{' [@child set] '}' | set [' ' @sibling set] -// TODO(mal): Unify file_set_separator and set_separator by allowing ',' and ';' everywhere ? -set_list : { [tag_list] set [set_separator @sibling set_list] } -set_separator : ' ' | '\n' // TODO(mal): | ',' | ';' +//// Labeled leaves +// file : [@child set] +// set : @fill 'A' | '{' [@child set] '}' | set [' ' @sibling set] + +//// Labels also on internal nodes ('iset' is an implicitly separated set) +// file : [@child set] +// set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set] +// iset : @fill 'A' [iset_tail] +// iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set +// scoped_set : '{' [@child set] '}' + +//// Tags +// file : [@child set] +// set : {[tag_list] untagged_set} +// iset : {[tag_list] untagged_iset} +// tag_list : '@' @tag tag ' ' [tag_list] +// tag : @fill 'T'['(' [@child set] ')'] +// untagged_set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set] +// untagged_iset : @fill 'A' [iset_tail] +// iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set +// scoped_set : '{' [@child set] '}' + +//// Alternative scope markers +// file : [@child set] +// set : {[tag_list] untagged_set} +// iset : {[tag_list] untagged_iset} +// tag_list : '@' @tag tag ' ' [tag_list] +// tag : @fill 'T'['(' [@child set] ')'] +// untagged_set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set] +// untagged_iset : @fill 'A' [iset_tail] +// iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set +// scoped_set : '{' [@child set] '}' | scope_beg [@child set] scope_end +// scope_beg : '(' | '[' +// scope_end : ')' | ']' + +//// Identifiers +// file : [@child set] +// set : {[tag_list] untagged_set} +// iset : {[tag_list] untagged_iset} +// tag_list : '@' @tag tag ' ' [tag_list] +// tag : @fill id['(' [@child set] ')'] +// untagged_set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set] +// untagged_iset : @fill 'A' [iset_tail] +// iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set +// scoped_set : '{' [@child set] '}' | scope_beg [@child set] scope_end +// scope_beg : '(' | '[' +// scope_end : ')' | ']' +// id : alpha [alphanumeric] +// alphanumeric : alpha [alphanumeric] | digit [alphanumeric] | '_' [alphanumeric] +// alpha : lowercase | uppercase +// lowercase : 'a'|'b'|'c'|'d'|'e'|'f'|'g'|'h'|'i'|'j'|'k'|'l'|'m'|'n'|'o'|'p'|'q'|'r'|'s'|'t'|'u'|'v'|'w'|'z'|'y'|'z' +// uppercase : 'A'|'B'|'C'|'D'|'E'|'F'|'G'|'H'|'I'|'J'|'K'|'L'|'M'|'N'|'O'|'P'|'Q'|'R'|'S'|'T'|'U'|'V'|'W'|'Z'|'Y'|'Z' +// digit : '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9' + +//// General labels +file : [@child set] +set : {[tag_list] untagged_set} +iset : {[tag_list] untagged_iset} tag_list : '@' @tag tag ' ' [tag_list] - -/* - tag : identifier [@markup '(' [@child set_list] @markup ')'] - set : @fill leaf | @fill identifier ':' @child @fill leaf | [@fill identifier ':'] '{' [@child set_list] '}' | [@fill identifier ':'] set_open [@child set_list] set_close - set : @fill leaf | @fill identifier ':' @child space_separated_set_list | [@fill identifier ':'] set_open [@child set_list] set_close -*/ - -tag : identifier -set : @fill leaf [':' @child space_separated_set_list '\n'] | @fill leaf [':' '{' [@child set_list] '}'] | space_separated_set_list '\n' | '{' [@child set_list] '}' - -space_separated_set_list : { [tag_list] set [' ' @sibling space_separated_set_list] } - - -// set_open : '[' | '(' -// set_close : ']' | ')' -set_open : '[' // TODO(mal): Should also accept '(' -set_close : ']' // TODO(mal): Should also accept ')' -leaf : identifier | integer_literal | char_literal | string_literal // TODO(mal): Also symbol_label -identifier : alpha [alphanumeric] -name_identifier : alpha [alphanumeric] +tag : @fill id['(' [@child set] ')'] +untagged_set : @fill label [':' @child iset '\n'] | [@fill label ':'] scoped_set | set [' ' @sibling set] +untagged_iset : @fill label [iset_tail] +iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set +scoped_set : '{' [@child set] '}' | scope_beg [@child set] scope_end +scope_beg : '(' | '[' +scope_end : ')' | ']' +id : alpha [alphanumeric] | '_' [alphanumeric] alphanumeric : alpha [alphanumeric] | digit [alphanumeric] | '_' [alphanumeric] - -integer_literal : { ['-'] natural_literal } -natural_literal : digit [natural_literal] - -char_literal : @markup '\'' [char_literal_items] @markup '\'' -char_literal_items : char_literal_item [char_literal_items] -char_literal_item : ascii_no_backslash_no_quotes | '"' | '\\' ascii -ascii : ascii_no_backslash_no_quotes | '\'' | '"' | '\\' -ascii_no_backslash_no_quotes : digit | alpha | symbol_no_backslash_no_quotes | space -symbol_no_backslash_no_quotes : symbol_no_backslash_no_quotes_1 | symbol_no_backslash_no_quotes_2 -symbol_no_backslash_no_quotes_1 : '!'|'#'|'$'|'%'|'&'|'('|')'|'*'|'+'|','|'-'|'.'|'/'|':'|';' -symbol_no_backslash_no_quotes_2 : '<'|'='|'>'|'?'|'@'|'['|']'|'^'|'_'|'`'|'{'|'|'|'}'|'~' - -string_literal : @markup '"' [string_literal_items] @markup '"' -string_literal_items : string_literal_item [string_literal_items] -string_literal_item : ascii_no_backslash_no_quotes | '\'' | '\\' ascii - -digit : '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9' alpha : lowercase | uppercase lowercase : 'a'|'b'|'c'|'d'|'e'|'f'|'g'|'h'|'i'|'j'|'k'|'l'|'m'|'n'|'o'|'p'|'q'|'r'|'s'|'t'|'u'|'v'|'w'|'z'|'y'|'z' uppercase : 'A'|'B'|'C'|'D'|'E'|'F'|'G'|'H'|'I'|'J'|'K'|'L'|'M'|'N'|'O'|'P'|'Q'|'R'|'S'|'T'|'U'|'V'|'W'|'Z'|'Y'|'Z' -space : ' ' - -symbol_label : '~'|'!'|'%'|'^'|'&'|'*'|'+'|'-'|'/'|'|'|'<'|'>'|'$'|'='|'.'|'?'|'_'|'\\' - -/* - -symbol_no_quote : symbol_label | symbol_colon | symbol_group - -symbol : symbol_label | symbol_colon | symbol_group - -symbol_group : symbol_open | symbol_close -symbol_open : '(' | '[' | '{' -symbol_close : ')' | ']' | '}' - -escaped_char : '\\' '\\' | '\\' '\'' | '\\' '\"' -symbol_colon : ':' - -*/ -*/ - +digit : '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9' +label : id | integer_literal | char_literal | string_literal | symbol_label +integer_literal : { ['-'] natural_literal } +natural_literal : digit [natural_literal] +char_literal : @markup '\'' [char_literal_items] @markup '\'' +char_literal_items : char_literal_item [char_literal_items] +char_literal_item : ascii_no_backslash_no_quotes | '"' | '\\' ascii +ascii : ascii_no_backslash_no_quotes | '\'' | '"' | '`' | '\\' +ascii_no_backslash_no_quotes : digit | alpha | symbol_no_backslash_no_quotes | ' ' +symbol_no_backslash_no_quotes : symbol_no_backslash_no_quotes_1 | symbol_no_backslash_no_quotes_2 +symbol_no_backslash_no_quotes_1 : '!'|'#'|'$'|'%'|'&'|'('|')'|'*'|'+'|','|'-'|'.'|'/'|':' +symbol_no_backslash_no_quotes_2 : ';'|'<'|'='|'>'|'?'|'@'|'['|']'|'^'|'_'|'{'|'|'|'}'|'~' +string_literal : @markup '"' [string_literal_items] @markup '"' | @markup '`' [string_literal_items] @markup '`' +string_literal_items : string_literal_item [string_literal_items] +string_literal_item : ascii_no_backslash_no_quotes | '\'' | '\\' ascii +symbol_label : '~'|'!'|'%'|'^'|'&'|'*'|'+'|'-'|'/'|'|'|'<'|'>'|'$'|'='|'.'|'?'|'$'