[grammar test] Fairly complete test grammars.

tests/grammar.c

@@ -141,22 +141,23 @@ static void PrintRule(MD_Node *rule)
     }
 }
 
-typedef enum AllowedOperationFlags AllowedOperationFlags;
+typedef enum OperationFlags OperationFlags;
-enum AllowedOperationFlags
+enum OperationFlags
 {
-    AllowedOperationFlag_Fill   = 1<<0,
+    OperationFlag_Fill   = 1<<0,
-    AllowedOperationFlag_Tag    = 1<<1,
+    OperationFlag_Markup = 1<<1,
+    OperationFlag_Tag    = 1<<2,
 };
 
-static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *cur_node, 
-                             AllowedOperationFlags allowed, MD_u32 max_depth, MD_u32 depth);
-
 static void Extend(MD_String8 *s, char c)
 {
     *s = MD_PushStringF("%.*s%c", MD_StringExpand(*s), c);
 }
 
-static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_node, AllowedOperationFlags allowed,
+static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *cur_node, 
+                             OperationFlags op_flags, MD_u32 max_depth, MD_u32 depth);
+
+static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_node, OperationFlags op_flags,
                        MD_u32 max_depth, MD_u32 depth)
 {
     for(MD_EachNode(rule_element, rule->first_child))
@@ -178,12 +179,12 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_
         if(expand)
         {
             MD_Node *node_to_tag = 0;
-            AllowedOperationFlags new_flags = 0;
+            OperationFlags old_op_flags = op_flags;
             for(MD_EachNode(tag_node, rule_element->first_tag)){
                 if(MD_StringMatch(tag_node->string, MD_S8Lit("child"), 0))
                 {
                     cur_node = NewChild(cur_node);
-                    allowed &= ~AllowedOperationFlag_Tag; // NOTE(mal): Tag parameters are not tags
+                    op_flags &= ~OperationFlag_Tag; // NOTE(mal): Tag parameters are not tags
                 }
                 else if(MD_StringMatch(tag_node->string, MD_S8Lit("sibling"), 0))
                 {
@@ -191,15 +192,19 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_
                 }
                 else if(MD_StringMatch(tag_node->string, MD_S8Lit("fill"), 0))
                 {
-                    new_flags |= AllowedOperationFlag_Fill;
+                    op_flags |= OperationFlag_Fill;
                 }
                 else if(MD_StringMatch(tag_node->string, MD_S8Lit("tag"), 0))
                 {
-                    new_flags |= AllowedOperationFlag_Tag;
+                    op_flags |= OperationFlag_Tag;
                     node_to_tag = cur_node;
                     cur_node = NewChild(0);
                     cur_node->kind = MD_NodeKind_Tag;
                 }
+                else if(MD_StringMatch(tag_node->string, MD_S8Lit("markup"), 0))
+                {
+                    op_flags |= OperationFlag_Markup;
+                }
                 else if(MD_StringMatch(tag_node->string, MD_S8Lit(OPTIONAL_TAG), 0))
                 {
                 }
@@ -209,12 +214,10 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_
                 }
             }
 
-            allowed |= new_flags;
-
             MD_b32 has_children = !MD_NodeIsNil(rule_element->first_child);
             if(has_children)
             {
-                ExpandRule(rule_element, out_strings, cur_node, allowed, max_depth, depth+1);
+                ExpandRule(rule_element, out_strings, cur_node, op_flags, max_depth, depth+1);
             }
             else
             {
@@ -238,17 +241,20 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_
                     MD_String8 character = MD_PushStringF("%c", c);
                     MD_PushStringToList(out_strings, character);
 
-                    if(allowed & (AllowedOperationFlag_Fill))
+                    if(op_flags & OperationFlag_Fill)
                     {
                         Extend(&cur_node->whole_string, c);
-                        Extend(&cur_node->string, c);
+                        if(!(op_flags & OperationFlag_Markup))
+                        {
+                            Extend(&cur_node->string, c);
+                        }
                     }
                 }
                 else        // NOTE(mal): Non-terminal production
                 {
                     MD_Node * production = MD_NodeTable_Lookup(globals.production_table, rule_element->string)->node;
                     MD_Assert(production);
-                    ExpandProduction(production, out_strings, cur_node, allowed, max_depth, depth+1);
+                    ExpandProduction(production, out_strings, cur_node, op_flags, max_depth, depth+1);
                 }
             }
 
@@ -258,13 +264,13 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_
                 cur_node = node_to_tag;
             }
 
-            allowed &= ~new_flags;
+            op_flags = old_op_flags;
         }
     }
 }
 
 static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *cur_node, 
-                             AllowedOperationFlags allowed, MD_u32 max_depth, MD_u32 depth)
+                             OperationFlags op_flags, MD_u32 max_depth, MD_u32 depth)
 {
     MD_i64 rule_count = MD_ChildCountFromNode(production);
 
@@ -276,7 +282,7 @@ static void ExpandProduction(MD_Node *production, MD_String8List *out, MD_Node *
         rule = MD_ChildFromIndex(production, rule_number);
     }while(GET_DEPTH(rule)+depth > max_depth);
 
-    ExpandRule(rule, out, cur_node, allowed, max_depth, depth);
+    ExpandRule(rule, out, cur_node, op_flags, max_depth, depth);
 }
 
 static MD_Node * FindNonTerminalProduction(MD_Node *node, MD_NodeTable *visited)
@@ -334,28 +340,27 @@ static MD_Node * FindNonTerminalProduction(MD_Node *node, MD_NodeTable *visited)
     return result;
 }
 
-// TODO: use MD_NodeDeepMatch instead
 static MD_b32 EqualTrees(MD_Node *a, MD_Node *b);
 static MD_b32 EqualList(MD_Node *a, MD_Node *b)
 {
     MD_b32 result = 1;
-
     while(!MD_NodeIsNil(a) || !MD_NodeIsNil(b))
     {
         if(!EqualTrees(a, b))
         {
             result = 0;
+            break;
         }
         a = a->next;
         b = b->next;
     }
-
     return result;
 }
 static MD_b32 EqualTrees(MD_Node *a, MD_Node *b)
 {
-    MD_b32 result = (a->kind == b->kind && MD_StringMatch(a->string, b->string, 0) && 
+    MD_b32 result = (a->kind == b->kind && 
-                     MD_StringMatch(a->string, b->string, 0));
+                     MD_StringMatch(a->string, b->string, 0) && 
+                     MD_StringMatch(a->whole_string, b->whole_string, 0));
     result &= EqualList(a->first_tag, b->first_tag);
     result &= EqualList(a->first_child, b->first_child);
     return result;
@@ -435,7 +440,6 @@ static void ComputeElementDepth(MD_Node *re)
     SET_DEPTH(re, result);
 }
 
-
 // NOTE(mal): Compares first by size then alphabetically
 static int StringCompare(MD_String8 a, MD_String8 b)
 {
@@ -537,11 +541,15 @@ int main(int argument_count, char **arguments)
 
     // NOTE(mal): Check that all branches lead to terminal nodes
     MD_NodeTable visited_productions = {0};
-    MD_Node *non_terminal_production = FindNonTerminalProduction(file_production, &visited_productions);
+
-    if(non_terminal_production)
+    for(MD_EachNode(production, productions->first_child))
     {
-        fprintf(stderr, "Error: Non-terminal production \"%.*s\"\n", MD_StringExpand(non_terminal_production->string));
+        MD_Node *non_terminal_production = FindNonTerminalProduction(production, &visited_productions);
-        goto error;
+        if(non_terminal_production)
+        {
+            fprintf(stderr, "Error: Non-terminal production \"%.*s\"\n", MD_StringExpand(non_terminal_production->string));
+            goto error;
+        }
     }
 
     // NOTE(mal): Check that all productions are reachable
@@ -644,13 +652,10 @@ int main(int argument_count, char **arguments)
         test.expected_output = NewChild(0);
         test.expected_output->kind = MD_NodeKind_File;
 
-        // static int loop = 0; ++loop; if(loop == 893) BP; 
-
         MD_String8List string_list = {0};
         // NOTE(mal): Generate a random MD file
         ExpandProduction(file_production_node, &string_list, test.expected_output, 0, max_production_depth, 0);
         test.input = MD_JoinStringList(string_list);
-        // if(MD_StringMatch(test.input, MD_S8Lit("A:A:A A\n"), 0)) BP;
 
         Test *prev = 0;
         for(Test *cur = first_test; cur; cur = cur->next)
@@ -699,7 +704,6 @@ int main(int argument_count, char **arguments)
             MD_OutputTree(stdout, test->expected_output); printf("\n");
             return -1;
         }
-
         ++i_test;
     }
 

tests/grammar.md

@@ -1,60 +1,3 @@
-// Arbitrarily deep tree, possibly empty
-// file            : [@child set]
-// set             : '{' [@child set] '}' | set [' ' @sibling set]
-
-// Labeled leaves
-// file            : [@child set]
-// set             : @fill 'A' | '{' [@child set] '}' | set [' ' @sibling set]
-
-// Labels also on internal nodes    ('iset' is an implicitly separated set)
-// file                            : [@child set]
-// set                             : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] curly_set  | set [' ' @sibling set]
-// iset /* implicit separator */  : @fill 'A' [':' @child iset]      |  @fill 'A'[':'  curly_set] | @fill 'A' [' ' @sibling iset] | @fill 'A' [' ' @sibling curly_set]
-// curly_set                       : '{' [@child set] '}'
-
-// Labels also on internal nodes (v2) ('iset' is an implicitly separated set)
-// file                            : [@child set]
-// set                             : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] curly_set | set [' ' @sibling set]
-// iset /* implicit separator */   : @fill 'A' [iset_tail]
-// iset_tail                       : ':' @child iset | ' ' @sibling iset | ':'  curly_set | ' ' @sibling curly_set
-// curly_set                       : '{' [@child set] '}'
-
-// Tags
-// file          : [@child set]
-// set           : {[tag_list] untagged_set}
-// iset          : {[tag_list] untagged_iset}
-// tag_list      : '@' @tag tag ' ' [tag_list]
-// tag           : 'T'
-// untagged_set  : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] curly_set | set [' ' @sibling set]
-// untagged_iset : @fill 'A' [iset_tail]
-// iset_tail     : ':' @child iset | ' ' @sibling iset | ':'  curly_set | ' ' @sibling curly_set
-// curly_set     : '{' [@child set] '}'
-
-// Tag parameters
-file          : [@child set]
-set           : {[tag_list] untagged_set}
-iset          : {[tag_list] untagged_iset}
-tag_list      : '@' @tag tag ' ' [tag_list]
-tag           : @fill 'T'['(' [@child set] ')']
-untagged_set  : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] curly_set | set [' ' @sibling set]
-untagged_iset : @fill 'A' [iset_tail]
-iset_tail     : ':' @child iset | ' ' @sibling iset | ':'  curly_set | ' ' @sibling curly_set
-curly_set     : '{' [@child set] '}'
-
-
-// top-level and set separators
-// file            : [@child file_set_list]
-// file_set_list   : set [file_set_separator @sibling file_set_list]
-// file_set_separator   : ' ' | '\n'
-// set             : '{' [@child set] '}' | set [separator @sibling set]
-// separator       : ' ' | ','
-
-
-
-/////////////////////////////////////////// REFERENCE ///////////////////////////////////////////////////////////
-
-/*
-
 /* MetaDesk grammar with semantic annotations
  *
  * Each line represents a BNF-esque production:
@@ -66,73 +9,94 @@ curly_set     : '{' [@child set] '}'
  *   and miscellaneous semantics (@fill, @markup)
  */
 
-file                 : [@child file_set_list]
+//// Arbitrarily deep tree, possibly empty
-file_set_list        : { [tag_list] set [file_set_separator @sibling file_set_list] }
+// file            : [@child set]
-file_set_separator   : ' ' | '\n'
+// set             : '{' [@child set] '}' | set [' ' @sibling set]
 
-// TODO(mal): Unify file_set_separator and set_separator by allowing ',' and ';' everywhere ?
+//// Labeled leaves
-set_list        : { [tag_list] set [set_separator @sibling set_list] }
+// file            : [@child set]
-set_separator   : ' ' | '\n'        // TODO(mal):  | ',' | ';'
+// set             : @fill 'A' | '{' [@child set] '}' | set [' ' @sibling set]
+
+//// Labels also on internal nodes ('iset' is an implicitly separated set)
+// file            : [@child set]
+// set             : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set]
+// iset            : @fill 'A' [iset_tail]
+// iset_tail       : ':' @child iset | ' ' @sibling iset | ':'  scoped_set | ' ' @sibling scoped_set
+// scoped_set      : '{' [@child set] '}'
+
+//// Tags
+// file            : [@child set]
+// set             : {[tag_list] untagged_set}
+// iset            : {[tag_list] untagged_iset}
+// tag_list        : '@' @tag tag ' ' [tag_list]
+// tag             : @fill 'T'['(' [@child set] ')']
+// untagged_set    : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set]
+// untagged_iset   : @fill 'A' [iset_tail]
+// iset_tail       : ':' @child iset | ' ' @sibling iset | ':'  scoped_set | ' ' @sibling scoped_set
+// scoped_set      : '{' [@child set] '}'
+
+//// Alternative scope markers
+// file            : [@child set]
+// set             : {[tag_list] untagged_set}
+// iset            : {[tag_list] untagged_iset}
+// tag_list        : '@' @tag tag ' ' [tag_list]
+// tag             : @fill 'T'['(' [@child set] ')']
+// untagged_set    : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set]
+// untagged_iset   : @fill 'A' [iset_tail]
+// iset_tail       : ':' @child iset | ' ' @sibling iset | ':'  scoped_set | ' ' @sibling scoped_set
+// scoped_set      : '{' [@child set] '}' | scope_beg [@child set] scope_end
+// scope_beg       : '(' | '['
+// scope_end       : ')' | ']'
+
+//// Identifiers
+// file            : [@child set]
+// set             : {[tag_list] untagged_set}
+// iset            : {[tag_list] untagged_iset}
+// tag_list        : '@' @tag tag ' ' [tag_list]
+// tag             : @fill id['(' [@child set] ')']
+// untagged_set    : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set]
+// untagged_iset   : @fill 'A' [iset_tail]
+// iset_tail       : ':' @child iset | ' ' @sibling iset | ':'  scoped_set | ' ' @sibling scoped_set
+// scoped_set      : '{' [@child set] '}' | scope_beg [@child set] scope_end
+// scope_beg       : '(' | '['
+// scope_end       : ')' | ']'
+// id              : alpha [alphanumeric]
+// alphanumeric    : alpha [alphanumeric] | digit [alphanumeric] | '_' [alphanumeric]
+// alpha           : lowercase | uppercase
+// lowercase       : 'a'|'b'|'c'|'d'|'e'|'f'|'g'|'h'|'i'|'j'|'k'|'l'|'m'|'n'|'o'|'p'|'q'|'r'|'s'|'t'|'u'|'v'|'w'|'z'|'y'|'z'
+// uppercase       : 'A'|'B'|'C'|'D'|'E'|'F'|'G'|'H'|'I'|'J'|'K'|'L'|'M'|'N'|'O'|'P'|'Q'|'R'|'S'|'T'|'U'|'V'|'W'|'Z'|'Y'|'Z'
+// digit           : '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9'
+
+//// General labels
+file            : [@child set]
+set             : {[tag_list] untagged_set}
+iset            : {[tag_list] untagged_iset}
 tag_list        : '@' @tag tag ' ' [tag_list]
-
+tag             : @fill id['(' [@child set] ')']
-/*
+untagged_set    : @fill label [':' @child iset '\n'] | [@fill label ':'] scoped_set | set [' ' @sibling set]
-    tag             : identifier [@markup '(' [@child set_list] @markup ')']
+untagged_iset   : @fill label [iset_tail]
-    set             : @fill leaf | @fill identifier ':' @child @fill leaf | [@fill identifier ':'] '{' [@child set_list] '}' | [@fill identifier ':'] set_open [@child set_list] set_close
+iset_tail       : ':' @child iset | ' ' @sibling iset | ':'  scoped_set | ' ' @sibling scoped_set
-    set             : @fill leaf | @fill identifier ':' @child space_separated_set_list | [@fill identifier ':'] set_open [@child set_list] set_close
+scoped_set      : '{' [@child set] '}' | scope_beg [@child set] scope_end
-*/
+scope_beg       : '(' | '['
-
+scope_end       : ')' | ']'
-tag             : identifier 
+id              : alpha [alphanumeric] | '_' [alphanumeric]
-set             : @fill leaf [':' @child space_separated_set_list '\n'] | @fill leaf [':' '{' [@child set_list] '}'] | space_separated_set_list '\n' | '{' [@child set_list] '}' 
-
-space_separated_set_list : { [tag_list] set [' ' @sibling space_separated_set_list] }
-
-
-// set_open        : '[' | '('
-// set_close       : ']' | ')'
-set_open        : '['           // TODO(mal): Should also accept '('
-set_close       : ']'           // TODO(mal): Should also accept ')'
-leaf            : identifier | integer_literal | char_literal | string_literal  // TODO(mal): Also symbol_label
-identifier      : alpha [alphanumeric]
-name_identifier : alpha [alphanumeric]
 alphanumeric    : alpha [alphanumeric] | digit [alphanumeric] | '_' [alphanumeric]
-
-integer_literal : { ['-'] natural_literal }
-natural_literal : digit [natural_literal]
-
-char_literal                    : @markup '\'' [char_literal_items] @markup '\''
-char_literal_items              : char_literal_item [char_literal_items]
-char_literal_item               : ascii_no_backslash_no_quotes | '"' | '\\' ascii
-ascii                           : ascii_no_backslash_no_quotes | '\'' | '"' | '\\'
-ascii_no_backslash_no_quotes    : digit | alpha | symbol_no_backslash_no_quotes | space
-symbol_no_backslash_no_quotes   : symbol_no_backslash_no_quotes_1 | symbol_no_backslash_no_quotes_2
-symbol_no_backslash_no_quotes_1 : '!'|'#'|'$'|'%'|'&'|'('|')'|'*'|'+'|','|'-'|'.'|'/'|':'|';'
-symbol_no_backslash_no_quotes_2 : '<'|'='|'>'|'?'|'@'|'['|']'|'^'|'_'|'`'|'{'|'|'|'}'|'~'
-
-string_literal                  : @markup '"' [string_literal_items] @markup '"'
-string_literal_items            : string_literal_item [string_literal_items]
-string_literal_item             : ascii_no_backslash_no_quotes | '\'' | '\\' ascii
-
-digit           : '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9'
 alpha           : lowercase | uppercase
 lowercase       : 'a'|'b'|'c'|'d'|'e'|'f'|'g'|'h'|'i'|'j'|'k'|'l'|'m'|'n'|'o'|'p'|'q'|'r'|'s'|'t'|'u'|'v'|'w'|'z'|'y'|'z'
 uppercase       : 'A'|'B'|'C'|'D'|'E'|'F'|'G'|'H'|'I'|'J'|'K'|'L'|'M'|'N'|'O'|'P'|'Q'|'R'|'S'|'T'|'U'|'V'|'W'|'Z'|'Y'|'Z'
-space           : ' '
+digit           : '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9'
-
+label           : id | integer_literal | char_literal | string_literal | symbol_label
-symbol_label    : '~'|'!'|'%'|'^'|'&'|'*'|'+'|'-'|'/'|'|'|'<'|'>'|'$'|'='|'.'|'?'|'_'|'\\'
+integer_literal : { ['-'] natural_literal }
-
+natural_literal : digit [natural_literal]
-/*
+char_literal                    : @markup '\'' [char_literal_items] @markup '\''
-
+char_literal_items              : char_literal_item [char_literal_items]
-symbol_no_quote : symbol_label | symbol_colon | symbol_group
+char_literal_item               : ascii_no_backslash_no_quotes | '"' | '\\' ascii
-
+ascii                           : ascii_no_backslash_no_quotes | '\'' | '"' | '`' | '\\'
-symbol          : symbol_label | symbol_colon | symbol_group
+ascii_no_backslash_no_quotes    : digit | alpha | symbol_no_backslash_no_quotes | ' '
-
+symbol_no_backslash_no_quotes   : symbol_no_backslash_no_quotes_1 | symbol_no_backslash_no_quotes_2
-symbol_group    : symbol_open | symbol_close
+symbol_no_backslash_no_quotes_1 : '!'|'#'|'$'|'%'|'&'|'('|')'|'*'|'+'|','|'-'|'.'|'/'|':'
-symbol_open     : '(' | '[' | '{'
+symbol_no_backslash_no_quotes_2 : ';'|'<'|'='|'>'|'?'|'@'|'['|']'|'^'|'_'|'{'|'|'|'}'|'~'
-symbol_close    : ')' | ']' | '}' 
+string_literal                  : @markup '"' [string_literal_items] @markup '"' | @markup '`' [string_literal_items] @markup '`'
-
+string_literal_items            : string_literal_item [string_literal_items]
-escaped_char    : '\\' '\\' | '\\' '\'' | '\\' '\"'
+string_literal_item             : ascii_no_backslash_no_quotes | '\'' | '\\' ascii
-symbol_colon    : ':'
+symbol_label                    : '~'|'!'|'%'|'^'|'&'|'*'|'+'|'-'|'/'|'|'|'<'|'>'|'$'|'='|'.'|'?'|'$'
-
-*/
-*/
-