diff --git a/samples/output_parse/examples/output/parsed_example.txt b/samples/output_parse/examples/output/parsed_example.txt index a9aad00..1b911ee 100644 --- a/samples/output_parse/examples/output/parsed_example.txt +++ b/samples/output_parse/examples/output/parsed_example.txt @@ -43,28 +43,28 @@ Node { Tag: @test_tag Node { Kind: Label, - Flags: 00000000000100000000000000000000, - Flag Names: Identifier, + Flags: 00000001000100000000000000000000, + Flag Names: BeforeComma, Identifier, String: y, Whole String: y, Node { Kind: Label, - Flags: 00000001000100000000000000000000, - Flag Names: BeforeComma, Identifier, + Flags: 00000000000100000000000000000000, + Flag Names: Identifier, String: float, Whole String: float, } } Node { Kind: Label, - Flags: 00000000000100000000000000000000, - Flag Names: Identifier, + Flags: 00000001010100000000000000000000, + Flag Names: BeforeComma, AfterComma, Identifier, String: z, Whole String: z, Node { Kind: Label, - Flags: 00000001000100000000000000000000, - Flag Names: BeforeComma, Identifier, + Flags: 00000000000100000000000000000000, + Flag Names: Identifier, String: i32, Whole String: i32, } diff --git a/source/md_impl.c b/source/md_impl.c index 0a300a7..d290c41 100644 --- a/source/md_impl.c +++ b/source/md_impl.c @@ -1683,20 +1683,29 @@ _MD_ParseSet(MD_ParseCtx *ctx, MD_Node *parent, _MD_ParseSetFlags flags, // NOTE(rjf): Separators. { MD_b32 result = 0; - if(MD_Parse_Require(ctx, MD_S8Lit(","), MD_TokenKind_Symbol)) + if(terminate_with_separator) + { + MD_Token next_token = MD_Parse_PeekSkipSome(ctx, 0); + if(next_token.kind == MD_TokenKind_Newline || + (next_token.kind == MD_TokenKind_Symbol && + (MD_StringMatch(next_token.string, MD_S8Lit(","), 0) || + MD_StringMatch(next_token.string, MD_S8Lit(";"), 0)))) + { + result = 1; + } + } + else if(MD_Parse_Require(ctx, MD_S8Lit(","), MD_TokenKind_Symbol)) { - result |= 1; child->flags |= MD_NodeFlag_BeforeComma; next_child_flags |= MD_NodeFlag_AfterComma; } else if(MD_Parse_Require(ctx, MD_S8Lit(";"), MD_TokenKind_Symbol)) { - result |= 1; child->flags |= MD_NodeFlag_BeforeSemicolon; next_child_flags |= MD_NodeFlag_AfterSemicolon; } - result |= (MD_Parse_PeekSkipSome(ctx, 0).kind == MD_TokenKind_Newline); - if(result && terminate_with_separator) + + if(result) { goto end_parse; } diff --git a/tests/grammar.md b/tests/grammar.md index aab017a..39ede76 100644 --- a/tests/grammar.md +++ b/tests/grammar.md @@ -11,56 +11,76 @@ //// Arbitrarily deep tree, possibly empty // file : [@child set] -// set : '{' [@child set] '}' | set [' ' @sibling set] +// set : '{' [@child set] '}' | set [separator @sibling set] +// separator : ' ' | '\n' | ',' | ';' //// Labeled leaves // file : [@child set] -// set : @fill 'A' | '{' [@child set] '}' | set [' ' @sibling set] +// set : @fill 'A' | '{' [@child set] '}' | set [separator @sibling set] +// separator : ' ' | '\n' | ',' | ';' -//// Labels also on internal nodes ('iset' is an implicitly separated set) +//// Labeled internal nodes // file : [@child set] -// set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set] -// iset : @fill 'A' [iset_tail] -// iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set +// set : @fill 'A' | [@fill 'A' ':'] scoped_set | set [separator @sibling set] +// separator : ' ' | '\n' | ',' | ';' // scoped_set : '{' [@child set] '}' +//// Unscoped tests (feels like there should be an easier way to express this) +// file : general_set +// general_set : { [@child set] | [@child uns_set] } +// set : @fill 'A' | set [sep1 @sibling set] | scoped_set | uns_set sep2 @sibling set +// uns_set : @fill 'A' [uns_set_tail] +// uns_set_tail : ':' @child uns_set | ' ' @sibling uns_set | ':' scoped_set | ' ' @sibling scoped_set +// scoped_set : '{' general_set '}' +// sep1 : ' ' | '\n' | ',' | ';' +// sep2 : '\n'| ',' | ';' + //// Tags -// file : [@child set] +// file : general_set +// general_set : { [@child set] | [@child uns_set] } // set : {[tag_list] untagged_set} -// iset : {[tag_list] untagged_iset} +// uns_set : {[tag_list] untagged_uns_set} // tag_list : '@' @tag tag ' ' [tag_list] -// tag : @fill 'T'['(' [@child set] ')'] -// untagged_set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set] -// untagged_iset : @fill 'A' [iset_tail] -// iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set -// scoped_set : '{' [@child set] '}' +// tag : @fill 'T'['(' general_set ')'] +// untagged_set : @fill 'A' | set [sep1 @sibling set] | scoped_set | uns_set sep2 @sibling set +// untagged_uns_set: @fill 'A' [uns_set_tail] +// uns_set_tail : ':' @child uns_set | ' ' @sibling uns_set | ':' scoped_set | ' ' @sibling scoped_set +// scoped_set : '{' general_set '}' +// sep1 : ' ' | '\n' | ',' | ';' +// sep2 : '\n'| ',' | ';' //// Alternative scope markers -// file : [@child set] +// file : general_set +// general_set : { [@child set] | [@child uns_set] } // set : {[tag_list] untagged_set} -// iset : {[tag_list] untagged_iset} +// uns_set : {[tag_list] untagged_uns_set} // tag_list : '@' @tag tag ' ' [tag_list] -// tag : @fill 'T'['(' [@child set] ')'] -// untagged_set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set] -// untagged_iset : @fill 'A' [iset_tail] -// iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set -// scoped_set : '{' [@child set] '}' | scope_beg [@child set] scope_end -// scope_beg : '(' | '[' -// scope_end : ')' | ']' +// tag : @fill 'T'['(' general_set ')'] +// untagged_set : @fill 'A' | set [sep1 @sibling set] | scoped_set | uns_set sep2 @sibling set +// untagged_uns_set: @fill 'A' [uns_set_tail] +// uns_set_tail : ':' @child uns_set | ' ' @sibling uns_set | ':' scoped_set | ' ' @sibling scoped_set +// scoped_set : '{' general_set '}' | alt_scope_beg general_set alt_scope_end +// alt_scope_beg : '(' | '[' +// alt_scope_end : ')' | ']' +// sep1 : ' ' | '\n' | ',' | ';' +// sep2 : '\n'| ',' | ';' //// Identifiers -// file : [@child set] +// file : general_set +// general_set : { [@child set] | [@child uns_set] } // set : {[tag_list] untagged_set} -// iset : {[tag_list] untagged_iset} +// uns_set : {[tag_list] untagged_uns_set} // tag_list : '@' @tag tag ' ' [tag_list] -// tag : @fill id['(' [@child set] ')'] -// untagged_set : @fill 'A' [':' @child iset '\n'] | [@fill 'A' ':'] scoped_set | set [' ' @sibling set] -// untagged_iset : @fill 'A' [iset_tail] -// iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set -// scoped_set : '{' [@child set] '}' | scope_beg [@child set] scope_end -// scope_beg : '(' | '[' -// scope_end : ')' | ']' -// id : alpha [alphanumeric] +// tag : @fill id['(' general_set ')'] +// untagged_set : @fill 'A' | set [sep1 @sibling set] | scoped_set | uns_set sep2 @sibling set +// untagged_uns_set: @fill 'A' [uns_set_tail] +// uns_set_tail : ':' @child uns_set | ' ' @sibling uns_set | ':' scoped_set | ' ' @sibling scoped_set +// scoped_set : '{' general_set '}' | alt_scope_beg general_set alt_scope_end +// alt_scope_beg : '(' | '[' +// alt_scope_end : ')' | ']' +// sep1 : ' ' | '\n' | ',' | ';' +// sep2 : '\n'| ',' | ';' +// id : alpha [alphanumeric] | '_' [alphanumeric] // alphanumeric : alpha [alphanumeric] | digit [alphanumeric] | '_' [alphanumeric] // alpha : lowercase | uppercase // lowercase : 'a'|'b'|'c'|'d'|'e'|'f'|'g'|'h'|'i'|'j'|'k'|'l'|'m'|'n'|'o'|'p'|'q'|'r'|'s'|'t'|'u'|'v'|'w'|'z'|'y'|'z' @@ -68,17 +88,20 @@ // digit : '0'|'1'|'2'|'3'|'4'|'5'|'6'|'7'|'8'|'9' //// General labels -file : [@child set] +file : general_set +general_set : { [@child set] | [@child uns_set] } set : {[tag_list] untagged_set} -iset : {[tag_list] untagged_iset} +uns_set : {[tag_list] untagged_uns_set} tag_list : '@' @tag tag ' ' [tag_list] -tag : @fill id['(' [@child set] ')'] -untagged_set : @fill label [':' @child iset '\n'] | [@fill label ':'] scoped_set | set [' ' @sibling set] -untagged_iset : @fill label [iset_tail] -iset_tail : ':' @child iset | ' ' @sibling iset | ':' scoped_set | ' ' @sibling scoped_set -scoped_set : '{' [@child set] '}' | scope_beg [@child set] scope_end -scope_beg : '(' | '[' -scope_end : ')' | ']' +tag : @fill id['(' general_set ')'] +untagged_set : @fill label | set [sep1 @sibling set] | scoped_set | uns_set sep2 @sibling set +untagged_uns_set: @fill label [uns_set_tail] +uns_set_tail : ':' @child uns_set | ' ' @sibling uns_set | ':' scoped_set | ' ' @sibling scoped_set +scoped_set : '{' general_set '}' | alt_scope_beg general_set alt_scope_end +alt_scope_beg : '(' | '[' +alt_scope_end : ')' | ']' +sep1 : ' ' | '\n' | ',' | ';' +sep2 : '\n'| ',' | ';' id : alpha [alphanumeric] | '_' [alphanumeric] alphanumeric : alpha [alphanumeric] | digit [alphanumeric] | '_' [alphanumeric] alpha : lowercase | uppercase