ed/metadesk
1
0
Fork 0
mirror of https://github.com/Ed94/metadesk.git synced 2026-08-07 16:18:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge

Browse Source
This commit is contained in:
Allen Webster
2021-06-27 10:21:41 -04:00
parent aba3c9a164 f2991a89bb
commit 0724d0a910
9 changed files with 304 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files
samples/output_parse/examples/output/parsed_example.txt
+19 -19
View File
@@ -1,32 +1,32 @@
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001100000100000000000000000000, Flags: 0000110000000000000000010000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, Identifier, Flag Names: BraceLeft, BraceRight,
String: foo, String: foo,
Whole String: foo, Whole String: foo,
Tag: @struct Tag: @struct
Node { Node {
Kind: Label, Kind: Label,
Flags: 11000001000100000000000000000000, Flags: 1100000010000000000000010000000000000000000000000000000000000000,
Flag Names: ParenLeft, ParenRight, BeforeComma, Identifier, Flag Names: ParenLeft, ParenRight, BeforeComma,
String: x, String: x,
Whole String: x, Whole String: x,
Node { Node {
Kind: Label, Kind: Label,
Flags: 00110000000000000000000000000000, Flags: 0011000000000000000000000000000000000000000000000000000000000000,
Flag Names: BracketLeft, BracketRight, Flag Names: BracketLeft, BracketRight,
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000000100000000000000000000, Flags: 0000000000000000000000010000000000000000000000000000000000000000,
Flag Names: Identifier, Flag Names: ,
String: MAX_PATH, String: MAX_PATH,
Whole String: MAX_PATH, Whole String: MAX_PATH,
} }
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000000100000000000000000000, Flags: 0000000000000000000000010000000000000000000000000000000000000000,
Flag Names: Identifier, Flag Names: ,
String: char, String: char,
Whole String: char, Whole String: char,
} }
@@ -35,36 +35,36 @@ Node {
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001100000100000000000000000000, Flags: 0000110000000000000000010000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, Identifier, Flag Names: BraceLeft, BraceRight,
String: bar, String: bar,
Whole String: bar, Whole String: bar,
Tag: @struct Tag: @struct
Tag: @test_tag Tag: @test_tag
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001000100000000000000000000, Flags: 0000000010000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, Identifier, Flag Names: BeforeComma,
String: y, String: y,
Whole String: y, Whole String: y,
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000000100000000000000000000, Flags: 0000000000000000000000010000000000000000000000000000000000000000,
Flag Names: Identifier, Flag Names: ,
String: float, String: float,
Whole String: float, Whole String: float,
} }
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001010100000000000000000000, Flags: 0000000011000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, AfterComma, Identifier, Flag Names: BeforeComma, AfterComma,
String: z, String: z,
Whole String: z, Whole String: z,
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000000100000000000000000000, Flags: 0000000000000000000000010000000000000000000000000000000000000000,
Flag Names: Identifier, Flag Names: ,
String: i32, String: i32,
Whole String: i32, Whole String: i32,
} }
samples/output_parse/examples/output/parsed_example2.txt
+71 -71
View File
@@ -1,138 +1,138 @@
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001100000100000000000000000000, Flags: 0000110000000000000000010000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, Identifier, Flag Names: BraceLeft, BraceRight,
String: a_set, String: a_set,
Whole String: a_set, Whole String: a_set,
Tag: @sets Tag: @sets
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001101000100000000000000000000, Flags: 0000110010000000000000010000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, BeforeComma, Identifier, Flag Names: BraceLeft, BraceRight, BeforeComma,
String: named_set, String: named_set,
Whole String: named_set, Whole String: named_set,
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001001000000000000000000000, Flags: 0000000010000000000000100000000000000000000000000000000000000000,
Flag Names: BeforeComma, Numeric, Flag Names: BeforeComma,
String: 1, String: 1,
Whole String: 1, Whole String: 1,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001011000000000000000000000, Flags: 0000000011000000000000100000000000000000000000000000000000000000,
Flag Names: BeforeComma, AfterComma, Numeric, Flag Names: BeforeComma, AfterComma,
String: 3, String: 3,
Whole String: 3, Whole String: 3,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000011000000000000000000000, Flags: 0000000001000000000000100000000000000000000000000000000000000000,
Flag Names: AfterComma, Numeric, Flag Names: AfterComma,
String: 7, String: 7,
Whole String: 7, Whole String: 7,
} }
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001101010000000000000000000000, Flags: 0000110011000000000000000000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, BeforeComma, AfterComma, Flag Names: BraceLeft, BraceRight, BeforeComma, AfterComma,
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001000100000000000000000000, Flags: 0000000010000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, Identifier, Flag Names: BeforeComma,
String: not, String: not,
Whole String: not, Whole String: not,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001010100000000000000000000, Flags: 0000000011000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, AfterComma, Identifier, Flag Names: BeforeComma, AfterComma,
String: named, String: named,
Whole String: named, Whole String: named,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000010100000000000000000000, Flags: 0000000001000000000000010000000000000000000000000000000000000000,
Flag Names: AfterComma, Identifier, Flag Names: AfterComma,
String: set, String: set,
Whole String: set, Whole String: set,
} }
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001101010000000000000000000000, Flags: 0000110011000000000000000000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, BeforeComma, AfterComma, Flag Names: BraceLeft, BraceRight, BeforeComma, AfterComma,
Tag: @tag_with_params Tag: @tag_with_params
Tag Children{ Tag Children{
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001000100000000000000000000, Flags: 0000000010000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, Identifier, Flag Names: BeforeComma,
String: p1, String: p1,
Whole String: p1, Whole String: p1,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001010100000000000000000000, Flags: 0000000011000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, AfterComma, Identifier, Flag Names: BeforeComma, AfterComma,
String: p2, String: p2,
Whole String: p2, Whole String: p2,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000010100000000000000000000, Flags: 0000000001000000000000010000000000000000000000000000000000000000,
Flag Names: AfterComma, Identifier, Flag Names: AfterComma,
String: p3, String: p3,
Whole String: p3, Whole String: p3,
} }
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001000100000000000000000000, Flags: 0000000010000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, Identifier, Flag Names: BeforeComma,
String: e1, String: e1,
Whole String: e1, Whole String: e1,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001010100000000000000000000, Flags: 0000000011000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, AfterComma, Identifier, Flag Names: BeforeComma, AfterComma,
String: e2, String: e2,
Whole String: e2, Whole String: e2,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000010100000000000000000000, Flags: 0000000001000000000000010000000000000000000000000000000000000000,
Flag Names: AfterComma, Identifier, Flag Names: AfterComma,
String: e3, String: e3,
Whole String: e3, Whole String: e3,
} }
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001101010000000000000000000000, Flags: 0000110011000000000000000000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, BeforeComma, AfterComma, Flag Names: BraceLeft, BraceRight, BeforeComma, AfterComma,
Tag: @tag_with_params Tag: @tag_with_params
Tag Children{ Tag Children{
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001000100000000000000000000, Flags: 0000000010000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, Identifier, Flag Names: BeforeComma,
String: p1, String: p1,
Whole String: p1, Whole String: p1,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001010100000000000000000000, Flags: 0000000011000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, AfterComma, Identifier, Flag Names: BeforeComma, AfterComma,
String: p2, String: p2,
Whole String: p2, Whole String: p2,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000010100000000000000000000, Flags: 0000000001000000000000010000000000000000000000000000000000000000,
Flag Names: AfterComma, Identifier, Flag Names: AfterComma,
String: p3, String: p3,
Whole String: p3, Whole String: p3,
} }
@@ -141,44 +141,44 @@ Node {
Tag Children{ Tag Children{
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001000100000000000000000000, Flags: 0000000010000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, Identifier, Flag Names: BeforeComma,
String: p, String: p,
Whole String: p, Whole String: p,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001010100000000000000000000, Flags: 0000000011000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, AfterComma, Identifier, Flag Names: BeforeComma, AfterComma,
String: pp, String: pp,
Whole String: pp, Whole String: pp,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000010100000000000000000000, Flags: 0000000001000000000000010000000000000000000000000000000000000000,
Flag Names: AfterComma, Identifier, Flag Names: AfterComma,
String: ppp, String: ppp,
Whole String: ppp, Whole String: ppp,
} }
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001000100000000000000000000, Flags: 0000000010000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, Identifier, Flag Names: BeforeComma,
String: e1, String: e1,
Whole String: e1, Whole String: e1,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000001010100000000000000000000, Flags: 0000000011000000000000010000000000000000000000000000000000000000,
Flag Names: BeforeComma, AfterComma, Identifier, Flag Names: BeforeComma, AfterComma,
String: e2, String: e2,
Whole String: e2, Whole String: e2,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000010100000000000000000000, Flags: 0000000001000000000000010000000000000000000000000000000000000000,
Flag Names: AfterComma, Identifier, Flag Names: AfterComma,
String: e3, String: e3,
Whole String: e3, Whole String: e3,
} }
@@ -187,8 +187,8 @@ Node {
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001100000100000000000000000000, Flags: 0000110000000000000000010000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, Identifier, Flag Names: BraceLeft, BraceRight,
String: empty, String: empty,
Whole String: empty, Whole String: empty,
Tag: @empty_set Tag: @empty_set
@@ -196,19 +196,19 @@ Node {
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001100000000000000000000000000, Flags: 0000110000000000000000000000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, Flag Names: BraceLeft, BraceRight,
Tag: @tagged_unnamed_set Tag: @tagged_unnamed_set
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001100000100000000000000000000, Flags: 0000110000000000000000010000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, Identifier, Flag Names: BraceLeft, BraceRight,
String: a_label, String: a_label,
Whole String: a_label, Whole String: a_label,
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000000100000000000000000000, Flags: 0000000000000000000000010000000000000000000000000000000000000000,
Flag Names: Identifier, Flag Names: ,
String: unnamed_set_element, String: unnamed_set_element,
Whole String: unnamed_set_element, Whole String: unnamed_set_element,
} }
@@ -217,63 +217,63 @@ Node {
Node { Node {
Kind: Label, Kind: Label,
Flags: 00001100000000000000000000000000, Flags: 0000110000000000000000000000000000000000000000000000000000000000,
Flag Names: BraceLeft, BraceRight, Flag Names: BraceLeft, BraceRight,
Tag: @showcase_mixed_set_scoping Tag: @showcase_mixed_set_scoping
Node { Node {
Kind: Label, Kind: Label,
Flags: 10010001000000000000000000000000, Flags: 1001000010000000000000000000000000000000000000000000000000000000,
Flag Names: ParenLeft, BracketRight, BeforeComma, Flag Names: ParenLeft, BracketRight, BeforeComma,
String: +, String: +,
Whole String: +, Whole String: +,
Tag: @symbol_label Tag: @symbol_label
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000001000000000000000000000, Flags: 0000000000000000000000100000000000000000000000000000000000000000,
Flag Names: Numeric, Flag Names: ,
String: 1, String: 1,
Whole String: 1, Whole String: 1,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000001000000000000000000000, Flags: 0000000000000000000000100000000000000000000000000000000000000000,
Flag Names: Numeric, Flag Names: ,
String: 2, String: 2,
Whole String: 2, Whole String: 2,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000001000000000000000000000, Flags: 0000000000000000000000100000000000000000000000000000000000000000,
Flag Names: Numeric, Flag Names: ,
String: 3, String: 3,
Whole String: 3, Whole String: 3,
} }
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 01100001010000000000000000000000, Flags: 0110000011000000000000000000000000000000000000000000000000000000,
Flag Names: ParenRight, BracketLeft, BeforeComma, AfterComma, Flag Names: ParenRight, BracketLeft, BeforeComma, AfterComma,
String: *, String: *,
Whole String: *, Whole String: *,
Tag: @symbol_label Tag: @symbol_label
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000000100000000000000000000, Flags: 0000000000000000000000010000000000000000000000000000000000000000,
Flag Names: Identifier, Flag Names: ,
String: x, String: x,
Whole String: x, Whole String: x,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000000100000000000000000000, Flags: 0000000000000000000000010000000000000000000000000000000000000000,
Flag Names: Identifier, Flag Names: ,
String: y, String: y,
Whole String: y, Whole String: y,
} }
Node { Node {
Kind: Label, Kind: Label,
Flags: 00000000000100000000000000000000, Flags: 0000000000000000000000010000000000000000000000000000000000000000,
Flag Names: Identifier, Flag Names: ,
String: z, String: z,
Whole String: z, Whole String: z,
} }
samples/output_parse/output_parse.c
+1 -1
View File
@@ -71,7 +71,7 @@ int main(int argument_count, char **arguments)
for(MD_EachNode(root, first)) for(MD_EachNode(root, first))
{ {
MD_String8 code_filename = MD_ChopExtension(MD_SkipFolder(root->filename)); MD_String8 code_filename = MD_ChopExtension(MD_SkipFolder(root->string));
MD_String8 info_filename = MD_PushStringF("parsed_%.*s.txt", MD_StringExpand(code_filename)); MD_String8 info_filename = MD_PushStringF("parsed_%.*s.txt", MD_StringExpand(code_filename));
printf("Parse Input -> Output: %.*s -> %.*s\n", MD_StringExpand(code_filename), MD_StringExpand(info_filename)); printf("Parse Input -> Output: %.*s -> %.*s\n", MD_StringExpand(code_filename), MD_StringExpand(info_filename));
samples/static_site_generator/static_site_generator.c
+6 -5
View File
@@ -127,7 +127,7 @@ int main(int argument_count, char **arguments)
{ {
PageInfo page_info = ParsePageInfo(root); PageInfo page_info = ParsePageInfo(root);
MD_String8 name_without_extension = MD_SkipFolder(MD_ChopExtension(root->filename)); MD_String8 name_without_extension = MD_SkipFolder(MD_ChopExtension(root->string));
FILE *file = fopen(MD_PushStringF("%.*s.html", MD_StringExpand(name_without_extension)).str, "wb"); FILE *file = fopen(MD_PushStringF("%.*s.html", MD_StringExpand(name_without_extension)).str, "wb");
if(file) if(file)
{ {
@@ -379,8 +379,7 @@ GeneratePageContent(MD_Map *index_table, SiteInfo *site_info, PageInfo *page_inf
//~ NOTE(rjf): Text blobs //~ NOTE(rjf): Text blobs
if(MD_NodeIsNil(node->first_child) && if(MD_NodeIsNil(node->first_child) &&
(node->flags & MD_NodeFlag_StringLiteral || (node->flags & MD_NodeFlag_StringLiteral))
node->flags & MD_NodeFlag_CharLiteral))
{ {
char *html_tag = "p"; char *html_tag = "p";
char *style = "paragraph"; char *style = "paragraph";
@@ -413,7 +412,8 @@ GeneratePageContent(MD_Map *index_table, SiteInfo *site_info, PageInfo *page_inf
{ {
if(strnode->string.str[i] == '@') if(strnode->string.str[i] == '@')
{ {
MD_ParseResult parse = MD_ParseOneNode(node->filename, MD_StringSubstring(strnode->string, i, strnode->string.size)); MD_Node *root = MD_RootFromNode(node);
MD_ParseResult parse = MD_ParseOneNode(root->string, MD_StringSubstring(strnode->string, i, strnode->string.size));
if(!MD_NodeIsNil(parse.node)) if(!MD_NodeIsNil(parse.node))
{ {
if(MD_NodeHasTag(node, MD_S8Lit("i"))) if(MD_NodeHasTag(node, MD_S8Lit("i")))
@@ -521,9 +521,10 @@ GeneratePageContent(MD_Map *index_table, SiteInfo *site_info, PageInfo *page_inf
if(slot->val) if(slot->val)
{ {
MD_Node *node = slot->val; MD_Node *node = slot->val;
MD_Node *root = MD_RootFromNode(node);
PageInfo info = ParsePageInfo(node); PageInfo info = ParsePageInfo(node);
MD_String8 filename = node->filename; MD_String8 filename = root->string;
MD_String8 filename_no_ext = MD_ChopExtension(MD_SkipFolder(filename)); MD_String8 filename_no_ext = MD_ChopExtension(MD_SkipFolder(filename));
MD_String8 link = MD_PushStringF("%.*s.html", MD_StringExpand(filename_no_ext)); MD_String8 link = MD_PushStringF("%.*s.html", MD_StringExpand(filename_no_ext));
MD_String8 name = info.title->string; MD_String8 name = info.title->string;
source/md.h
+42 -49
View File
@@ -13,13 +13,6 @@
// [ ] Helpers for parsing NodeFlags, figuring out which nodes in a set are // [ ] Helpers for parsing NodeFlags, figuring out which nodes in a set are
// separated by a semicolon, something like MD_SeekNodeWithFlags(node) -> node ? // separated by a semicolon, something like MD_SeekNodeWithFlags(node) -> node ?
// [ ] Escaping characters from strings // [ ] Escaping characters from strings
// [x] Get rid of MD_JoinStringListWithSeparator, just have a separator argument on
// MD_JoinStringList.
// [x] MD_StringMap_Next, for iterating matching slots in an MD_Map, that all
// share the same key (important in the case of hash collisions)
// [x] Helper for making a reference for a node, e.g. MD_ReferenceFromNode
// [x] Organization decision for C generator helpers: splitting from md.h? file name? folder?
// [ ] Collapse down map types
// [ ] Fill in more String -> Integer helpers // [ ] Fill in more String -> Integer helpers
// [ ] Memory Management Strategy // [ ] Memory Management Strategy
// [ ] Gather map of current memory management situation // [ ] Gather map of current memory management situation
@@ -299,6 +292,8 @@ MD_WordStyle;
typedef enum MD_NodeKind typedef enum MD_NodeKind
{ {
// NOTE(rjf): Must be kept in sync with MD_StringFromNodeKind.
MD_NodeKind_Nil, MD_NodeKind_Nil,
MD_NodeKind_File, MD_NodeKind_File,
MD_NodeKind_List, MD_NodeKind_List,
@@ -309,13 +304,21 @@ typedef enum MD_NodeKind
MD_NodeKind_Label, MD_NodeKind_Label,
MD_NodeKind_Tag, MD_NodeKind_Tag,
MD_NodeKind_ErrorMarker, MD_NodeKind_ErrorMarker,
MD_NodeKind_MAX, MD_NodeKind_COUNT,
} }
MD_NodeKind; MD_NodeKind;
typedef MD_u32 MD_NodeFlags; typedef MD_u64 MD_NodeFlags;
#define MD_NodeFlag_AfterFromBefore(f) ((f) << 1)
enum enum
{ {
// NOTE(rjf): Must be kept in sync with MD_StringListFromNodeFlags.
// NOTE(rjf): Because of MD_NodeFlag_AfterFromBefore, it is *required* that
// every single pair of "Before*" or "After*" flags be in the correct order
// which is that the Before* flag comes first, and the After* flag comes
// immediately after (After* being the more significant bit).
MD_NodeFlag_ParenLeft = (1<<0), MD_NodeFlag_ParenLeft = (1<<0),
MD_NodeFlag_ParenRight = (1<<1), MD_NodeFlag_ParenRight = (1<<1),
MD_NodeFlag_BracketLeft = (1<<2), MD_NodeFlag_BracketLeft = (1<<2),
@@ -324,18 +327,22 @@ enum
MD_NodeFlag_BraceRight = (1<<5), MD_NodeFlag_BraceRight = (1<<5),
MD_NodeFlag_BeforeSemicolon = (1<<6), MD_NodeFlag_BeforeSemicolon = (1<<6),
MD_NodeFlag_BeforeComma = (1<<7), MD_NodeFlag_AfterSemicolon = (1<<7),
MD_NodeFlag_AfterSemicolon = (1<<8), MD_NodeFlag_BeforeComma = (1<<8),
MD_NodeFlag_AfterComma = (1<<9), MD_NodeFlag_AfterComma = (1<<9),
MD_NodeFlag_Numeric = (1<<10), MD_NodeFlag_StringSingleQuote = (1<<10),
MD_NodeFlag_Identifier = (1<<11), MD_NodeFlag_StringDoubleQuote = (1<<13),
MD_NodeFlag_StringLiteral = (1<<12), MD_NodeFlag_StringTick = (1<<15),
MD_NodeFlag_CharLiteral = (1<<13), MD_NodeFlag_StringTripletSingleQuote= (1<<16),
}; MD_NodeFlag_StringTripletDoubleQuote= (1<<18),
MD_NodeFlag_StringTripletTick = (1<<20),
#define MD_NodeFlag_AfterFromBefore(f) ((f) << 2) MD_NodeFlag_Numeric = (1<<22),
MD_NodeFlag_Identifier = (1<<23),
MD_NodeFlag_StringLiteral = (1<<24),
};
typedef struct MD_Node MD_Node; typedef struct MD_Node MD_Node;
struct MD_Node struct MD_Node
@@ -363,8 +370,6 @@ struct MD_Node
MD_String8 comment_after; MD_String8 comment_after;
// Source code location information. // Source code location information.
MD_String8 filename;
MD_u8 *file_contents;
MD_u8 *at; MD_u8 *at;
// Reference. // Reference.
@@ -420,34 +425,15 @@ typedef enum MD_TokenKind
MD_TokenKind_Nil, MD_TokenKind_Nil,
MD_TokenKind_RegularMin, MD_TokenKind_RegularMin,
// A group of characters that begins with an underscore or alphabetic character,
// and consists of numbers, alphabetic characters, or underscores after that.
MD_TokenKind_Identifier, MD_TokenKind_Identifier,
// A group of characters beginning with a numeric character or a '-', and then
// consisting of only numbers, alphabetic characters, or '.'s after that.
MD_TokenKind_NumericLiteral, MD_TokenKind_NumericLiteral,
MD_TokenKind_StringLiteralSingleQuote,
// A group of arbitrary characters, grouped together by a " character, OR by a MD_TokenKind_StringLiteralSingleQuoteTriplet,
// """ symbol at the beginning and end of the group. String literals beginning with MD_TokenKind_StringLiteralDoubleQuote,
// " are to only be specified on a single line, but """ strings can exist across MD_TokenKind_StringLiteralDoubleQuoteTriplet,
// many lines. MD_TokenKind_StringLiteralTick,
MD_TokenKind_StringLiteral, MD_TokenKind_StringLiteralTickTriplet,
// A group of arbitrary characters, grouped together by a ' character at the beginning,
// and a ' character at the end.
MD_TokenKind_CharLiteral,
// A group of symbolic characters, where symbolic characters means any of the following:
// ~!@#$%^&*()-+=[{]}:;<>,./?|\
//
// Groups of multiple characters are only allowed in specific circumstances. Most of these
// are only 1 character long, but some groups are allowed:
//
// "<<", ">>", "<=", ">=", "+=", "-=", "*=", "/=", "::", ":=", "==", "&=", "|=", "->"
MD_TokenKind_Symbol, MD_TokenKind_Symbol,
MD_TokenKind_RegularMax, MD_TokenKind_RegularMax,
MD_TokenKind_Comment, MD_TokenKind_Comment,
@@ -457,10 +443,10 @@ typedef enum MD_TokenKind
MD_TokenKind_Newline, MD_TokenKind_Newline,
MD_TokenKind_WhitespaceMax, MD_TokenKind_WhitespaceMax,
MD_TokenKind_BadCharacter,
// Character outside currently supported encodings // Character outside currently supported encodings
MD_TokenKind_BadCharacter,
MD_TokenKind_MAX, MD_TokenKind_COUNT,
} }
MD_TokenKind; MD_TokenKind;
@@ -625,6 +611,11 @@ MD_FUNCTION void MD_MemoryCopy(void *dst, void *src, MD_u64 size);
MD_FUNCTION void* MD_AllocZero(MD_u64 size); MD_FUNCTION void* MD_AllocZero(MD_u64 size);
#define MD_PushArray(T,c) (T*)MD_AllocZero(sizeof(T)*(c)) #define MD_PushArray(T,c) (T*)MD_AllocZero(sizeof(T)*(c))
// NOTE(rjf): Right now, both calls just automatically zero their memory,
// but I'm explicitly splitting this out to ensure that we don't accidentally
// assume that we have zeroed memory incorrectly in the future (when our
// allocation approach changes).
#define MD_PushArrayZero(T,c) (T*)MD_AllocZero(sizeof(T)*(c))
//~ Characters //~ Characters
MD_FUNCTION MD_b32 MD_CharIsAlpha(MD_u8 c); MD_FUNCTION MD_b32 MD_CharIsAlpha(MD_u8 c);
@@ -710,6 +701,8 @@ MD_FUNCTION MD_MapSlot* MD_MapOverwrite(MD_Map *map, MD_MapKey key, void *val);
//~ Parsing //~ Parsing
MD_FUNCTION MD_NodeFlags MD_NodeFlagsFromTokenKind(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsWhitespace(MD_TokenKind kind); MD_FUNCTION MD_b32 MD_TokenKindIsWhitespace(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsComment(MD_TokenKind kind); MD_FUNCTION MD_b32 MD_TokenKindIsComment(MD_TokenKind kind);
MD_FUNCTION MD_b32 MD_TokenKindIsRegular(MD_TokenKind kind); MD_FUNCTION MD_b32 MD_TokenKindIsRegular(MD_TokenKind kind);
@@ -737,15 +730,14 @@ MD_FUNCTION MD_ParseResult MD_ParseWholeString(MD_String8 filename, MD_String8 c
MD_FUNCTION MD_ParseResult MD_ParseWholeFile(MD_String8 filename); MD_FUNCTION MD_ParseResult MD_ParseWholeFile(MD_String8 filename);
//~ Location Conversion //~ Location Conversion
MD_FUNCTION MD_CodeLoc MD_CodeLocFromFileOffset(MD_String8 filename, MD_u8 *base, MD_u8 *off); MD_FUNCTION MD_CodeLoc MD_CodeLocFromFileBaseOff(MD_String8 filename, MD_u8 *base, MD_u8 *off);
MD_FUNCTION MD_CodeLoc MD_CodeLocFromNode(MD_Node *node); MD_FUNCTION MD_CodeLoc MD_CodeLocFromNode(MD_Node *node);
//~ Tree/List Building //~ Tree/List Building
MD_FUNCTION MD_b32 MD_NodeIsNil(MD_Node *node); MD_FUNCTION MD_b32 MD_NodeIsNil(MD_Node *node);
MD_FUNCTION MD_Node *MD_NilNode(void); MD_FUNCTION MD_Node *MD_NilNode(void);
MD_FUNCTION MD_Node *MD_MakeNode(MD_NodeKind kind, MD_String8 string, MD_FUNCTION MD_Node *MD_MakeNode(MD_NodeKind kind, MD_String8 string,
MD_String8 whole_string, MD_String8 filename, MD_String8 whole_string, MD_u8 *at);
MD_u8 *file_contents, MD_u8 *at);
MD_FUNCTION void MD_PushChild(MD_Node *parent, MD_Node *new_child); MD_FUNCTION void MD_PushChild(MD_Node *parent, MD_Node *new_child);
MD_FUNCTION void MD_PushTag(MD_Node *node, MD_Node *tag); MD_FUNCTION void MD_PushTag(MD_Node *node, MD_Node *tag);
MD_FUNCTION MD_Node *MD_PushReference(MD_Node *list, MD_Node *target); MD_FUNCTION MD_Node *MD_PushReference(MD_Node *list, MD_Node *target);
@@ -757,6 +749,7 @@ MD_FUNCTION void MD_PushSibling(MD_Node **first, MD_Node **last, MD_Node *ne
MD_FUNCTION MD_Node * MD_NodeFromString(MD_Node *first, MD_Node *last, MD_String8 string); MD_FUNCTION MD_Node * MD_NodeFromString(MD_Node *first, MD_Node *last, MD_String8 string);
MD_FUNCTION MD_Node * MD_NodeFromIndex(MD_Node *first, MD_Node *last, int n); MD_FUNCTION MD_Node * MD_NodeFromIndex(MD_Node *first, MD_Node *last, int n);
MD_FUNCTION int MD_IndexFromNode(MD_Node *node); MD_FUNCTION int MD_IndexFromNode(MD_Node *node);
MD_FUNCTION MD_Node * MD_RootFromNode(MD_Node *node);
MD_FUNCTION MD_Node * MD_NextNodeSibling(MD_Node *last, MD_String8 string); MD_FUNCTION MD_Node * MD_NextNodeSibling(MD_Node *last, MD_String8 string);
MD_FUNCTION MD_Node * MD_ChildFromString(MD_Node *node, MD_String8 child_string); MD_FUNCTION MD_Node * MD_ChildFromString(MD_Node *node, MD_String8 child_string);
MD_FUNCTION MD_Node * MD_TagFromString(MD_Node *node, MD_String8 tag_string); MD_FUNCTION MD_Node * MD_TagFromString(MD_Node *node, MD_String8 tag_string);
source/md_c_helpers.c
-1
View File
@@ -235,7 +235,6 @@ _MD_NodeParse_ConsumeLiteral(_MD_NodeParseCtx *ctx, MD_Node **out)
{ {
MD_b32 result = 0; MD_b32 result = 0;
if(ctx->at->flags & MD_NodeFlag_StringLiteral || if(ctx->at->flags & MD_NodeFlag_StringLiteral ||
ctx->at->flags & MD_NodeFlag_CharLiteral ||
ctx->at->flags & MD_NodeFlag_Numeric) ctx->at->flags & MD_NodeFlag_Numeric)
{ {
result = 1; result = 1;
source/md_impl.c
+129 -86
View File
@@ -4,13 +4,8 @@
#define MD_PRIVATE_FUNCTION_IMPL MD_FUNCTION_IMPL #define MD_PRIVATE_FUNCTION_IMPL MD_FUNCTION_IMPL
#define MD_UNTERMINATED_TOKEN_LEN_CAP 20 #define MD_UNTERMINATED_TOKEN_LEN_CAP 20
//~ //~ Nil Node Definition
// NOTE(allen): Review @rjf; Building in C++
// While very latest version of C++ have designated initializers
// I would like to be able to build on more simple versions, so I
// ditched the designated initializers in favor of the extra work
// of maintaining order based initializers.
static MD_Node _md_nil_node = static MD_Node _md_nil_node =
{ {
&_md_nil_node, // next &_md_nil_node, // next
@@ -27,26 +22,25 @@ static MD_Node _md_nil_node =
0xdeadffffffffffull, // string_hash 0xdeadffffffffffull, // string_hash
MD_ZERO_STRUCT, // comment_before MD_ZERO_STRUCT, // comment_before
MD_ZERO_STRUCT, // comment_after MD_ZERO_STRUCT, // comment_after
{(MD_u8*)"`NIL DD NODE`", 13}, // filename
0, // file_contents
0, // at 0, // at
&_md_nil_node, // ref_target &_md_nil_node, // ref_target
}; };
//~ Memory Operations //~ Memory Operations
MD_PRIVATE_FUNCTION_IMPL void
MD_FUNCTION_IMPL void
MD_MemoryZero(void *memory, MD_u64 size) MD_MemoryZero(void *memory, MD_u64 size)
{ {
memset(memory, 0, size); memset(memory, 0, size);
} }
MD_PRIVATE_FUNCTION_IMPL void MD_FUNCTION_IMPL void
MD_MemoryCopy(void *dest, void *src, MD_u64 size) MD_MemoryCopy(void *dest, void *src, MD_u64 size)
{ {
memcpy(dest, src, size); memcpy(dest, src, size);
} }
MD_PRIVATE_FUNCTION_IMPL void * MD_FUNCTION_IMPL void *
MD_AllocZero(MD_u64 size) MD_AllocZero(MD_u64 size)
{ {
#if !defined(MD_IMPL_Alloc) #if !defined(MD_IMPL_Alloc)
@@ -620,7 +614,7 @@ MD_FUNCTION_IMPL MD_String8
MD_StringFromNodeKind(MD_NodeKind kind) MD_StringFromNodeKind(MD_NodeKind kind)
{ {
// NOTE(rjf): Must be kept in sync with MD_NodeKind enum. // NOTE(rjf): Must be kept in sync with MD_NodeKind enum.
static char *cstrs[MD_NodeKind_MAX] = static char *cstrs[MD_NodeKind_COUNT] =
{ {
"Nil", "Nil",
"File", "File",
@@ -647,15 +641,21 @@ MD_StringListFromNodeFlags(MD_NodeFlags flags)
"BraceRight", "BraceRight",
"BeforeSemicolon", "BeforeSemicolon",
"BeforeComma",
"AfterSemicolon", "AfterSemicolon",
"BeforeComma",
"AfterComma", "AfterComma",
"StringSingleQuote",
"StringDoubleQuote",
"StringTick",
"StringTripletSingleQuote",
"StringTripletDoubleQuote",
"StringTripletTick",
"Numeric", "Numeric",
"Identifier", "Identifier",
"StringLiteral", "StringLiteral",
"CharLiteral",
}; };
MD_String8List list = MD_ZERO_STRUCT; MD_String8List list = MD_ZERO_STRUCT;
@@ -923,6 +923,7 @@ MD_FUNCTION_IMPL MD_u64
MD_HashPointer(void *p) MD_HashPointer(void *p)
{ {
MD_u64 h = (MD_u64)p; MD_u64 h = (MD_u64)p;
// TODO(rjf): Do we want our own equivalent of UINT64_C?
h = (h ^ (h >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); h = (h ^ (h >> 30)) * UINT64_C(0xbf58476d1ce4e5b9);
h = (h ^ (h >> 27)) * UINT64_C(0x94d049bb133111eb); h = (h ^ (h >> 27)) * UINT64_C(0x94d049bb133111eb);
h = h ^ (h >> 31); h = h ^ (h >> 31);
@@ -935,9 +936,7 @@ MD_MapMakeBucketCount(MD_u64 bucket_count){
// make most sense with a parameter // make most sense with a parameter
MD_Map result = {0}; MD_Map result = {0};
result.bucket_count = bucket_count; result.bucket_count = bucket_count;
// TODO(allen): push array zero result.buckets = MD_PushArrayZero(MD_MapBucket, bucket_count);
result.buckets = MD_PushArray(MD_MapBucket, bucket_count);
memset(result.buckets, 0, sizeof(*result.buckets)*bucket_count);
return(result); return(result);
} }
@@ -1041,35 +1040,31 @@ MD_MapOverwrite(MD_Map *map, MD_MapKey key, void *val){
//~ Parsing //~ Parsing
// TODO(allen): This helper only helps because `ctx` bundles two elements together MD_FUNCTION_IMPL MD_NodeFlags
// that the "low level" MD_MakeNode treats as seperate. However they aren't very MD_NodeFlagsFromTokenKind(MD_TokenKind kind)
// useful as seperate concepts. If we get the "handle of file" concept down to
// a single root node pointer, then this can be eliminated and users can get
// this effect by composing 'ctx->file_root' with 'MD_MakeNode'.
MD_PRIVATE_FUNCTION_IMPL MD_Node *
_MD_MakeNode_Ctx(MD_ParseCtx *ctx, MD_NodeKind kind,
MD_String8 string, MD_String8 outer, MD_u8 *at)
{
return MD_MakeNode(kind, string, outer, ctx->filename, ctx->file_contents.str, at);
}
MD_PRIVATE_FUNCTION_IMPL void _MD_ParseTagList(MD_ParseCtx *ctx, MD_Node **first_out, MD_Node **last_out);
MD_PRIVATE_FUNCTION_IMPL MD_NodeFlags
_MD_NodeFlagsFromTokenKind(MD_TokenKind kind)
{ {
MD_NodeFlags result = 0; MD_NodeFlags result = 0;
switch (kind){ switch (kind){
case MD_TokenKind_Identifier: result = MD_NodeFlag_Identifier; break; case MD_TokenKind_Identifier: result = MD_NodeFlag_Identifier; break;
case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric; break; case MD_TokenKind_NumericLiteral: result = MD_NodeFlag_Numeric; break;
case MD_TokenKind_StringLiteral: result = MD_NodeFlag_StringLiteral; break; case MD_TokenKind_StringLiteralSingleQuote: result |= MD_NodeFlag_StringSingleQuote; goto string_lit;
case MD_TokenKind_CharLiteral: result = MD_NodeFlag_CharLiteral; break; case MD_TokenKind_StringLiteralDoubleQuote: result |= MD_NodeFlag_StringDoubleQuote; goto string_lit;
case MD_TokenKind_StringLiteralTick: result |= MD_NodeFlag_StringTick; goto string_lit;
case MD_TokenKind_StringLiteralSingleQuoteTriplet: result |= MD_NodeFlag_StringTripletSingleQuote; goto string_lit;
case MD_TokenKind_StringLiteralDoubleQuoteTriplet: result |= MD_NodeFlag_StringTripletDoubleQuote; goto string_lit;
case MD_TokenKind_StringLiteralTickTriplet: result |= MD_NodeFlag_StringTripletTick; goto string_lit;
string_lit:;
{
result |= MD_NodeFlag_StringLiteral;
}break;
} }
return(result); return(result);
} }
MD_PRIVATE_FUNCTION_IMPL void _MD_ParseTagList(MD_ParseCtx *ctx, MD_Node **first_out, MD_Node **last_out);
MD_PRIVATE_FUNCTION_IMPL MD_b32 MD_PRIVATE_FUNCTION_IMPL MD_b32
_MD_StringLiteralIsBalanced(MD_Token token) _MD_TokenBoundariesAreBalanced(MD_Token token)
{ {
MD_u64 front_len = token.string.str - token.outer_string.str; MD_u64 front_len = token.string.str - token.outer_string.str;
MD_u64 back_len = (token.outer_string.str + token.outer_string.size) - (token.string.str + token.string.size); MD_u64 back_len = (token.outer_string.str + token.outer_string.size) - (token.string.str + token.string.size);
@@ -1105,10 +1100,20 @@ _MD_ParseTagList(MD_ParseCtx *ctx, MD_Node **first_out, MD_Node **last_out)
MD_Parse_Bump(ctx, next_token); MD_Parse_Bump(ctx, next_token);
MD_Token name = MD_ZERO_STRUCT; MD_Token name = MD_ZERO_STRUCT;
// TODO(rjf): Do we actually care to prohibit people from using
// something other than identifiers as their tag names? If so,
// why? If we can't come up with a good answer for it, then I
// think it makes sense to just allow anything that would've
// been a legal label string here too.
if(MD_Parse_RequireKind(ctx, MD_TokenKind_Identifier, &name)) if(MD_Parse_RequireKind(ctx, MD_TokenKind_Identifier, &name))
{ {
MD_Node *tag = _MD_MakeNode_Ctx(ctx, MD_NodeKind_Tag, MD_Node *tag = MD_MakeNode(MD_NodeKind_Tag, name.string, name.outer_string, name.outer_string.str);
name.string, name.outer_string, name.outer_string.str);
// TODO(rjf): Don't we care if this is a MD_TokenKind_Symbol?
// for the sake of consistency with regular sets, I think it
// makes sense to disallow @foo"("), for example...
MD_Token token = MD_Parse_PeekSkipSome(ctx, 0); MD_Token token = MD_Parse_PeekSkipSome(ctx, 0);
if(MD_StringMatch(token.string, MD_S8Lit("("), 0)) if(MD_StringMatch(token.string, MD_S8Lit("("), 0))
{ {
@@ -1139,7 +1144,7 @@ _MD_ParseTagList(MD_ParseCtx *ctx, MD_Node **first_out, MD_Node **last_out)
MD_FUNCTION_IMPL MD_b32 MD_FUNCTION_IMPL MD_b32
MD_TokenKindIsWhitespace(MD_TokenKind kind) MD_TokenKindIsWhitespace(MD_TokenKind kind)
{ {
return kind > MD_TokenKind_WhitespaceMin && kind < MD_TokenKind_WhitespaceMax; return MD_TokenKind_WhitespaceMin < kind && kind < MD_TokenKind_WhitespaceMax;
} }
MD_FUNCTION_IMPL MD_b32 MD_FUNCTION_IMPL MD_b32
@@ -1151,7 +1156,7 @@ MD_TokenKindIsComment(MD_TokenKind kind)
MD_FUNCTION_IMPL MD_b32 MD_FUNCTION_IMPL MD_b32
MD_TokenKindIsRegular(MD_TokenKind kind) MD_TokenKindIsRegular(MD_TokenKind kind)
{ {
return(kind > MD_TokenKind_RegularMin && kind < MD_TokenKind_RegularMax); return(MD_TokenKind_RegularMin < kind && kind < MD_TokenKind_RegularMax);
} }
MD_FUNCTION void MD_FUNCTION void
@@ -1219,9 +1224,10 @@ MD_PushNodeErrorF(MD_ParseCtx *ctx, MD_Node *node, MD_MessageKind kind, char *fm
MD_FUNCTION void MD_FUNCTION void
MD_PushTokenError(MD_ParseCtx *ctx, MD_Token token, MD_MessageKind kind, MD_String8 str){ MD_PushTokenError(MD_ParseCtx *ctx, MD_Token token, MD_MessageKind kind, MD_String8 str){
MD_Node *stub = _MD_MakeNode_Ctx(ctx, MD_NodeKind_ErrorMarker, MD_Node *stub_file = MD_MakeNode(MD_NodeKind_ErrorMarker, ctx->file_contents, ctx->file_contents, ctx->file_contents.str);
token.string, token.outer_string, token.outer_string.str); MD_Node *stub = MD_MakeNode(MD_NodeKind_ErrorMarker, token.string, token.outer_string, token.outer_string.str);
MD_PushNodeError(ctx, stub, kind, str); MD_PushNodeError(ctx, stub, kind, str);
MD_PushChild(stub_file, stub);
} }
MD_FUNCTION void MD_FUNCTION void
@@ -1358,11 +1364,6 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
// NOTE(allen): Strings // NOTE(allen): Strings
case '"': case '"':
case '\'': case '\'':
// NOTE(rjf): "Bundle-of-tokens" strings (`stuff` or ```stuff```)
// In practice no different than a regular string, but provides an
// alternate syntax which will allow tools like 4coder to treat the
// contents as regular tokens.
case '`': case '`':
{ {
// TODO(allen): proposal: // TODO(allen): proposal:
@@ -1448,13 +1449,27 @@ MD_Parse_LexNext(MD_ParseCtx *ctx)
} }
// set token kind // set token kind
token.kind = MD_TokenKind_StringLiteral; if(is_triplet)
// TODO(allen): I don't see any place where this actually proves useful. {
// I think it'd tidy things up to drop it. we already use this as a string switch(d)
// in a lot of usages of metadesk. {
if (d == '\'' && !is_triplet){ case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuoteTriplet; break;
token.kind = MD_TokenKind_CharLiteral; case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuoteTriplet; break;
case '`': token.kind = MD_TokenKind_StringLiteralTickTriplet; break;
default: break;
} }
}
else
{
switch(d)
{
case '\'': token.kind = MD_TokenKind_StringLiteralSingleQuote; break;
case '"': token.kind = MD_TokenKind_StringLiteralDoubleQuote; break;
case '`': token.kind = MD_TokenKind_StringLiteralTick; break;
default: break;
}
}
}break; }break;
// NOTE(allen): Identifiers, Numbers, Operators // NOTE(allen): Identifiers, Numbers, Operators
@@ -1818,9 +1833,7 @@ MD_ParseOneNodeFromCtx(MD_ParseCtx *ctx)
MD_StringMatch(next_token.string, MD_S8Lit("{"), 0) || MD_StringMatch(next_token.string, MD_S8Lit("{"), 0) ||
MD_StringMatch(next_token.string, MD_S8Lit("["), 0))) MD_StringMatch(next_token.string, MD_S8Lit("["), 0)))
{ {
result.node = _MD_MakeNode_Ctx(ctx, MD_NodeKind_Label, result.node = MD_MakeNode(MD_NodeKind_Label, MD_S8Lit(""), MD_S8Lit(""), next_token.outer_string.str);
MD_S8Lit(""), MD_S8Lit(""),
next_token.outer_string.str);
MD_Parse_Set(ctx, result.node, MD_Parse_Set(ctx, result.node,
MD_ParseSetFlag_Paren | MD_ParseSetFlag_Paren |
@@ -1830,35 +1843,57 @@ MD_ParseOneNodeFromCtx(MD_ParseCtx *ctx)
} }
// NOTE(rjf): Labels // NOTE(rjf): Labels
else if(MD_Parse_RequireKind(ctx, MD_TokenKind_Identifier, &token) || else if(next_token.kind == MD_TokenKind_Identifier ||
MD_Parse_RequireKind(ctx, MD_TokenKind_NumericLiteral, &token) || next_token.kind == MD_TokenKind_NumericLiteral ||
MD_Parse_RequireKind(ctx, MD_TokenKind_StringLiteral, &token) || next_token.kind == MD_TokenKind_StringLiteralTick ||
MD_Parse_RequireKind(ctx, MD_TokenKind_CharLiteral, &token) || next_token.kind == MD_TokenKind_StringLiteralSingleQuote ||
MD_Parse_RequireKind(ctx, MD_TokenKind_Symbol, &token)) next_token.kind == MD_TokenKind_StringLiteralDoubleQuote ||
next_token.kind == MD_TokenKind_StringLiteralTickTriplet ||
next_token.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet ||
next_token.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet ||
next_token.kind == MD_TokenKind_Symbol )
{ {
result.node = _MD_MakeNode_Ctx(ctx, MD_NodeKind_Label, MD_Parse_Bump(ctx, next_token);
token.string, token.outer_string, token.outer_string.str); result.node = MD_MakeNode(MD_NodeKind_Label, next_token.string, next_token.outer_string, next_token.outer_string.str);
result.node->flags |= _MD_NodeFlagsFromTokenKind(token.kind); result.node->flags |= MD_NodeFlagsFromTokenKind(next_token.kind);
if(token.kind == MD_TokenKind_CharLiteral || token.kind == MD_TokenKind_StringLiteral) // TODO(rjf): Before we were just able to check one kind. I think preserving
// which kind of string literal was used is very important, for the same reason
// that preserving which symbols were used to delimit a set is important.
// But, having to manage this "group of kinds" is a little bit annoying.
// Maybe we should define the set of legal kinds for certain syntactic
// contexts somewhere unified, so that the parser is never duplicating this?
//
// It's also possible that it never matters and we only ever use this group
// in one place, but I just got that "we're duplicating stuff" allergy that
// I usually get.
//
// If that turned out to be a good idea, maybe we could do something like
// MD_TokenKindIsLegalLabelHead, MD_TokenKindNeedsBalancing?? I don't know.
if(next_token.kind == MD_TokenKind_StringLiteralTick ||
next_token.kind == MD_TokenKind_StringLiteralSingleQuote ||
next_token.kind == MD_TokenKind_StringLiteralDoubleQuote ||
next_token.kind == MD_TokenKind_StringLiteralTickTriplet ||
next_token.kind == MD_TokenKind_StringLiteralSingleQuoteTriplet ||
next_token.kind == MD_TokenKind_StringLiteralDoubleQuoteTriplet)
{ {
if(!_MD_StringLiteralIsBalanced(token)) if(!_MD_TokenBoundariesAreBalanced(next_token))
{ {
MD_String8 capped = MD_StringPrefix(token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP); MD_String8 capped = MD_StringPrefix(next_token.outer_string, MD_UNTERMINATED_TOKEN_LEN_CAP);
MD_PushNodeErrorF(ctx, result.node, MD_MessageKind_CatastrophicError, MD_PushNodeErrorF(ctx, result.node, MD_MessageKind_CatastrophicError,
"Unterminated text literal \"%.*s\"", MD_StringExpand(capped)); "Unterminated text literal \"%.*s\"", MD_StringExpand(capped));
} }
} }
else if(token.kind == MD_TokenKind_Symbol && token.string.size == 1 && MD_CharIsReservedSymbol(token.string.str[0])) else if(next_token.kind == MD_TokenKind_Symbol && next_token.string.size == 1 && MD_CharIsReservedSymbol(next_token.string.str[0]))
{ {
MD_u8 c = token.string.str[0]; MD_u8 c = next_token.string.str[0];
if(c == '}' || c == ']' || c == ')') if(c == '}' || c == ']' || c == ')')
{ {
MD_PushTokenErrorF(ctx, token, MD_MessageKind_CatastrophicError, "Unbalanced \"%c\"", c); MD_PushTokenErrorF(ctx, next_token, MD_MessageKind_CatastrophicError, "Unbalanced \"%c\"", c);
} }
else else
{ {
MD_PushTokenErrorF(ctx, token, MD_MessageKind_Error, "Unexpected reserved symbol \"%c\"", MD_PushTokenErrorF(ctx, next_token, MD_MessageKind_Error, "Unexpected reserved symbol \"%c\"",
c); c);
} }
} }
@@ -1979,11 +2014,8 @@ MD_FUNCTION_IMPL MD_ParseResult
MD_ParseWholeString(MD_String8 filename, MD_String8 contents) MD_ParseWholeString(MD_String8 filename, MD_String8 contents)
{ {
MD_ParseResult result = MD_ZERO_STRUCT; MD_ParseResult result = MD_ZERO_STRUCT;
// TODO(allen): we want to make the string for this actually just MD_String8 root_string = filename;
// be the filename in the root/file idea. MD_Node *root = MD_MakeNode(MD_NodeKind_File, root_string, root_string, contents.str);
MD_String8 root_string = MD_PushStringF("`DD Parsed From \"%.*s\"`", MD_StringExpand(filename));
MD_Node *root = MD_MakeNode(MD_NodeKind_File, root_string, root_string,
filename, contents.str, contents.str);
if(contents.size > 0) if(contents.size > 0)
{ {
// NOTE(allen): setup parse context // NOTE(allen): setup parse context
@@ -2045,13 +2077,15 @@ MD_ParseWholeFile(MD_String8 filename)
//~ Location Conversions //~ Location Conversions
MD_PRIVATE_FUNCTION_IMPL MD_CodeLoc MD_FUNCTION_IMPL MD_CodeLoc
MD_CodeLocFromFileOffset(MD_String8 filename, MD_u8 *base, MD_u8 *at) MD_CodeLocFromFileBaseOff(MD_String8 filename, MD_u8 *base, MD_u8 *at)
{ {
MD_CodeLoc loc; MD_CodeLoc loc;
loc.filename = filename; loc.filename = filename;
loc.line = 1; loc.line = 1;
loc.column = 1; loc.column = 1;
if(base != 0)
{
for(MD_u64 i = 0; base+i < at && base[i]; i += 1) for(MD_u64 i = 0; base+i < at && base[i]; i += 1)
{ {
if(base[i] == '\n') if(base[i] == '\n')
@@ -2064,13 +2098,15 @@ MD_CodeLocFromFileOffset(MD_String8 filename, MD_u8 *base, MD_u8 *at)
loc.column += 1; loc.column += 1;
} }
} }
}
return loc; return loc;
} }
MD_FUNCTION_IMPL MD_CodeLoc MD_FUNCTION_IMPL MD_CodeLoc
MD_CodeLocFromNode(MD_Node *node) MD_CodeLocFromNode(MD_Node *node)
{ {
MD_CodeLoc loc = MD_CodeLocFromFileOffset(node->filename, node->file_contents, node->at); MD_Node *root = MD_RootFromNode(node);
MD_CodeLoc loc = MD_CodeLocFromFileBaseOff(root->string, root->at, node->at);
return loc; return loc;
} }
@@ -2087,8 +2123,7 @@ MD_NilNode(void) { return &_md_nil_node; }
MD_FUNCTION_IMPL MD_Node * MD_FUNCTION_IMPL MD_Node *
MD_MakeNode(MD_NodeKind kind, MD_String8 string, MD_MakeNode(MD_NodeKind kind, MD_String8 string,
MD_String8 whole_string, MD_String8 filename, MD_String8 whole_string, MD_u8 *at)
MD_u8 *file_contents, MD_u8 *at)
{ {
MD_Node *node = MD_PushArray(MD_Node, 1); MD_Node *node = MD_PushArray(MD_Node, 1);
node->kind = kind; node->kind = kind;
@@ -2097,8 +2132,6 @@ MD_MakeNode(MD_NodeKind kind, MD_String8 string,
node->next = node->prev = node->parent = node->next = node->prev = node->parent =
node->first_child = node->last_child = node->first_child = node->last_child =
node->first_tag = node->last_tag = node->ref_target = MD_NilNode(); node->first_tag = node->last_tag = node->ref_target = MD_NilNode();
node->filename = filename;
node->file_contents = file_contents;
node->at = at; node->at = at;
return node; return node;
} }
@@ -2136,8 +2169,7 @@ MD_PushTag(MD_Node *node, MD_Node *tag)
MD_FUNCTION_IMPL MD_Node* MD_FUNCTION_IMPL MD_Node*
MD_PushReference(MD_Node *list, MD_Node *target) MD_PushReference(MD_Node *list, MD_Node *target)
{ {
MD_Node *n = MD_MakeNode(MD_NodeKind_Reference, target->string, target->whole_string, MD_Node *n = MD_MakeNode(MD_NodeKind_Reference, target->string, target->whole_string, target->at);
target->filename, target->file_contents, target->at);
n->ref_target = target; n->ref_target = target;
MD_PushChild(list, n); MD_PushChild(list, n);
return(n); return(n);
@@ -2190,6 +2222,17 @@ MD_IndexFromNode(MD_Node *node)
return idx; return idx;
} }
MD_FUNCTION MD_Node *
MD_RootFromNode(MD_Node *node)
{
MD_Node *parent = node;
for(MD_Node *p = parent; !MD_NodeIsNil(p); p = p->parent)
{
parent = p;
}
return parent;
}
MD_FUNCTION_IMPL MD_Node * MD_FUNCTION_IMPL MD_Node *
MD_NextNodeSibling(MD_Node *last, MD_String8 string) MD_NextNodeSibling(MD_Node *last, MD_String8 string)
{ {
tests/grammar.c
+7 -7
View File
@@ -66,7 +66,7 @@ static MD_Node * NewChildLabel(MD_Node *parent, MD_String8 label)
{ {
MD_Node *result = 0; MD_Node *result = 0;
result = MD_MakeNode(MD_NodeKind_Label, label, label, MD_S8Lit(""), 0, 0); result = MD_MakeNode(MD_NodeKind_Label, label, label, 0);
if(parent) if(parent)
{ {
MD_PushChild(parent, result); MD_PushChild(parent, result);
@@ -86,7 +86,7 @@ static MD_Node * NewChild(MD_Node *parent)
#define SET_DEPTH(depth_map, node, depth) MD_MapOverwrite(depth_map, MD_MapKeyPtr(node), (void *)(depth)) #define SET_DEPTH(depth_map, node, depth) MD_MapOverwrite(depth_map, MD_MapKeyPtr(node), (void *)(depth))
static void PrintRule(MD_Map *depth_map, MD_Node *rule) static void PrintRule(MD_Map *depth_map, MD_Node *rule)
{ {
MD_b32 is_literal_char = rule->flags & MD_NodeFlag_CharLiteral; MD_b32 is_literal_char = rule->flags & MD_NodeFlag_StringLiteral;
MD_b32 optional = MD_NodeHasTag(rule, MD_S8Lit(OPTIONAL_TAG)); MD_b32 optional = MD_NodeHasTag(rule, MD_S8Lit(OPTIONAL_TAG));
@@ -221,7 +221,7 @@ static void ExpandRule(MD_Node *rule, MD_String8List *out_strings, MD_Node *cur_
} }
else else
{ {
if(rule_element->flags & MD_NodeFlag_CharLiteral) // NOTE(mal): Terminal production if(rule_element->flags & MD_NodeFlag_StringLiteral) // NOTE(mal): Terminal production
{ {
char c = 0; char c = 0;
if(rule_element->string.size == 2 && rule_element->string.str[0] == '\\') if(rule_element->string.size == 2 && rule_element->string.str[0] == '\\')
@@ -308,7 +308,7 @@ static MD_Node * FindNonTerminalProduction(MD_Node *node, MD_Map *visited)
{ {
if(MD_NodeIsNil(node->first_child)) if(MD_NodeIsNil(node->first_child))
{ {
if(node->flags & MD_NodeFlag_CharLiteral) if(node->flags & MD_NodeFlag_StringLiteral)
{ {
} }
else else
@@ -425,7 +425,7 @@ static void ComputeElementDepth(MD_Map *depth_map, MD_Node *re)
} }
else else
{ {
if(re->flags & MD_NodeFlag_CharLiteral) // NOTE(mal): Terminal production if(re->flags & MD_NodeFlag_StringLiteral) // NOTE(mal): Terminal production
{ {
result = 1; result = 1;
} }
@@ -561,7 +561,7 @@ int main(int argument_count, char **arguments)
else else
{ {
if(MD_StringMatch(rule_element->string, MD_S8Lit("|"), 0) && if(MD_StringMatch(rule_element->string, MD_S8Lit("|"), 0) &&
!(rule_element->flags & MD_NodeFlag_CharLiteral)) !(rule_element->flags & MD_NodeFlag_StringLiteral))
{ {
rule = NewChild(production); rule = NewChild(production);
} }
@@ -658,7 +658,7 @@ int main(int argument_count, char **arguments)
{ {
MD_u64 depth = 0; MD_u64 depth = 0;
MD_Assert(MD_NodeIsNil(rule_element->first_child)); MD_Assert(MD_NodeIsNil(rule_element->first_child));
if(!(rule_element->flags & MD_NodeFlag_CharLiteral)) if(!(rule_element->flags & MD_NodeFlag_StringLiteral))
{ {
MD_Node * production = MD_MapLookup(globals.production_table, MD_MapKeyStr(rule_element->string))->val; MD_Node * production = MD_MapLookup(globals.production_table, MD_MapKeyStr(rule_element->string))->val;
depth = GET_DEPTH(depth_map, production); depth = GET_DEPTH(depth_map, production);
tests/sanity_tests.c
+11 -4
View File
@@ -31,6 +31,13 @@ TestResult(MD_b32 result)
test_ctx.number_of_tests += 1; test_ctx.number_of_tests += 1;
test_ctx.number_passed += !!result; test_ctx.number_passed += !!result;
printf(result ? "." : "X"); printf(result ? "." : "X");
#if 0
if(result == 0)
{
__debugbreak();
}
#endif
} }
static void static void
@@ -58,7 +65,7 @@ EndTest(void)
static MD_Node * static MD_Node *
MakeTestNode(MD_NodeKind kind, MD_String8 string) MakeTestNode(MD_NodeKind kind, MD_String8 string)
{ {
return MD_MakeNode(kind, string, string, MD_S8Lit("`TEST_NODE`"), 0, 0); return MD_MakeNode(kind, string, string, 0);
} }
static MD_C_Expr * static MD_C_Expr *
@@ -280,7 +287,7 @@ int main(void)
TestResult(MD_ParseOneNode(MD_S8Lit(""), MD_S8Lit("\"foo\"")).node->flags & TestResult(MD_ParseOneNode(MD_S8Lit(""), MD_S8Lit("\"foo\"")).node->flags &
MD_NodeFlag_StringLiteral); MD_NodeFlag_StringLiteral);
TestResult(MD_ParseOneNode(MD_S8Lit(""), MD_S8Lit("'foo'")).node->flags & TestResult(MD_ParseOneNode(MD_S8Lit(""), MD_S8Lit("'foo'")).node->flags &
MD_NodeFlag_CharLiteral); MD_NodeFlag_StringLiteral);
} }
Test("Expression Evaluation") Test("Expression Evaluation")
@@ -388,11 +395,11 @@ int main(void)
{ {
TestResult(MD_StringMatch(MD_StringFromNodeKind(MD_NodeKind_Label), MD_S8Lit("Label"), 0)); TestResult(MD_StringMatch(MD_StringFromNodeKind(MD_NodeKind_Label), MD_S8Lit("Label"), 0));
TestResult(MD_StringMatch(MD_StringFromNodeKind(MD_NodeKind_Label), MD_S8Lit("Label"), 0)); TestResult(MD_StringMatch(MD_StringFromNodeKind(MD_NodeKind_Label), MD_S8Lit("Label"), 0));
MD_String8List list = MD_StringListFromNodeFlags(MD_NodeFlag_CharLiteral | MD_NodeFlag_ParenLeft | MD_NodeFlag_BeforeSemicolon); MD_String8List list = MD_StringListFromNodeFlags(MD_NodeFlag_StringLiteral | MD_NodeFlag_ParenLeft | MD_NodeFlag_BeforeSemicolon);
MD_b32 match = 1; MD_b32 match = 1;
for(MD_String8Node *node = list.first; node; node = node->next) for(MD_String8Node *node = list.first; node; node = node->next)
{ {
if(!MD_StringMatch(node->string, MD_S8Lit("CharLiteral"), 0) && if(!MD_StringMatch(node->string, MD_S8Lit("StringLiteral"), 0) &&
!MD_StringMatch(node->string, MD_S8Lit("ParenLeft"), 0) && !MD_StringMatch(node->string, MD_S8Lit("ParenLeft"), 0) &&
!MD_StringMatch(node->string, MD_S8Lit("BeforeSemicolon"), 0)) !MD_StringMatch(node->string, MD_S8Lit("BeforeSemicolon"), 0))
{ {