Fixes for serializations found with last commit's test.

Should be fine to move on to next major feature....
This commit is contained in:
Edward R. Gonzalez 2023-08-06 13:28:19 -04:00
parent 34f286d218
commit 00f6c45f15
4 changed files with 171 additions and 114 deletions

View File

@ -589,7 +589,7 @@ String AST::to_string()
if ( Name == nullptr)
{
result.append( "struct\n{\n%s\n};", Body->to_string() );
result.append_fmt( "struct\n{\n%s\n};", Body->to_string() );
break;
}

View File

@ -297,6 +297,7 @@ namespace Parser
case '#':
{
char const* hash = scanner;
Tokens.append( { hash, 1, TokType::Preprocess_Hash, line, column, false } );
move_forward();
SkipWhitespace();
@ -354,12 +355,13 @@ namespace Parser
if ( current == '\r' )
{
move_forward();
// move_forward();
// token.Length++;
}
if ( current == '\n' )
{
move_forward();
// move_forward();
// token.Length++;
break;
}
@ -483,9 +485,15 @@ namespace Parser
}
}
if ( current == '\r' )
{
// move_forward();
// content.Length++;
}
if ( current == '\n' )
{
move_forward();
// move_forward();
// content.Length++;
break;
}
@ -826,36 +834,38 @@ namespace Parser
if ( current == '/' )
{
token.Type = TokType::Comment;
token.Type = TokType::Comment_Start;
token.Length = 2;
Tokens.append( token );
move_forward();
token.Text = scanner;
token.Length = 0;
Token content = { scanner, 1, TokType::Comment, line, column, false };
while ( left && current != '\n' && current != '\r' )
{
move_forward();
token.Length++;
content.Length++;
}
Tokens.append( content );
if ( current == '\r' )
{
move_forward();
}
if ( current == '\n' )
{
move_forward();
// token.Length++;
}
continue;
}
else if ( current == '*' )
{
token.Type = TokType::Comment;
token.Type = TokType::Comment_Start;
token.Length = 2;
Tokens.append( token );
move_forward();
token.Text = scanner;
token.Length = 0;
Token content = { scanner, 1, TokType::Comment, line, column, false };
bool star = current == '*';
bool slash = scanner[1] == '/';
@ -863,14 +873,20 @@ namespace Parser
while ( left && ! at_end )
{
move_forward();
token.Length++;
content.Length++;
star = current == '*';
slash = scanner[1] == '/';
at_end = star && slash;
}
Tokens.append( content );
Token end = { scanner, 2, TokType::Comment_End, line, column, false };
move_forward();
move_forward();
Tokens.append( end );
continue;
}
}
goto FoundToken;
@ -1116,6 +1132,28 @@ internal CodeUsing parse_using ();
constexpr bool inplace_def = true;
internal inline
CodeComment parse_comment()
{
using namespace Parser;
push_scope();
eat( TokType::Comment_Start );
CodeComment
result = (CodeComment) make_code();
result->Type = ECode::Comment;
result->Content = get_cached_string( currtok );
result->Name = result->Content;
eat( TokType::Comment );
if ( check( TokType::Comment_End ) )
eat( TokType::Comment_End );
Context.pop();
return result;
}
internal inline
CodeDefine parse_define()
{
@ -2460,6 +2498,9 @@ CodeBody parse_class_struct_body( Parser::TokType which )
Context.Scope->Start = currtok_noskip;
if ( currtok.Type == TokType::Preprocess_Hash )
eat( TokType::Preprocess_Hash );
switch ( currtok_noskip.Type )
{
case TokType::NewLine:
@ -2467,9 +2508,8 @@ CodeBody parse_class_struct_body( Parser::TokType which )
eat( TokType::NewLine );
break;
case TokType::Comment:
member = def_comment( currtok );
eat( TokType::Comment );
case TokType::Comment_Start:
member = parse_comment();
break;
case TokType::Access_Public:
@ -2819,6 +2859,9 @@ CodeBody parse_global_nspace( CodeT which )
Context.Scope->Start = currtok_noskip;
if ( currtok.Type == TokType::Preprocess_Hash )
eat( TokType::Preprocess_Hash );
switch ( currtok_noskip.Type )
{
case TokType::NewLine:
@ -2827,9 +2870,8 @@ CodeBody parse_global_nspace( CodeT which )
eat( TokType::NewLine );
break;
case TokType::Comment:
member = def_comment( currtok );
eat( TokType::Comment );
case TokType::Comment_Start:
member = parse_comment();
break;
case TokType::Decl_Class:
@ -3141,6 +3183,9 @@ CodeEnum parse_enum( bool inplace_def )
while ( left && currtok_noskip.Type != TokType::BraceCurly_Close )
{
if ( currtok.Type == TokType::Preprocess_Hash )
eat( TokType::Preprocess_Hash );
switch ( currtok_noskip.Type )
{
case TokType::NewLine:
@ -3148,9 +3193,8 @@ CodeEnum parse_enum( bool inplace_def )
eat( TokType::NewLine );
break;
case TokType::Comment:
member = def_comment( currtok );
eat( TokType::Comment );
case TokType::Comment_Start:
member = parse_comment();
break;
case TokType::Preprocess_Define:
@ -4212,6 +4256,14 @@ CodeUnion parse_union( bool inplace_def )
while ( ! check_noskip( TokType::BraceCurly_Close ) )
{
if ( currtok.Type == TokType::Preprocess_Hash )
eat( TokType::Preprocess_Hash );
if ( currtok.Line >= 3826 )
{
log_fmt("here");
}
Code member = { nullptr };
switch ( currtok_noskip.Type )
{
@ -4221,9 +4273,8 @@ CodeUnion parse_union( bool inplace_def )
eat( TokType::NewLine );
break;
case TokType::Comment:
member = def_comment( currtok );
eat( TokType::Comment );
case TokType::Comment_Start:
member = parse_comment();
break;
case TokType::Decl_Class:

View File

@ -15,94 +15,97 @@ namespace Parser
Entry( API_Import, "GEN_API_Import_Code" )
#endif
# define Define_TokType \
Entry( Invalid, "INVALID" ) \
Entry( Access_Private, "private" ) \
Entry( Access_Protected, "protected" ) \
Entry( Access_Public, "public" ) \
Entry( Access_MemberSymbol, "." ) \
Entry( Access_StaticSymbol, "::") \
Entry( Ampersand, "&" ) \
Entry( Ampersand_DBL, "&&" ) \
Entry( Assign_Classifer, ":" ) \
Entry( Attribute_Open, "[[" ) \
Entry( Attribute_Close, "]]" ) \
Entry( BraceCurly_Open, "{" ) \
Entry( BraceCurly_Close, "}" ) \
Entry( BraceSquare_Open, "[" ) \
Entry( BraceSquare_Close, "]" ) \
Entry( Capture_Start, "(" ) \
Entry( Capture_End, ")" ) \
Entry( Comment, "__comment__" ) \
Entry( Char, "__character__" ) \
Entry( Comma, "," ) \
Entry( Decl_Class, "class" ) \
Entry( Decl_GNU_Attribute, "__attribute__" ) \
Entry( Decl_MSVC_Attribute, "__declspec" ) \
Entry( Decl_Enum, "enum" ) \
Entry( Decl_Extern_Linkage, "extern" ) \
Entry( Decl_Friend, "friend" ) \
Entry( Decl_Module, "module" ) \
Entry( Decl_Namespace, "namespace" ) \
Entry( Decl_Operator, "operator" ) \
Entry( Decl_Struct, "struct" ) \
Entry( Decl_Template, "template" ) \
Entry( Decl_Typedef, "typedef" ) \
Entry( Decl_Using, "using" ) \
Entry( Decl_Union, "union" ) \
Entry( Identifier, "__identifier__" ) \
Entry( Module_Import, "import" ) \
Entry( Module_Export, "export" ) \
Entry( NewLine, "__NewLine__" ) \
Entry( Number, "__number__" ) \
Entry( Operator, "__operator__" ) \
Entry( Preprocess_Define, "define") \
Entry( Preprocess_If, "if") \
Entry( Preprocess_IfDef, "ifdef") \
Entry( Preprocess_IfNotDef, "ifndef") \
Entry( Preprocess_ElIf, "elif") \
Entry( Preprocess_Else, "else") \
Entry( Preprocess_EndIf, "endif") \
Entry( Preprocess_Include, "include" ) \
Entry( Preprocess_Pragma, "pragma") \
Entry( Preprocess_Content, "__macro_content__") \
Entry( Preprocess_Macro, "__macro__") \
Entry( Preprocess_Unsupported, "__unsupported__" ) \
Entry( Spec_Alignas, "alignas" ) \
Entry( Spec_Const, "const" ) \
Entry( Spec_Consteval, "consteval" ) \
Entry( Spec_Constexpr, "constexpr" ) \
Entry( Spec_Constinit, "constinit" ) \
Entry( Spec_Explicit, "explicit" ) \
Entry( Spec_Extern, "extern" ) \
Entry( Spec_Final, "final" ) \
Entry( Spec_Global, "global" ) \
Entry( Spec_Inline, "inline" ) \
Entry( Spec_Internal_Linkage, "internal" ) \
Entry( Spec_LocalPersist, "local_persist" ) \
Entry( Spec_Mutable, "mutable" ) \
Entry( Spec_NeverInline, "neverinline" ) \
Entry( Spec_Override, "override" ) \
Entry( Spec_Static, "static" ) \
Entry( Spec_ThreadLocal, "thread_local" ) \
Entry( Spec_Volatile, "volatile") \
Entry( Star, "*" ) \
Entry( Statement_End, ";" ) \
Entry( StaticAssert, "static_assert" ) \
Entry( String, "__string__" ) \
Entry( Type_Unsigned, "unsigned" ) \
Entry( Type_Signed, "signed" ) \
Entry( Type_Short, "short" ) \
Entry( Type_Long, "long" ) \
Entry( Type_char, "char" ) \
Entry( Type_int, "int" ) \
Entry( Type_double, "double" ) \
Entry( Type_MS_int8, "__int8" ) \
Entry( Type_MS_int16, "__int16" ) \
Entry( Type_MS_int32, "__int32" ) \
Entry( Type_MS_int64, "__int64" ) \
Entry( Type_MS_W64, "_W64" ) \
Entry( Varadic_Argument, "..." ) \
# define Define_TokType \
Entry( Invalid, "INVALID" ) \
Entry( Access_Private, "private" ) \
Entry( Access_Protected, "protected" ) \
Entry( Access_Public, "public" ) \
Entry( Access_MemberSymbol, "." ) \
Entry( Access_StaticSymbol, "::") \
Entry( Ampersand, "&" ) \
Entry( Ampersand_DBL, "&&" ) \
Entry( Assign_Classifer, ":" ) \
Entry( Attribute_Open, "[[" ) \
Entry( Attribute_Close, "]]" ) \
Entry( BraceCurly_Open, "{" ) \
Entry( BraceCurly_Close, "}" ) \
Entry( BraceSquare_Open, "[" ) \
Entry( BraceSquare_Close, "]" ) \
Entry( Capture_Start, "(" ) \
Entry( Capture_End, ")" ) \
Entry( Comment, "__comment__" ) \
Entry( Comment_End, "__comment_end__" ) \
Entry( Comment_Start, "__comment start__" ) \
Entry( Char, "__character__" ) \
Entry( Comma, "," ) \
Entry( Decl_Class, "class" ) \
Entry( Decl_GNU_Attribute, "__attribute__" ) \
Entry( Decl_MSVC_Attribute, "__declspec" ) \
Entry( Decl_Enum, "enum" ) \
Entry( Decl_Extern_Linkage, "extern" ) \
Entry( Decl_Friend, "friend" ) \
Entry( Decl_Module, "module" ) \
Entry( Decl_Namespace, "namespace" ) \
Entry( Decl_Operator, "operator" ) \
Entry( Decl_Struct, "struct" ) \
Entry( Decl_Template, "template" ) \
Entry( Decl_Typedef, "typedef" ) \
Entry( Decl_Using, "using" ) \
Entry( Decl_Union, "union" ) \
Entry( Identifier, "__identifier__" ) \
Entry( Module_Import, "import" ) \
Entry( Module_Export, "export" ) \
Entry( NewLine, "__NewLine__" ) \
Entry( Number, "__number__" ) \
Entry( Operator, "__operator__" ) \
Entry( Preprocess_Hash, "#" ) \
Entry( Preprocess_Define, "define") \
Entry( Preprocess_If, "if") \
Entry( Preprocess_IfDef, "ifdef") \
Entry( Preprocess_IfNotDef, "ifndef") \
Entry( Preprocess_ElIf, "elif") \
Entry( Preprocess_Else, "else") \
Entry( Preprocess_EndIf, "endif") \
Entry( Preprocess_Include, "include" ) \
Entry( Preprocess_Pragma, "pragma") \
Entry( Preprocess_Content, "__macro_content__") \
Entry( Preprocess_Macro, "__macro__") \
Entry( Preprocess_Unsupported, "__unsupported__" ) \
Entry( Spec_Alignas, "alignas" ) \
Entry( Spec_Const, "const" ) \
Entry( Spec_Consteval, "consteval" ) \
Entry( Spec_Constexpr, "constexpr" ) \
Entry( Spec_Constinit, "constinit" ) \
Entry( Spec_Explicit, "explicit" ) \
Entry( Spec_Extern, "extern" ) \
Entry( Spec_Final, "final" ) \
Entry( Spec_Global, "global" ) \
Entry( Spec_Inline, "inline" ) \
Entry( Spec_Internal_Linkage, "internal" ) \
Entry( Spec_LocalPersist, "local_persist" ) \
Entry( Spec_Mutable, "mutable" ) \
Entry( Spec_NeverInline, "neverinline" ) \
Entry( Spec_Override, "override" ) \
Entry( Spec_Static, "static" ) \
Entry( Spec_ThreadLocal, "thread_local" ) \
Entry( Spec_Volatile, "volatile") \
Entry( Star, "*" ) \
Entry( Statement_End, ";" ) \
Entry( StaticAssert, "static_assert" ) \
Entry( String, "__string__" ) \
Entry( Type_Unsigned, "unsigned" ) \
Entry( Type_Signed, "signed" ) \
Entry( Type_Short, "short" ) \
Entry( Type_Long, "long" ) \
Entry( Type_char, "char" ) \
Entry( Type_int, "int" ) \
Entry( Type_double, "double" ) \
Entry( Type_MS_int8, "__int8" ) \
Entry( Type_MS_int16, "__int16" ) \
Entry( Type_MS_int32, "__int32" ) \
Entry( Type_MS_int64, "__int64" ) \
Entry( Type_MS_W64, "_W64" ) \
Entry( Varadic_Argument, "..." ) \
Entry( __Attributes_Start, "__attrib_start__" )
namespace ETokType

View File

@ -16,6 +16,8 @@ BraceSquare_Close, "]"
Capture_Start, "("
Capture_End, ")"
Comment, "__comemnt__"
Comment_End, "__comment_end__"
Comment_Start, "__comment_start__"
Char, "__character__"
Comma, ","
Decl_Class, "class"
@ -38,6 +40,7 @@ Module_Export, "export"
NewLine, "__new_line__"
Number, "__number__"
Operator, "__operator__"
Preprocess_Hash, "#"
Preprocess_Define, "define"
Preprocess_If, "if"
Preprocess_IfDef, "ifdef"

1 Invalid __invalid__
16 Capture_Start (
17 Capture_End )
18 Comment __comemnt__
19 Comment_End __comment_end__
20 Comment_Start __comment_start__
21 Char __character__
22 Comma ,
23 Decl_Class class
40 NewLine __new_line__
41 Number __number__
42 Operator __operator__
43 Preprocess_Hash #
44 Preprocess_Define define
45 Preprocess_If if
46 Preprocess_IfDef ifdef