From 00f6c45f1585ae85631de6987ddddc64009b28f7 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 6 Aug 2023 13:28:19 -0400 Subject: [PATCH] Fixes for serializations found with last commit's test. Should be fine to move on to next major feature.... --- project/components/ast.cpp | 2 +- project/components/interface.parsing.cpp | 101 +++++++++---- project/components/temp/etoktype.cpp | 179 ++++++++++++----------- project/enums/ETokType.csv | 3 + 4 files changed, 171 insertions(+), 114 deletions(-) diff --git a/project/components/ast.cpp b/project/components/ast.cpp index 0ee2a04..05057d5 100644 --- a/project/components/ast.cpp +++ b/project/components/ast.cpp @@ -589,7 +589,7 @@ String AST::to_string() if ( Name == nullptr) { - result.append( "struct\n{\n%s\n};", Body->to_string() ); + result.append_fmt( "struct\n{\n%s\n};", Body->to_string() ); break; } diff --git a/project/components/interface.parsing.cpp b/project/components/interface.parsing.cpp index bcf9307..51649fe 100644 --- a/project/components/interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -297,6 +297,7 @@ namespace Parser case '#': { char const* hash = scanner; + Tokens.append( { hash, 1, TokType::Preprocess_Hash, line, column, false } ); move_forward(); SkipWhitespace(); @@ -354,12 +355,13 @@ namespace Parser if ( current == '\r' ) { - move_forward(); + // move_forward(); + // token.Length++; } if ( current == '\n' ) { - move_forward(); + // move_forward(); // token.Length++; break; } @@ -483,9 +485,15 @@ namespace Parser } } + if ( current == '\r' ) + { + // move_forward(); + // content.Length++; + } + if ( current == '\n' ) { - move_forward(); + // move_forward(); // content.Length++; break; } @@ -826,36 +834,38 @@ namespace Parser if ( current == '/' ) { - token.Type = TokType::Comment; + token.Type = TokType::Comment_Start; + token.Length = 2; + Tokens.append( token ); move_forward(); - token.Text = scanner; - token.Length = 0; + Token content = { scanner, 1, TokType::Comment, line, column, false }; while ( left && current != '\n' && current != '\r' ) { move_forward(); - token.Length++; + content.Length++; } + Tokens.append( content ); if ( current == '\r' ) { move_forward(); } - if ( current == '\n' ) { move_forward(); - // token.Length++; } + continue; } else if ( current == '*' ) { - token.Type = TokType::Comment; + token.Type = TokType::Comment_Start; + token.Length = 2; + Tokens.append( token ); move_forward(); - token.Text = scanner; - token.Length = 0; + Token content = { scanner, 1, TokType::Comment, line, column, false }; bool star = current == '*'; bool slash = scanner[1] == '/'; @@ -863,14 +873,20 @@ namespace Parser while ( left && ! at_end ) { move_forward(); - token.Length++; + content.Length++; star = current == '*'; slash = scanner[1] == '/'; at_end = star && slash; } + Tokens.append( content ); + + Token end = { scanner, 2, TokType::Comment_End, line, column, false }; move_forward(); move_forward(); + + Tokens.append( end ); + continue; } } goto FoundToken; @@ -1116,6 +1132,28 @@ internal CodeUsing parse_using (); constexpr bool inplace_def = true; +internal inline +CodeComment parse_comment() +{ + using namespace Parser; + push_scope(); + + eat( TokType::Comment_Start ); + + CodeComment + result = (CodeComment) make_code(); + result->Type = ECode::Comment; + result->Content = get_cached_string( currtok ); + result->Name = result->Content; + eat( TokType::Comment ); + + if ( check( TokType::Comment_End ) ) + eat( TokType::Comment_End ); + + Context.pop(); + return result; +} + internal inline CodeDefine parse_define() { @@ -2460,6 +2498,9 @@ CodeBody parse_class_struct_body( Parser::TokType which ) Context.Scope->Start = currtok_noskip; + if ( currtok.Type == TokType::Preprocess_Hash ) + eat( TokType::Preprocess_Hash ); + switch ( currtok_noskip.Type ) { case TokType::NewLine: @@ -2467,9 +2508,8 @@ CodeBody parse_class_struct_body( Parser::TokType which ) eat( TokType::NewLine ); break; - case TokType::Comment: - member = def_comment( currtok ); - eat( TokType::Comment ); + case TokType::Comment_Start: + member = parse_comment(); break; case TokType::Access_Public: @@ -2819,6 +2859,9 @@ CodeBody parse_global_nspace( CodeT which ) Context.Scope->Start = currtok_noskip; + if ( currtok.Type == TokType::Preprocess_Hash ) + eat( TokType::Preprocess_Hash ); + switch ( currtok_noskip.Type ) { case TokType::NewLine: @@ -2827,9 +2870,8 @@ CodeBody parse_global_nspace( CodeT which ) eat( TokType::NewLine ); break; - case TokType::Comment: - member = def_comment( currtok ); - eat( TokType::Comment ); + case TokType::Comment_Start: + member = parse_comment(); break; case TokType::Decl_Class: @@ -3141,6 +3183,9 @@ CodeEnum parse_enum( bool inplace_def ) while ( left && currtok_noskip.Type != TokType::BraceCurly_Close ) { + if ( currtok.Type == TokType::Preprocess_Hash ) + eat( TokType::Preprocess_Hash ); + switch ( currtok_noskip.Type ) { case TokType::NewLine: @@ -3148,9 +3193,8 @@ CodeEnum parse_enum( bool inplace_def ) eat( TokType::NewLine ); break; - case TokType::Comment: - member = def_comment( currtok ); - eat( TokType::Comment ); + case TokType::Comment_Start: + member = parse_comment(); break; case TokType::Preprocess_Define: @@ -4212,6 +4256,14 @@ CodeUnion parse_union( bool inplace_def ) while ( ! check_noskip( TokType::BraceCurly_Close ) ) { + if ( currtok.Type == TokType::Preprocess_Hash ) + eat( TokType::Preprocess_Hash ); + + if ( currtok.Line >= 3826 ) + { + log_fmt("here"); + } + Code member = { nullptr }; switch ( currtok_noskip.Type ) { @@ -4221,9 +4273,8 @@ CodeUnion parse_union( bool inplace_def ) eat( TokType::NewLine ); break; - case TokType::Comment: - member = def_comment( currtok ); - eat( TokType::Comment ); + case TokType::Comment_Start: + member = parse_comment(); break; case TokType::Decl_Class: diff --git a/project/components/temp/etoktype.cpp b/project/components/temp/etoktype.cpp index 609134e..9f5553c 100644 --- a/project/components/temp/etoktype.cpp +++ b/project/components/temp/etoktype.cpp @@ -15,94 +15,97 @@ namespace Parser Entry( API_Import, "GEN_API_Import_Code" ) #endif -# define Define_TokType \ - Entry( Invalid, "INVALID" ) \ - Entry( Access_Private, "private" ) \ - Entry( Access_Protected, "protected" ) \ - Entry( Access_Public, "public" ) \ - Entry( Access_MemberSymbol, "." ) \ - Entry( Access_StaticSymbol, "::") \ - Entry( Ampersand, "&" ) \ - Entry( Ampersand_DBL, "&&" ) \ - Entry( Assign_Classifer, ":" ) \ - Entry( Attribute_Open, "[[" ) \ - Entry( Attribute_Close, "]]" ) \ - Entry( BraceCurly_Open, "{" ) \ - Entry( BraceCurly_Close, "}" ) \ - Entry( BraceSquare_Open, "[" ) \ - Entry( BraceSquare_Close, "]" ) \ - Entry( Capture_Start, "(" ) \ - Entry( Capture_End, ")" ) \ - Entry( Comment, "__comment__" ) \ - Entry( Char, "__character__" ) \ - Entry( Comma, "," ) \ - Entry( Decl_Class, "class" ) \ - Entry( Decl_GNU_Attribute, "__attribute__" ) \ - Entry( Decl_MSVC_Attribute, "__declspec" ) \ - Entry( Decl_Enum, "enum" ) \ - Entry( Decl_Extern_Linkage, "extern" ) \ - Entry( Decl_Friend, "friend" ) \ - Entry( Decl_Module, "module" ) \ - Entry( Decl_Namespace, "namespace" ) \ - Entry( Decl_Operator, "operator" ) \ - Entry( Decl_Struct, "struct" ) \ - Entry( Decl_Template, "template" ) \ - Entry( Decl_Typedef, "typedef" ) \ - Entry( Decl_Using, "using" ) \ - Entry( Decl_Union, "union" ) \ - Entry( Identifier, "__identifier__" ) \ - Entry( Module_Import, "import" ) \ - Entry( Module_Export, "export" ) \ - Entry( NewLine, "__NewLine__" ) \ - Entry( Number, "__number__" ) \ - Entry( Operator, "__operator__" ) \ - Entry( Preprocess_Define, "define") \ - Entry( Preprocess_If, "if") \ - Entry( Preprocess_IfDef, "ifdef") \ - Entry( Preprocess_IfNotDef, "ifndef") \ - Entry( Preprocess_ElIf, "elif") \ - Entry( Preprocess_Else, "else") \ - Entry( Preprocess_EndIf, "endif") \ - Entry( Preprocess_Include, "include" ) \ - Entry( Preprocess_Pragma, "pragma") \ - Entry( Preprocess_Content, "__macro_content__") \ - Entry( Preprocess_Macro, "__macro__") \ - Entry( Preprocess_Unsupported, "__unsupported__" ) \ - Entry( Spec_Alignas, "alignas" ) \ - Entry( Spec_Const, "const" ) \ - Entry( Spec_Consteval, "consteval" ) \ - Entry( Spec_Constexpr, "constexpr" ) \ - Entry( Spec_Constinit, "constinit" ) \ - Entry( Spec_Explicit, "explicit" ) \ - Entry( Spec_Extern, "extern" ) \ - Entry( Spec_Final, "final" ) \ - Entry( Spec_Global, "global" ) \ - Entry( Spec_Inline, "inline" ) \ - Entry( Spec_Internal_Linkage, "internal" ) \ - Entry( Spec_LocalPersist, "local_persist" ) \ - Entry( Spec_Mutable, "mutable" ) \ - Entry( Spec_NeverInline, "neverinline" ) \ - Entry( Spec_Override, "override" ) \ - Entry( Spec_Static, "static" ) \ - Entry( Spec_ThreadLocal, "thread_local" ) \ - Entry( Spec_Volatile, "volatile") \ - Entry( Star, "*" ) \ - Entry( Statement_End, ";" ) \ - Entry( StaticAssert, "static_assert" ) \ - Entry( String, "__string__" ) \ - Entry( Type_Unsigned, "unsigned" ) \ - Entry( Type_Signed, "signed" ) \ - Entry( Type_Short, "short" ) \ - Entry( Type_Long, "long" ) \ - Entry( Type_char, "char" ) \ - Entry( Type_int, "int" ) \ - Entry( Type_double, "double" ) \ - Entry( Type_MS_int8, "__int8" ) \ - Entry( Type_MS_int16, "__int16" ) \ - Entry( Type_MS_int32, "__int32" ) \ - Entry( Type_MS_int64, "__int64" ) \ - Entry( Type_MS_W64, "_W64" ) \ - Entry( Varadic_Argument, "..." ) \ +# define Define_TokType \ + Entry( Invalid, "INVALID" ) \ + Entry( Access_Private, "private" ) \ + Entry( Access_Protected, "protected" ) \ + Entry( Access_Public, "public" ) \ + Entry( Access_MemberSymbol, "." ) \ + Entry( Access_StaticSymbol, "::") \ + Entry( Ampersand, "&" ) \ + Entry( Ampersand_DBL, "&&" ) \ + Entry( Assign_Classifer, ":" ) \ + Entry( Attribute_Open, "[[" ) \ + Entry( Attribute_Close, "]]" ) \ + Entry( BraceCurly_Open, "{" ) \ + Entry( BraceCurly_Close, "}" ) \ + Entry( BraceSquare_Open, "[" ) \ + Entry( BraceSquare_Close, "]" ) \ + Entry( Capture_Start, "(" ) \ + Entry( Capture_End, ")" ) \ + Entry( Comment, "__comment__" ) \ + Entry( Comment_End, "__comment_end__" ) \ + Entry( Comment_Start, "__comment start__" ) \ + Entry( Char, "__character__" ) \ + Entry( Comma, "," ) \ + Entry( Decl_Class, "class" ) \ + Entry( Decl_GNU_Attribute, "__attribute__" ) \ + Entry( Decl_MSVC_Attribute, "__declspec" ) \ + Entry( Decl_Enum, "enum" ) \ + Entry( Decl_Extern_Linkage, "extern" ) \ + Entry( Decl_Friend, "friend" ) \ + Entry( Decl_Module, "module" ) \ + Entry( Decl_Namespace, "namespace" ) \ + Entry( Decl_Operator, "operator" ) \ + Entry( Decl_Struct, "struct" ) \ + Entry( Decl_Template, "template" ) \ + Entry( Decl_Typedef, "typedef" ) \ + Entry( Decl_Using, "using" ) \ + Entry( Decl_Union, "union" ) \ + Entry( Identifier, "__identifier__" ) \ + Entry( Module_Import, "import" ) \ + Entry( Module_Export, "export" ) \ + Entry( NewLine, "__NewLine__" ) \ + Entry( Number, "__number__" ) \ + Entry( Operator, "__operator__" ) \ + Entry( Preprocess_Hash, "#" ) \ + Entry( Preprocess_Define, "define") \ + Entry( Preprocess_If, "if") \ + Entry( Preprocess_IfDef, "ifdef") \ + Entry( Preprocess_IfNotDef, "ifndef") \ + Entry( Preprocess_ElIf, "elif") \ + Entry( Preprocess_Else, "else") \ + Entry( Preprocess_EndIf, "endif") \ + Entry( Preprocess_Include, "include" ) \ + Entry( Preprocess_Pragma, "pragma") \ + Entry( Preprocess_Content, "__macro_content__") \ + Entry( Preprocess_Macro, "__macro__") \ + Entry( Preprocess_Unsupported, "__unsupported__" ) \ + Entry( Spec_Alignas, "alignas" ) \ + Entry( Spec_Const, "const" ) \ + Entry( Spec_Consteval, "consteval" ) \ + Entry( Spec_Constexpr, "constexpr" ) \ + Entry( Spec_Constinit, "constinit" ) \ + Entry( Spec_Explicit, "explicit" ) \ + Entry( Spec_Extern, "extern" ) \ + Entry( Spec_Final, "final" ) \ + Entry( Spec_Global, "global" ) \ + Entry( Spec_Inline, "inline" ) \ + Entry( Spec_Internal_Linkage, "internal" ) \ + Entry( Spec_LocalPersist, "local_persist" ) \ + Entry( Spec_Mutable, "mutable" ) \ + Entry( Spec_NeverInline, "neverinline" ) \ + Entry( Spec_Override, "override" ) \ + Entry( Spec_Static, "static" ) \ + Entry( Spec_ThreadLocal, "thread_local" ) \ + Entry( Spec_Volatile, "volatile") \ + Entry( Star, "*" ) \ + Entry( Statement_End, ";" ) \ + Entry( StaticAssert, "static_assert" ) \ + Entry( String, "__string__" ) \ + Entry( Type_Unsigned, "unsigned" ) \ + Entry( Type_Signed, "signed" ) \ + Entry( Type_Short, "short" ) \ + Entry( Type_Long, "long" ) \ + Entry( Type_char, "char" ) \ + Entry( Type_int, "int" ) \ + Entry( Type_double, "double" ) \ + Entry( Type_MS_int8, "__int8" ) \ + Entry( Type_MS_int16, "__int16" ) \ + Entry( Type_MS_int32, "__int32" ) \ + Entry( Type_MS_int64, "__int64" ) \ + Entry( Type_MS_W64, "_W64" ) \ + Entry( Varadic_Argument, "..." ) \ Entry( __Attributes_Start, "__attrib_start__" ) namespace ETokType diff --git a/project/enums/ETokType.csv b/project/enums/ETokType.csv index 42162a8..ad8849a 100644 --- a/project/enums/ETokType.csv +++ b/project/enums/ETokType.csv @@ -16,6 +16,8 @@ BraceSquare_Close, "]" Capture_Start, "(" Capture_End, ")" Comment, "__comemnt__" +Comment_End, "__comment_end__" +Comment_Start, "__comment_start__" Char, "__character__" Comma, "," Decl_Class, "class" @@ -38,6 +40,7 @@ Module_Export, "export" NewLine, "__new_line__" Number, "__number__" Operator, "__operator__" +Preprocess_Hash, "#" Preprocess_Define, "define" Preprocess_If, "if" Preprocess_IfDef, "ifdef"