From 86cd0e1fb7ebe2f860ae1be547eceb5f882f62db Mon Sep 17 00:00:00 2001 From: Ed_ Date: Tue, 18 Apr 2023 22:47:59 -0400 Subject: [PATCH] WIP: Fleshing out parsing constructor This code commit will compile just backing up stuff before I switch the functions to use the lexer instead of manually sifting through the string. --- .vscode/settings.json | 4 +- project/Bloat.hpp | 2 + project/gen.cpp | 916 ++++++++++++++++++++++++++++++++++++++---- project/gen.hpp | 34 +- thirdparty/zpl.h | 61 +-- 5 files changed, 892 insertions(+), 125 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 54ab575..02706d3 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -14,7 +14,9 @@ "exception": "cpp", "optional": "cpp", "tuple": "cpp", - "xmemory": "cpp" + "xmemory": "cpp", + "algorithm": "cpp", + "limits": "cpp" }, "C_Cpp.intelliSenseEngineFallback": "disabled" } \ No newline at end of file diff --git a/project/Bloat.hpp b/project/Bloat.hpp index 58ed307..d3b9afd 100644 --- a/project/Bloat.hpp +++ b/project/Bloat.hpp @@ -60,6 +60,7 @@ using zpl::EFileError_NONE; using zpl::alloc; using zpl::arena_allocator; using zpl::arena_init_from_memory; +using zpl::arena_init_from_allocator; using zpl::arena_free; using zpl::bprintf; using zpl::char_is_alpha; @@ -79,6 +80,7 @@ using zpl::snprintf_va; using zpl::string_appendc; using zpl::string_append_fmt; using zpl::string_append_length; +using zpl::string_make_length; using zpl::string_length; using zpl::string_make; using zpl::strnlen; diff --git a/project/gen.cpp b/project/gen.cpp index 53b8a39..3a73a0d 100644 --- a/project/gen.cpp +++ b/project/gen.cpp @@ -1585,6 +1585,37 @@ namespace gen return result; } + Code def_typedef( u32 length, char const* name, Code type ) + { + name_check( def_typedef, length, name ); + null_check( def_typedef, type ); + + if ( type->Type != ECode::Typename ) + { + log_failure( "gen::def_typedef: type was not a Typename" ); + return Code::Invalid; + } + + // Registering the type. + Code registered_type = def_type( length, name ); + + if ( ! registered_type ) + { + log_failure( "gen::def_typedef: failed to register type" ); + return Code::Invalid; + } + + Code + result = make_code(); + result->Name = get_cached_string( name, length ); + result->Type = ECode::Typedef; + + result->add_entry( type ); + result.lock(); + + return result; + } + Code def_type( u32 length, char const* name, Code specifiers ) { name_check( def_type, length, name ); @@ -1595,16 +1626,15 @@ namespace gen return Code::Invalid; } + + Code result = make_code(); result->Name = get_cached_string( name, length ); result->Type = ECode::Typename; if ( specifiers ) - { - result->Entries = make_code_entries(); result->add_entry( specifiers ); - } result.lock(); return result; @@ -1615,6 +1645,14 @@ namespace gen name_check( def_using, length, name ); null_check( def_using, type ); + Code register_type = def_type( length, name ); + + if ( ! register_type ) + { + log_failure( "gen::def_using: failed to register type" ); + return Code::Invalid; + } + Code result = make_code(); result->Name = get_cached_string( name, length ); @@ -1639,8 +1677,6 @@ namespace gen /* Body related functions typically follow the same implementation pattern. So I opted to use inline helper macros to get the implementaiton done. - I didn't want to use a macro for it since I didn't want to make half the library - redundancies hide behind macros. The implementation pattern is as follows: * Validate a valid parameter num was provided, or code array @@ -2234,6 +2270,7 @@ namespace gen # pragma endregion Upfront Constructors # pragma region Incremetnal Constructors +# ifdef GEN_FEATURE_INCREMENTAL Code make_class( s32 length, char const* name, Code parent, Code specifiers ) { using namespace ECode; @@ -2468,6 +2505,19 @@ namespace gen # pragma endregion Incremetnal Constructions # pragma region Parsing Constructors +# ifdef GEN_FEATURE_PARSING +/* + These constructors are the most implementation intensive other than the edtior or scanner. + + The current implementation is very convervative and does not use a lexer to process the input first. + Instead, it just sifts through the content directly looking for the syntax pattern for the code type + and then constructs the AST after. + + Eventually I will problably end up using a dedicated lexer for parser constructors to lower the sloc. + + It uses the upfront constructors to help keep code from getitng to large since the target is to keep the sloc low +*/ + # pragma region Helper Macros # define check_parse_args( func, length, def ) \ if ( length <= 0 ) \ @@ -2507,13 +2557,26 @@ namespace gen left--; \ scanner++ -# define SkipWhitespace() \ - while ( left && char_is_space( current ) ) \ - { \ - move_forward(); \ +# define SkipWhitespace() \ + while ( left && char_is_space( current ) ) \ + { \ + move_forward(); \ + } + +# define SkipWhitespace_Checked( Context_, Msg_, ... ) \ + while ( left && char_is_space( current ) ) \ + { \ + move_forward(); \ + } \ + if ( left <= 0 ) \ + { \ + log_failure( "gen::" txt(Context_) ": " Msg_, __VA_ARGS__ ); \ + return Code::Invalid; \ } # define GetWord() \ + word = scanner; \ + word_length = 0; \ while ( left && char_is_alphanumeric( current ) || current == '_' ) \ { \ move_forward(); \ @@ -2533,11 +2596,547 @@ namespace gen continue; \ } \ } + +# define curr_tok ( * tokens ) + +# define eat( Type_ ) \ + if ( curr_tok.Type != Type_ ) \ + { \ + String token_str = string_make_length( g_allocator, curr_tok.Address, curr_tok.Length ); \ + log_failure( "gen::" txt(context) ": expected %s, got %s", txt(Type_), curr_tok.Type ); \ + return Code::Invalid; \ + } \ + tokens++; \ + left-- # pragma endregion Helper Macros +# pragma region Lexer +/* + This is a simple lexer that focuses on tokenizing only tokens relevant to the library. + It will not be capable of lexing C++ code with unsupported features. +*/ + +// Angle brackets not supported as they are used for template arguments outside of expressions +// Any angle brackets found will be considered an operator token. + +# define Define_TokType \ + Entry( Access_Public, "public" ) \ + Entry( Access_Protected, "protected" ) \ + Entry( Access_Private, "private" ) \ + Entry( Access_MemberSymbol, "." ) \ + Entry( Access_StaticSymbol, "::") \ + Entry( Ampersand, "&" ) \ + Entry( Ampersand_DBL, "&&" ) \ + Entry( Assign_Classifer, ":" ) \ + Entry( BraceCurly_Open, "{" ) \ + Entry( BraceCurly_Close, "}" ) \ + Entry( BraceSquare_Open, "[" ) \ + Entry( BraceSquare_Close, "]" ) \ + Entry( Capture_Start, "(" ) \ + Entry( Capture_End, ")" ) \ + Entry( Comment, "__comment__" ) \ + Entry( Char, "__char__" ) \ + Entry( Comma, "," ) \ + Entry( Decl_Class, "class" ) \ + Entry( Decl_Enum, "enum" ) \ + Entry( Decl_Friend, "friend" ) \ + Entry( Decl_Namespace, "namespace" ) \ + Entry( Decl_Struct, "struct" ) \ + Entry( Decl_Using, "using" ) \ + Entry( Decl_Union, "union" ) \ + Entry( Identifier, "__SymID__" ) \ + Entry( Number, "number" ) \ + Entry( Operator, "operator" ) \ + Entry( Spec_API, txt(API_Keyword) ) \ + Entry( Spec_Alignas, "alignas" ) \ + Entry( Spec_CLinkage, "extern \"C\"" ) \ + Entry( Spec_Const, "const" ) \ + Entry( Spec_Consteval, "consteval" ) \ + Entry( Spec_Constexpr, "constexpr" ) \ + Entry( Spec_Constinit, "constinit" ) \ + Entry( Spec_Export, "export" ) \ + Entry( Spec_Extern, "extern" ) \ + Entry( Spec_Import, "import" ) \ + Entry( Spec_Inline, "inline" ) \ + Entry( Spec_Module, "module" ) \ + Entry( Spec_Static, "static" ) \ + Entry( Spec_ThreadLocal, "thread_local" ) \ + Entry( Spec_Volatile, "volatile") \ + Entry( Star, "*" ) \ + Entry( Statement_End, ";" ) \ + Entry( String, "__String__" ) + + enum class TokType : u32 + { +# define Entry( Name_, Str_ ) Name_, + Define_TokType +# undef Entry + + Num, + Invalid + }; + + struct Token + { + char const* Address; + s32 Length; + TokType Type; + }; + + TokType get_token_type( char const* word, s32 length ) + { + local_persist + char const* lookup[(u32)TokType::Num] = + { +# define Entry( Name_, Str_ ) Str_, + Define_TokType +# undef Entry + }; + + for ( u32 index = 0; index < (u32)TokType::Num; index++ ) + { + if ( str_compare( word, lookup[index], length ) == 0 ) + return scast(TokType, index); + } + + return TokType::Invalid; + } + + Arena LexAllocator; + + Array(Token) lex( s32 length, char const* content) + { + do_once_start + arena_init_from_allocator( & LexAllocator, heap(), megabytes(10) ); + + if ( LexAllocator.physical_start == nullptr ) + { + log_failure( "gen::lex: failed to allocate memory for parsing constructor's lexer"); + return nullptr; + } + do_once_end + + local_persist thread_local + Array(Token) Tokens = nullptr; + + s32 left = length; + char const* scanner = content; + + char const* word = scanner; + s32 word_length = 0; + + SkipWhitespace(); + if ( left <= 0 ) + { + log_failure( "gen::lex: no tokens found (only whitespace provided)" ); + return Tokens; + } + + if ( Tokens ) + array_clear( Tokens ); + + array_init_reserve( Tokens, LexAllocator, length / 8 ); + + while (left ) + { + Token token = { nullptr, 0, TokType::Invalid }; + + switch ( current ) + { + case '.': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Access_MemberSymbol; + + if (left) + move_forward(); + goto FoundToken; + + case '&' : + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Ampersand; + + if (left) + move_forward(); + + if ( current == '&' ) // && + { + token.Length = 2; + token.Type = TokType::Ampersand_DBL; + + if (left) + move_forward(); + } + + goto FoundToken; + + case ':': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Assign_Classifer; + + if (left) + move_forward(); + + if ( current == ':' ) + { + move_forward(); + token.Type = TokType::Access_StaticSymbol; + token.Length++; + } + goto FoundToken; + + case '{': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::BraceCurly_Open; + goto FoundToken; + + case '}': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::BraceCurly_Close; + goto FoundToken; + + case '[': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::BraceSquare_Open; + if ( left ) + { + move_forward(); + + if ( current == ']' ) + { + token.Length = 2; + token.Type = TokType::Operator; + move_forward(); + } + } + goto FoundToken; + + case ']': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::BraceSquare_Close; + goto FoundToken; + + case '(': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Capture_Start; + if ( left ) + { + move_forward(); + + if ( current == ')' ) + { + token.Length = 2; + token.Type = TokType::Operator; + move_forward(); + } + } + goto FoundToken; + + case ')': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Capture_End; + goto FoundToken; + + case '\'': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Char; + + move_forward(); + + while ( left && current != '\'' ) + { + move_forward(); + token.Length++; + } + + if ( left ) + { + move_forward(); + token.Length++; + } + goto FoundToken; + + case ',': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Comma; + goto FoundToken; + + case '*': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Star; + goto FoundToken; + + case ';': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Statement_End; + goto FoundToken; + + case '"': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::String; + + move_forward(); + while ( left ) + { + if ( current == '"' ) + { + move_forward(); + break; + } + + if ( current == '\\' ) + { + move_forward(); + token.Length++; + + if ( left ) + { + move_forward(); + token.Length++; + } + continue; + } + + move_forward(); + token.Length++; + } + goto FoundToken; + + // All other operators we just label as an operator and move forward. + case '+': + case '%': + case '^': + case '~': + case '!': + case '=': + case '<': + case '>': + case '|': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Operator; + + if (left) + move_forward(); + + if ( current == '=' ) + { + token.Length++; + + if (left) + move_forward(); + } + else while ( left && current == *(scanner - 1) && length < 3 ) + { + token.Length++; + + if (left) + move_forward(); + } + goto FoundToken; + + // Dash is unfortunatlly a bit more complicated... + case '-': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Operator; + if ( left ) + { + move_forward(); + + if ( current == '>' ) + { + token.Length++; + move_forward(); + + if ( current == '*' ) + { + token.Length++; + move_forward(); + } + } + else if ( current == '=' ) + { + token.Length++; + + if (left) + move_forward(); + } + else while ( left && current == *(scanner - 1) && length < 3 ) + { + token.Length++; + + if (left) + move_forward(); + } + } + goto FoundToken; + + case '/': + token.Address = scanner; + token.Length = 1; + token.Type = TokType::Operator; + + if ( left ) + { + move_forward(); + + if ( current == '/' ) + { + token.Type = TokType::Comment; + + while ( left && current != '\n' ) + { + move_forward(); + token.Length++; + } + } + else if ( current == '*' ) + { + token.Type = TokType::Comment; + + while ( left && ( current != '*' || *(scanner + 1) != '/' ) ) + { + move_forward(); + token.Length++; + } + move_forward(); + move_forward(); + } + } + goto FoundToken; + } + + SkipWhitespace(); + if ( left <= 0 ) + break; + + if ( char_is_alpha( current ) || current == '_' ) + { + token.Address = scanner; + token.Length = 1; + move_forward(); + + while ( left && ( char_is_alphanumeric(current) || current == '_' ) ) + { + move_forward(); + token.Length++; + } + + goto FoundToken; + } + else + { + String context_str = zpl::string_sprintf_buf( g_allocator, "%s", scanner, min( 100, left ) ); + + log_failure( "Failed to lex token %s", context_str ); + + // Skip to next whitespace since we can't know if anything else is valid until then. + while ( left && ! char_is_space( current ) ) + { + move_forward(); + } + } + + FoundToken: + + if ( token.Type != TokType::Invalid ) + { + array_append( Tokens, token ); + continue; + } + + TokType type = get_token_type( token.Address, token.Length ); + + if ( type != TokType::Invalid ) + { + token.Type = type; + array_append( Tokens, token ); + } + else + { + // Its most likely an identifier... + + + String tok_str = zpl::string_sprintf_buf( g_allocator, "%s", token.Address, token.Length ); + + log_failure( "Failed to lex token %s", tok_str ); + + // Skip to next whitespace since we can't know if anything else is valid until then. + while ( left && ! char_is_space( current ) ) + { + move_forward(); + } + } + } + + return Tokens; + } +# pragma endregion Lexer + Code parse_class( s32 length, char const* def ) { - not_implemented( parse_class ); + Array(Token) tokens = lex( length, def ); + + if ( tokens == nullptr || array_count( tokens ) == 0 ) + { + log_failure( "gen::parse_class: no tokens found" ); + return Code::Invalid; + } + + Token name { nullptr, 0, TokType::Invalid }; + + Code parent = { nullptr }; + Code speciifes = { nullptr }; + Code body = { nullptr }; + + Token& curr_token = * tokens; + +# define eat( Type_ ) \ + if ( curr_token.Type != Type_ ) \ + { \ + String token_str = string_make_length( g_allocator, curr_token.Address, curr_token.Length ); \ + log_failure( "gen::parse_class: expected %s, got %s", txt(Type_), curr_token.Type ); \ + return Code::Invalid; \ + } \ + tokens++; \ + curr_token = * tokens; \ + left-- + + s32 left = array_count( tokens ); + do + { + Token token = tokens[ array_count(tokens) - left ]; + + eat( TokType::Decl_Class ); + + // check for api specifier... + + name = curr_token; + + switch ( token.Type ) + { + case TokType::Identifier: + { + } + } + } + while ( left--, left > 0 ); + +# undef eat } Code parse_enum( s32 length, char const* def ) @@ -2561,12 +3160,7 @@ namespace gen bool is_enum_class = false; - SkipWhitespace(); - if ( left <= 0 ) - { - log_failure( "gen::parse_enum: enum definition was empty" ); - return Code::Invalid; - } + SkipWhitespace_Checked( parse_enum, "enum definition was empty" ); GetWord(); if ( word_length != 4 || str_compare( word, "enum", word_length ) != 0 ) @@ -2575,24 +3169,14 @@ namespace gen return Code::Invalid; } - SkipWhitespace(); - if ( left <= 0 ) - { - log_failure( "gen::parse_enum: enum definition did not have a name" ); - return Code::Invalid; - } + SkipWhitespace_Checked( parse_enum, "enum definition did not have a name" ); GetWord(); if ( word_length == 5 && str_compare( word, "class", word_length ) == 0 ) { is_enum_class = true; - } - SkipWhitespace(); - if ( left <= 0 ) - { - log_failure( "gen::parse_enum: enum definition did not have a name" ); - return Code::Invalid; + SkipWhitespace_Checked( parse_enum, "enum definition did not have a name" ); } GetWord(); @@ -2607,7 +3191,7 @@ namespace gen SkipWhitespace(); GetWord(); - def_type( word_length, word, type ); + type = def_type( word_length, word, type ); } SkipWhitespace(); @@ -2633,12 +3217,7 @@ namespace gen break; } - SkipWhitespace(); - if ( left <= 0 ) - { - log_failure( "gen::parse_enum: enum definition did not have a body" ); - return Code::Invalid; - } + SkipWhitespace_Checked( parse_enum, "enum definition did not have a body" ); GetWord(); if ( word_length == 0 ) @@ -2693,7 +3272,37 @@ namespace gen Code parse_friend( s32 length, char const* def ) { - not_implemented( parse_friend ); +# define curr_tok tokens[0] +# define context parse_friend + +# define eat( Type_ ) \ + if ( curr_tok.Type != Type_ ) \ + { \ + String token_str = string_make_length( g_allocator, curr_tok.Address, curr_tok.Length ); \ + log_failure( "gen::" txt(context) ": expected %s, got %s", txt(Type_), curr_tok.Type ); \ + return Code::Invalid; \ + } \ + tokens++; \ + left-- + + check_parse_args( parse_friend, length, def ); + + Array(Token) tokens = lex( length, def ); + if ( tokens == nullptr ) + { + log_failure( "gen::parse_friend: no tokens found for provided definition" ); + return Code::Invalid; + } + + s32 left = array_count( tokens ); + + eat( TokType::Decl_Friend ); + + // This can either be a simple type, or a function declaration. + // If its a function declaration, it will have a return type, followed by a name, followed by a parameter list. + // If its a simple type, it will have a type, followed by a name. + +# undef eat } Code parse_global_body( s32 length, char const* def ) @@ -2876,11 +3485,73 @@ namespace gen not_implemented( parse_variable ); } + inline + bool parse_type_helper_tok( char const* func_name + , s32& left, Array(Token)& tokens + , u8& num_specifiers, SpecifierT* specs_found + , Code& array_expr + ) + { + + } + + Code parse_type( s32 length, char const* def ) + { +# define context parse_type + + check_parse_args( parse_type, length, def ); + + Array(Token) tokens = lex( length, def ); + + if ( tokens == nullptr ) + { + log_failure( "gen::parse_friend: no tokens found for provided definition" ); + return Code::Invalid; + } + + s32 left = array_count( tokens ); + + Token* name = nullptr; + Code array_expr = { nullptr }; + + SpecifierT specs_found[16] { ESpecifier::Num_Specifiers }; + u8 num_specifiers = 0; + + bool helper_result = parse_type_helper_tok( txt(parse_type) + , left, tokens + , num_specifiers, specs_found + , array_expr + ); + + if ( ! helper_result ) + return Code::Invalid; + + using namespace ECode; + + Code + result = make_code(); + result->Type = Typename; + result->Name = get_cached_string( name->Address, name->Length ); + + if (num_specifiers) + { + Code specifiers = def_specifiers( num_specifiers, specs_found ); + + result->add_entry( specifiers ); + } + + if ( array_expr ) + result->add_entry( array_expr ); + + result.lock(); + return result; + } + inline bool parse__type_helper( char const* func_name - , s32 length, char const* def - , s32 name_length, char const* name - , u8 num_specifiers, SpecifierT* specs_found + , s32& length, char const* def + , s32& name_length, char const*& name + , u8& num_specifiers, SpecifierT* specs_found , Code& array_expr) { s32 left = length; @@ -2893,8 +3564,6 @@ namespace gen do { // Clearing any whitespace - SkipWhitespace(); - if ( left <= 0 ) { log_failure( "gen::%s: Error, reached end of string before finding typename", func_name ); @@ -2913,7 +3582,6 @@ namespace gen do { SkipWhitespace(); - if ( left <= 0 ) break; @@ -2947,7 +3615,6 @@ namespace gen move_forward(); SkipWhitespace(); - if ( left <= 0 ) { log_failure( "gen::%s: Error, reached end of string before finding array expression", func_name ); @@ -2984,7 +3651,7 @@ namespace gen while (1); } - Code parse_type( s32 length, char const* def ) + Code parse_type_non_lex( s32 length, char const* def ) { check_parse_args( parse_type, length, def ); @@ -3013,9 +3680,12 @@ namespace gen result->Type = Typename; result->Name = get_cached_string( name, name_length ); - Code specifiers = def_specifiers( num_specifiers, specs_found ); + if (num_specifiers) + { + Code specifiers = def_specifiers( num_specifiers, specs_found ); - result->add_entry( specifiers ); + result->add_entry( specifiers ); + } if ( array_expr ) result->add_entry( array_expr ); @@ -3047,13 +3717,7 @@ namespace gen do { - SkipWhitespace(); - - if ( left <= 0 ) - { - log_failure( "gen::parse_typedef: Error, reached end of string before finding typename" ); - return Code::Invalid; - } + SkipWhitespace_Checked( parse_typedef, "Error, reached end of string before finding typename" ); GetWord(); if ( str_compare( word, "typedef", word_length ) != 0 ) @@ -3074,42 +3738,17 @@ namespace gen if ( ! helper_result ) return Code::Invalid; - type = make_code(); - type->Type = Typename; - type->Name = get_cached_string( name, name_length ); - - Code specifiers = def_specifiers( num_specifiers, specs_found ); - - type->add_entry( specifiers ); - - if ( array_expr ) - type->add_entry( array_expr ); - - type.lock(); + type = def_type( name_length, name, def_specifiers( num_specifiers, specs_found ) ); // End typename - SkipWhitespace(); + SkipWhitespace_Checked( parse_typedef, "Error, reached end of string before finding name" ); - if ( left <= 0 ) - { - log_failure( "gen::parse_typedef: Error, reached end of string before finding name" ); - return Code::Invalid; - } - word = scanner; - word_length = 0; GetWord(); - name = word; name_length = word_length; - SkipWhitespace(); - - if ( left <= 0 ) - { - log_failure( "gen::parse_typedef: Error, reached end of string before finding ';' for typedef" ); - return Code::Invalid; - } + SkipWhitespace_Checked( parse_typedef, "Error, reached end of string before finding ';'" ); if ( current == ';') { @@ -3137,9 +3776,101 @@ namespace gen Code parse_using( s32 length, char const* def ) { - not_implemented( parse_using ); + check_parse_args( parse_typedef, length, def ); + + using namespace ECode; + + SpecifierT specs_found[16] { ESpecifier::Num_Specifiers }; + s32 num_specifiers = 0; + + char const* name = nullptr; + s32 name_length = 0; + + s32 left = length; + char const* scanner = def; + + Code array_expr = { nullptr }; + Code type = { nullptr }; + + char const* word = scanner; + s32 word_length = 0; + + bool is_namespace = false; + + SkipWhitespace_Checked( parse_using, "Error, reached end of string before finding 'using'" ); + + GetWord(); + if ( str_compare( word, "using", word_length ) != 0 ) + { + log_failure( "gen::parse_using: Error, expected 'using' but found '%.*s'", word_length, word ); + return Code::Invalid; + } + + SkipWhitespace_Checked( parse_using, "Error, reached end of string before finding 'namespace' or typename" ); + + GetWord(); + if ( str_compare( word, "namespace", word_length ) == 0 ) + { + is_namespace = true; + + SkipWhitespace_Checked( parse_using, "Error, reached end of string before finding name" ); + + GetWord(); + name = word; + name_length = word_length; + + SkipWhitespace_Checked( parse_using, "Error, reached end of string before finding end statement \';\'"); + + if ( current == ';' ) + { + goto Finished; + }; + } + else + { + name = word; + name_length = word_length; + } + + move_forward(); + SkipWhitespace_Checked( parse_using, "Error, reached end of string before finding end statement ';'" ); + + if ( current == '=' ) + { + move_forward(); + SkipWhitespace_Checked( parse_using, "Error, reached end of string before finding the 'typename definition after the = sign" ); + + bool helper_result = parse__type_helper( txt(parse_using) + , left, scanner + , name_length, name + , num_specifiers, specs_found + , array_expr + ); + + if ( helper_result ) + { + type = def_type( name_length, name, def_specifiers( num_specifiers, specs_found ) ); + } + else + { + return Code::Invalid; + } + } + Finished: + + Code + result = make_code(); + result->Type = is_namespace ? Using : Using_Namespace; + result->Name = get_cached_string( name, name_length ); + + result->add_entry( type ); + + result.lock(); + return result; } + + s32 parse_classes( s32 length, char const* class_defs, Code* out_class_codes ) { not_implemented( parse_classes ); @@ -3189,6 +3920,8 @@ namespace gen { not_implemented( parse_usings ); } +// End GEN_FEATURE_PARSING +# endif # pragma endregion Parsing Constructors # pragma region Untyped Constructors @@ -3278,11 +4011,28 @@ namespace gen } #pragma endregion Builder +#pragma region File Lexer +// The Editor and Scanner require this lexer. +#if defined(GEN_FEATURE_EDITOR) || defined(GEN_FEATURE_SCANNER) +/* + This is a more robust lexer than the ones used for the lexer in the parse constructors interface. + Its needed to scan a C++ file and have awareness to skip content unsupported by the library. +*/ + +#endif +#pragma endregion File Lexer + #pragma region Editor +#ifdef GEN_FEATURE_EDITOR +#endif #pragma endregion Editor #pragma region Scanner +#ifdef GEN_FEATURE_SCANNER + + +#endif #pragma endregion Scanner } -// end gentime +// end gen_time #endif diff --git a/project/gen.hpp b/project/gen.hpp index c1eed8f..4f1c732 100644 --- a/project/gen.hpp +++ b/project/gen.hpp @@ -15,8 +15,8 @@ * Macro or template generation : This library is to avoid those, adding support for them adds unnecessary complexity. * Vendor provided dynamic dispatch (virtuals) : Roll your own, this library might roll its own vtable/interface generation helpers in the future. - * RTTI : This is kinda covered with the last point, but just wanted to emphasize. - * Exceptions : Most fo the + * RTTI + * Exceptions * Execution statement validation : Execution expressions are defined using the untyped string API. Keywords in from "Modern C++": @@ -383,6 +383,9 @@ #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS // #define GEN_DONT_USE_FATAL #define GEN_ENFORCE_READONLY_AST + +#define GEN_FEATURE_INCREMENTAL +#define GEN_FEATURE_PARSING #define GEN_FEATURE_EDITOR #define GEN_FEATURE_SCANNER @@ -439,6 +442,7 @@ namespace gen Entry( Variable ) \ Entry( Typedef ) \ Entry( Typename ) \ + Entry( Union ) \ Entry( Using ) \ Entry( Using_Namespace ) @@ -498,7 +502,7 @@ namespace gen Entry( Assgin_Divide, /= ) \ Entry( Assgin_Modulo, %= ) \ Entry( Assgin_BAnd, &= ) \ - Entry( Assgin_BOr, &= ) \ + Entry( Assgin_BOr, |= ) \ Entry( Assign_BXOr, ^= ) \ Entry( Assign_LShift, <<= ) \ Entry( Assign_RShift, >>= ) \ @@ -535,7 +539,7 @@ namespace gen enum Type : u32 { -# define Entry( Type, Token ) Type, +# define Entry( Type_, Token_ ) Type_, Define_Operators # undef Entry Comma, @@ -552,7 +556,7 @@ namespace gen local_persist char const* lookup[ Num_Ops ] = { -# define Entry( Type, Token ) txt(Token), +# define Entry( Type_, Token_ ) txt(Token_), Define_Operators # undef Entry "," @@ -570,15 +574,11 @@ namespace gen # if defined(ZPL_SYSTEM_WINDOWS) # define API_Export_Code __declspec(dllexport) # define API_Import_Code __declspec(dllimport) +# define API_Keyword __declspec # elif defined(ZPL_SYSTEM_MACOS) # define API_Export_Code __attribute__ ((visibility ("default"))) # define API_Import_Code __attribute__ ((visibility ("default"))) -# endif - -# if defined(ZPL_MODULE_THREADING) -# define Thread_Local_Code thread_local -# else -# define Thread_Local_Code "NOT DEFINED" +# define API_Keyword __attribute__ # endif # define Define_Specifiers \ @@ -587,8 +587,8 @@ namespace gen Entry( Attribute, "You cannot stringize an attribute this way" ) \ Entry( Alignas, alignas ) \ Entry( Array_Decl, "You cannot stringize an array declare this way" ) \ - Entry( Const, const ) \ Entry( C_Linkage, extern "C" ) \ + Entry( Const, const ) \ Entry( Consteval, consteval ) \ Entry( Constexpr, constexpr ) \ Entry( Constinit, constinit ) \ @@ -605,7 +605,7 @@ namespace gen Entry( Register, register ) \ Entry( RValue, && ) \ Entry( Static_Member, static ) \ - Entry( Thread_Local, Thread_Local_Code ) \ + Entry( Thread_Local, thread_local ) \ Entry( Volatile, volatile ) enum Type : u32 @@ -1100,6 +1100,7 @@ namespace gen # pragma endregion Upfront # pragma region Incremental +# ifdef GEN_FEATURE_INCREMENTAL Code make_class ( s32 length, char const* name, Code parent = NoCode, Code specifiers = NoCode ); Code make_enum ( s32 length, char const* name, Code type = NoCode, EnumT specifier = EnumRegular ); Code make_function ( s32 length, char const* name, Code params = NoCode, Code ret_type = NoCode, Code specifiers = NoCode ); @@ -1109,9 +1110,11 @@ namespace gen Code make_params (); Code make_specifiers (); Code make_struct ( s32 length, char const* name, Code parent = NoCode, Code specifiers = NoCode ); +# endif # pragma endregion Incremental # pragma region Parsing +# ifdef GEN_FEATURE_PARSING Code parse_class ( s32 length, char const* class_def ); Code parse_enum ( s32 length, char const* enum_def ); Code parse_execution ( s32 length, char const* exec_def ); @@ -1136,6 +1139,7 @@ namespace gen s32 parse_variables ( s32 length, char const* vars_def, Code* out_var_codes ); s32 parse_typedefs ( s32 length, char const* typedef_def, Code* out_typedef_codes ); s32 parse_usings ( s32 length, char const* usings_def, Code* out_using_codes ); +# endif # pragma endregion Parsing # pragma region Untyped text @@ -1364,10 +1368,13 @@ namespace gen # define operator_body( ... ) gen::def_operator_body ( macro_num_args( __VA_ARGS__ ), __VA_ARGS__ ) # define struct_body( ... ) gen::def_struct_body ( macro_num_args( __VA_ARGS__ ), __VA_ARGS__ ) +# ifdef GEN_FEATURE_INCREMENTAL // Incremental # define make( ConstructType_, Name_, ... ) Code Name_ = make_##ConstructType_( txt_n_len(Name_), __VA_ARGS__ ); +# endif +# ifdef GEN_FEATURE_PARSING // Parsing # define class_code( ... ) gen::parse_class ( txt_n_len( __VA_ARGS__ )) @@ -1381,6 +1388,7 @@ namespace gen # define type_code( ... ) gen::parse_type ( txt_n_len( __VA_ARGS__ )) # define typedef_code( ... ) gen::parse_typedef ( txt_n_len( __VA_ARGS__ )) # define using_code( ... ) gen::parse_code ( txt_n_len( __VA_ARGS__ )) +# endif // Untyped diff --git a/thirdparty/zpl.h b/thirdparty/zpl.h index b3734c9..3191528 100644 --- a/thirdparty/zpl.h +++ b/thirdparty/zpl.h @@ -35,7 +35,7 @@ GitHub: Version History: 19.0.0 - Check all results of zpl_alloc() when using JSON parser/writer (rheatley-pervasid) - + 18.1.5 - set parent to parsed JSON nodes (fixed) - fix zpl_json/csv_write_string off-by-one issue 18.1.4 - fix zpl_random_gen_isize/zpl_random_range_isize 32bit overflow @@ -51,13 +51,13 @@ Version History: 18.0.0 - removed coroutines module - removed timer module - rename zpl_adt_get -> zpl_adt_query - + 17.0.0 - ADT API changes zpl_adt_inset_* -> zpl_adt_append_* zpl_adt_node now holds a parent field, methods no longer require a pointer to the parent methods are now documented - add zpl_thread_init_nowait (gaopeng) - + 16.1.1 - fix scientific notation parsing 16.1.0 - introduce ZPL_PARSER_DISABLE_ANALYSIS that disables extra parsing capabilities to offer better raw performance at a cost of lack of node metadata. @@ -69,7 +69,7 @@ Version History: - fix memory leak when parsing a json array (gaopeng) - add zpl_strntok (gaopeng) - add zpl_semaphore_trywait (gaopeng) - + 15.0.3 - fix zpl_sign call in math failing to compile on macos devices 15.0.2 - zpl_sign0 was introduced @@ -77,7 +77,7 @@ Version History: - zpl_sign(0) returns 0 15.0.0 - Rework zpl ring buffer - various code improvements - + 14.1.7 - fix zpl_random_range_i64 - set thread's is_running before we start a thread 14.1.6 - remove windows.h dependency for header part @@ -87,7 +87,7 @@ Version History: 14.1.0 - add hashtable map_mut method 14.0.1 - fix zpl_array_remove_at boundary bug 14.0.0 - heap memory allocator analysis - + 13.4.1 - adt optimizations 13.4.0 - new adt manipulation methods 13.3.3 - fix zpl_str_skip_literal bug @@ -102,7 +102,7 @@ Version History: 13.1.1 - fix emscripten support 13.1.0 - abstract data tree naming update 13.0.0 - text parsers refactor - + 12.8.0 - zpl_opts improvements 12.7.0 - math improvements 12.6.2 - remove register usage (BeastLe9enD) @@ -119,7 +119,7 @@ Version History: 12.1.0 - Add rectangle partitioning 12.0.1 - Optimize zpl_strlen 12.0.0 - JSON API revamp + improvements - + 11.3.0 - JSON zpl_json_str_to_flt + cleanup 11.2.5 - fix small atomics typo 11.2.4 - JSON rewrite core parser @@ -141,7 +141,7 @@ Version History: 11.0.0 - New jobs system - Rewrite the timer module - zpl_ring rework - + 10.13.0 - Initial ARM threading support 10.12.1 - Fix missing zpL_alloc_str 10.12.0 - Add zpl_crc64 @@ -185,7 +185,7 @@ Version History: 10.0.4 - Flush tester output to fix ordering 10.0.3 - Fix ZPL_STATIC_ASSERT under MSVC 10.0.0 - Major overhaul of the library - + 9.8.10 - JSON fix array-based documents with objects 9.8.9 - JSON document structured as array now properly recognizes the root object as array. 9.8.8 - Fixed an incorrect parsing of empty array nodes. @@ -218,7 +218,7 @@ Version History: 9.1.0 - get_env rework and fixes 9.0.3 - Small fixes and removals 9.0.0 - New documentation format, removed deprecated code, changed styles - + 8.14.1 - Fix string library 8.14.0 - Added zpl_re_match_all 8.13.0 - Update system command API @@ -237,7 +237,7 @@ Version History: 8.10.0 - Added zpl_strchr 8.9.0 - API improvements for JSON5 parser 8.8.4 - Add support for SJSON formatting http://bitsquid.blogspot.com/2009/10/simplified-json-notation.html - + 6.8.3 - JSON5 exp fix 6.8.2 - Bugfixes applied from gb 6.8.1 - Performance improvements for JSON5 parser @@ -257,7 +257,7 @@ Version History: 6.0.2 - Fixed warnings for json5 i64 printfs 6.0.1 - Fixed warnings for particual win compiler in dirlist method 6.0.0 - New build, include/ was renamed to code/ - + 5.8.3 - Naming fixes 5.8.2 - Job system now supports prioritized tasks 5.8.1 - Renames zpl_pad to zpl_ring @@ -285,7 +285,7 @@ Version History: 5.0.2 - Fix segfault when using zpl_stack_memory 5.0.1 - Small code improvements 5.0.0 - Project structure changes - + 4.7.2 - Got rid of size arg for zpl_str_split_lines 4.7.1 - Added an example 4.7.0 - Added zpl_path_dirlist @@ -310,7 +310,7 @@ Version History: 4.0.2 - Warning fix for _LARGEFILE64_SOURCE 4.0.1 - include stdlib.h for getenv (temp) 4.0.0 - ARM support, coding style changes and various improvements - + 3.4.1 - zpl_memcopy now uses memcpy for ARM arch-family 3.4.0 - Removed obsolete code 3.3.4 - Added Travis CI config @@ -331,7 +331,7 @@ Version History: 3.0.2 - Fixed linux part, and removed trailing spaces 3.0.1 - Small bugfix in zpl_file_open 3.0.0 - Added several fixes and features - + 2.4.0 - Added remove to hash table 2.3.3 - Removed redundant code 2.3.2 - Eliminated extra warnings @@ -344,7 +344,7 @@ Version History: 2.0.8 - Small adjustments 2.0.7 - MinGW related fixes 2.0.0 - New NPM based version - + 1.2.2 - Small fix 1.2.1 - Macro fixes 1.2.0 - Added zpl_async macro @@ -354,15 +354,15 @@ Version History: License: This Software is dual licensed under the following licenses: - + Unlicense This is free and unencumbered software released into the public domain. - + Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. - + In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit @@ -370,7 +370,7 @@ License: successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. - + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. @@ -378,15 +378,15 @@ License: OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - + For more information, please refer to - + BSD 3-Clause Copyright (c) 2016-2021 Dominik Madarász. All rights reserved. - + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - + 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, @@ -395,7 +395,7 @@ License: 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE @@ -8367,7 +8367,9 @@ ZPL_END_NAMESPACE # endif # if ! defined( thread_local ) -# if defined( _MSC_VER ) && _MSC_VER >= 1300 +# if defined( __cplusplus ) && __cplusplus >= 201103L +# define thread_local thread_local +# elif defined( _MSC_VER ) && _MSC_VER >= 1300 # define thread_local __declspec( thread ) # elif defined( __GNUC__ ) # define thread_local __thread @@ -8803,7 +8805,10 @@ ZPL_END_NAMESPACE # endif # else # if ! defined( thread_local ) -# define thread_local +# if defined( __cplusplus ) && __cplusplus >= 201103L +# define thread_local thread_local +# else +# define thread_local # endif # endif