diff --git a/codegen/gen_src.cpp b/codegen/gen_src.cpp index 7c4062bf9..69c550f8f 100644 --- a/codegen/gen_src.cpp +++ b/codegen/gen_src.cpp @@ -5,9 +5,9 @@ #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS #define GEN_ENFORCE_STRING_CODE_TYPES #define GEN_EXPOSE_BACKEND -#include "gencpp/gen.cpp" -#include "gencpp/gen.builder.cpp" -#include "gencpp/gen.scanner.cpp" +#define GEN_IMPLEMENTATION +// #define GEN_DONT_USE_FATAL +#include "gencpp/gen.hpp" using namespace gen; #ifdef GEN_SYSTEM_WINDOWS @@ -24,18 +24,11 @@ using namespace gen; #pragma endregion Directories -inline -CodeBody parse_file( char const* path ) { - FileContents content = file_read_contents( GlobalAllocator, true, path ); - CodeBody code = parse_global_body( StrC { content.size, (char const*)content.data }); - return code; -} - inline void git_restore_file( char const* path ) { #define git_restore_cmd "git restore " - String command = String::make( GlobalAllocator, git_restore_cmd ); + StrBuilder command = StrBuilder::make( GlobalAllocator, git_restore_cmd ); command.append( path ); log_fmt("Running git restore on: %s", path); system(command); @@ -51,7 +44,7 @@ void format_file( char const* path ) #define cf_format_inplace "-i " #define cf_style "-style=file:" "./scripts/.clang-format " #define cf_verbose "-verbose " - String command = String::make( GlobalAllocator, clang_format ); + StrBuilder command = StrBuilder::make( GlobalAllocator, clang_format ); command.append( cf_format_inplace ); command.append( cf_style ); command.append( cf_verbose ); @@ -80,33 +73,30 @@ Array get_odin_ast_kinds() ++ done_once; } - CodeType t_char_const_ptr = parse_type(code(char const*)); + CodeTypename t_char_const_ptr = parse_type(code(char const*)); CodeBody ast_kinds_header = parse_file( path_codegen "ast_kinds.hpp" ); for ( Code code = ast_kinds_header.begin(); code != ast_kinds_header.end(); ++ code ) { switch (code->Type) { - using namespace ECode; - case Comment: - case Preprocess_Pragma: + case CT_Comment: + case CT_Preprocess_Pragma: // Ignore continue; - case Variable: + case CT_Variable: { - Odin_AstKind entry { {nullptr}, {} }; + Odin_AstKind entry {}; - CodeVar var = code.cast(); + CodeVar var = cast(CodeVar, code); if ( ! var->ValueType.is_equal( t_char_const_ptr ) ) { - __debugbreak(); log_failure("Expected all globally defined variables to be char cons* type"); return kinds; } if ( ! var->Value || ! var->Value->Content ) { - __debugbreak(); log_failure("Expected all globally defined variable to have a string assigned to it"); return kinds; } @@ -116,9 +106,8 @@ Array get_odin_ast_kinds() ++ code; // Grab the definition - if ( code->Type != Struct && code->Type != Typedef ) + if ( code->Type != CT_Struct && code->Type != CT_Typedef ) { - __debugbreak(); log_failure("Expected a struct or typedef for the entry definition"); return kinds; } @@ -128,8 +117,7 @@ Array get_odin_ast_kinds() } continue; - case Struct: - __debugbreak(); + case CT_Struct: log_failure("Expected a description definition as char const* first"); return kinds; break; @@ -151,13 +139,13 @@ Array get_odin_type_kinds() CodeBody ast_types_header = parse_file( path_codegen "type_kinds.hpp" ); for ( Code code = ast_types_header.begin(); code != ast_types_header.end(); ++ code ) switch (code->Type) { - case ECode::Comment: - case ECode::Preprocess_Pragma: + case CT_Comment: + case CT_Preprocess_Pragma: // Ignore continue; - case ECode::Typedef: - case ECode::Struct: + case CT_Typedef: + case CT_Struct: { types.append(code); } @@ -170,7 +158,7 @@ int gen_main() gen::init(); log_fmt("Generating code for Odin's src\n"); - StrC str_GB_STATIC_ASSERT = txt("GB_STATIC_ASSERT("); + Str str_GB_STATIC_ASSERT = txt("GB_STATIC_ASSERT("); PreprocessorDefines.append( get_cached_string(str_GB_STATIC_ASSERT) ); // Remove TOKEN_KINDS usage in tokenizer.cpp @@ -181,61 +169,64 @@ int gen_main() { char scratch_mem[kilobytes(32)]; Arena scratch = Arena::init_from_memory( scratch_mem, sizeof(scratch_mem) ); - file_read_contents( scratch, zero_terminate, path_codegen "token_kinds.csv" ); + file_read_contents( scratch, file_zero_terminate, path_codegen "token_kinds.csv" ); csv_parse( &csv_nodes, scratch_mem, GlobalAllocator, false ); } Array enum_strs = csv_nodes.nodes[0].nodes; Array str_strs = csv_nodes.nodes[1].nodes; - String enum_entries = String::make_reserve( GlobalAllocator, kilobytes(32) ); - String to_str_entries = String::make_reserve( GlobalAllocator, kilobytes(32) ); + StrBuilder enum_entries = StrBuilder::make_reserve( GlobalAllocator, kilobytes(32) ); + StrBuilder to_str_entries = StrBuilder::make_reserve( GlobalAllocator, kilobytes(32) ); to_str_entries.append(txt("{")); - for (uw idx = 0; idx < enum_strs.num(); idx++) + for (usize idx = 0; idx < enum_strs.num(); idx++) { char const* enum_str = enum_strs[idx].string; - StrC entry_to_str = to_str(str_strs [idx].string); + Str entry_to_str = to_str_from_c_str(str_strs [idx].string); + #pragma push_macro("cast") + #undef cast enum_entries.append_fmt( "Token_%s,\n", enum_str ); - to_str_entries.append( token_fmt( "str", (StrC)entry_to_str, stringize( + to_str_entries.append( token_fmt( "str", (Str)entry_to_str, stringize( { cast(u8 *) "", gb_size_of("") -1 },\n ))); + #pragma pop_macro("cast") } to_str_entries.append(txt("}")); char const* path_tokenizer = path_src "tokenizer.cpp"; git_restore_file( path_tokenizer ); CodeBody src_tokenizer_cpp = parse_file( path_src "tokenizer.cpp" ); - CodeBody body = def_body( ECode::Global_Body ); + CodeBody body = def_body( CT_Global_Body ); body.append( def_comment(txt("NOTICE(github: Ed94): This is a generated variant of tokenizer.cpp using /codegen/gen_src.cpp"))); body.append(fmt_newline); for (Code code = src_tokenizer_cpp.begin(); code != src_tokenizer_cpp.end(); ++ code) switch (code->Type) { - case ECode::Preprocess_Define: + case CT_Preprocess_Define: if ( code->Name.starts_with( txt("TOKEN_KINDS"))) { // Skip, we don't want it. continue; } continue; - case ECode::Enum: + case CT_Enum: { if ( code->Name.starts_with(txt("TokenKind"))) { - CodeEnum enum_code = code.cast(); + CodeEnum enum_code = cast(CodeEnum, code); enum_code->Body = untyped_str(enum_entries); } body.append(code); } continue; - case ECode::Variable: + case CT_Variable: if ( code->Name.starts_with(txt("token_strings"))) { - CodeVar var = code.cast(); + CodeVar var = cast(CodeVar, code); var->Value = untyped_str(to_str_entries); } body.append(code); @@ -259,7 +250,7 @@ int gen_main() char const* path_parser = path_src "parser.hpp"; git_restore_file( path_parser ); CodeBody src_parser_header = parse_file( path_src "parser.hpp" ); - CodeBody body = def_body( ECode::Global_Body ); + CodeBody body = def_body( CT_Global_Body ); body.append( def_comment(txt("NOTICE(github: Ed94): This is a generated variant of parser.hpp using /codegen/gen_src.cpp"))); body.append(fmt_newline); @@ -270,7 +261,7 @@ int gen_main() { switch (code->Type) { - case ECode::Preprocess_Define: + case CT_Preprocess_Define: if ( code->Name.starts_with( txt("AST_KINDS"))) { // Skip, we don't want it. continue; @@ -284,26 +275,26 @@ int gen_main() body.append(code); continue; - case ECode::Untyped: + case CT_Untyped: if (code->Content.starts_with(txt("AST_KINDS"))) continue; body.append(code); continue; - case ECode::Enum: + case CT_Enum: { if (code->Name.starts_with( txt("AstKind"))) { // Swap with generated variant - CodeBody swap_body = def_body( ECode::Enum_Body ); + CodeBody swap_body = def_body( CT_Enum_Body ); { swap_body.append( code_str(Ast_Invalid,)); for (Odin_AstKind& kind : ast_kinds) - swap_body.append( untyped_str( String::fmt_buf(GlobalAllocator, "Ast_%S,", kind.def->Name ))); + swap_body.append( untyped_str( StrBuilder::fmt_buf(GlobalAllocator, "Ast_%S,", kind.def->Name ))); swap_body.append( code_str(Ast_COUNT)); } - CodeEnum swapped_enum = code.cast().duplicate(); + CodeEnum swapped_enum = cast(CodeEnum, code).duplicate(); swapped_enum->Body = swap_body; body.append(swapped_enum); } @@ -312,24 +303,32 @@ int gen_main() } continue; - case ECode::Variable: + case CT_Variable: { if (code->Name.starts_with(txt("ast_strings"))) { // Swap with generated table - String generated_table = String::make_reserve(GlobalAllocator, kilobytes(32)); + StrBuilder generated_table = StrBuilder::make_reserve(GlobalAllocator, kilobytes(32)); { + #pragma push_macro("cast") + #undef cast for (Odin_AstKind& kind : ast_kinds) - generated_table.append(token_fmt("desc", (StrC)kind.desc, stringize( + generated_table.append(token_fmt("desc", (Str)kind.desc, stringize( { cast(u8 *) , gb_size_of() -1 }, ))); + #pragma pop_macro("cast") } - CodeVar swapped_table = code.cast().duplicate(); - swapped_table->Value = code_fmt( "kinds", (StrC)generated_table, stringize( + + CodeVar swapped_table = cast(CodeVar, code).duplicate(); + #pragma push_macro("cast") + #undef cast + swapped_table->Value = code_fmt( "kinds", (Str)generated_table, stringize( { - {cast(u8 *)"invalid node", gb_size_of("invalid node")},\n + { cast(u8 *)"invalid node", gb_size_of("invalid node")},\n })); + #pragma pop_macro("cast") + body.append(swapped_table); body.append(fmt_newline); @@ -339,7 +338,7 @@ int gen_main() for (Odin_AstKind& kind : ast_kinds) { Code def = kind.def.duplicate(); - def->Name = get_cached_string( String::fmt_buf(GlobalAllocator, "Ast%S", kind.def->Name)); + def->Name = get_cached_string( StrBuilder::fmt_buf(GlobalAllocator, "Ast%S", kind.def->Name)); body.append( def ); body.append(fmt_newline); } @@ -349,15 +348,15 @@ int gen_main() if (code->Name.starts_with(txt("ast_variant_sizes"))) { // Swap with generated table - String generated_table = String::make_reserve(GlobalAllocator, kilobytes(32)); + StrBuilder generated_table = StrBuilder::make_reserve(GlobalAllocator, kilobytes(32)); { for (Odin_AstKind& kind : ast_kinds) - generated_table.append(token_fmt( "name", (StrC)kind.def->Name, stringize( + generated_table.append(token_fmt( "name", (Str)kind.def->Name, stringize( gb_size_of(Ast),\n ))); } - CodeVar swapped_table = code.cast().duplicate(); - swapped_table->Value = code_fmt( "kinds", (StrC)generated_table, stringize( + CodeVar swapped_table = cast(CodeVar, code).duplicate(); + swapped_table->Value = code_fmt( "kinds", (Str)generated_table, stringize( { 0,\n @@ -369,21 +368,21 @@ int gen_main() } continue; - case ECode::Struct: + case CT_Struct: { - CodeStruct code_struct = code.cast(); + CodeStruct code_struct = cast(CodeStruct, code); if (code->Name.starts_with(txt("Ast"))) for (Code ast_code : code_struct->Body) switch (ast_code->Type) { - case ECode::Union: + case CT_Union: { // Swap out the union's contents with the generated member definitions - CodeBody body_swap = def_body(ECode::Union_Body); + CodeBody body_swap = def_body(CT_Union_Body); for (Odin_AstKind kind : ast_kinds) - body_swap.append( parse_variable( token_fmt( "name", (StrC)kind.def->Name, stringize( + body_swap.append( parse_variable( token_fmt( "name", (Str)kind.def->Name, stringize( Ast ; )))); - ast_code->Body = rcast(AST*, body_swap.ast); + ast_code->Body = body_swap; } break; default: @@ -412,7 +411,7 @@ int gen_main() char const* path_types = path_src "types.cpp"; git_restore_file( path_types ); CodeBody src_types_cpp = parse_file( path_src "types.cpp" ); - CodeBody body = def_body( ECode::Global_Body ); + CodeBody body = def_body( CT_Global_Body ); body.append( def_comment(txt("NOTICE(github: Ed94): This is a generated variant of types.cpp using /codegen/gen_src.cpp"))); body.append(fmt_newline); @@ -421,7 +420,7 @@ int gen_main() for (Code code = src_types_cpp.begin(); code != src_types_cpp.end(); ++ code) switch (code->Type) { - case ECode::Preprocess_Define: + case CT_Preprocess_Define: { if ( code->Name.starts_with( txt("TYPE_KINDS"))) { // Skip, we don't want it. @@ -437,19 +436,19 @@ int gen_main() } continue; - case ECode::Enum: + case CT_Enum: { if ( code->Name.starts_with( txt("TypeKind"))) { - CodeBody swap_body = def_body( ECode::Enum_Body); + CodeBody swap_body = def_body( CT_Enum_Body); { swap_body.append( code_str(Type_Invalid, )); { for (Code type : type_kinds) - swap_body.append( untyped_str( String::fmt_buf(GlobalAllocator, "Type_%S,", type->Name ))); + swap_body.append( untyped_str( StrBuilder::fmt_buf(GlobalAllocator, "Type_%S,", type->Name ))); swap_body.append( code_str(Type_COUNT)); } - CodeEnum swapped_enum = code.cast().duplicate(); + CodeEnum swapped_enum = cast(CodeEnum, code).duplicate(); swapped_enum->Body = swap_body; body.append(swapped_enum); } @@ -459,24 +458,27 @@ int gen_main() } continue; - case ECode::Variable: + case CT_Variable: { if (code->Name.starts_with(txt("type_strings"))) { + #pragma push_macro("cast") + #undef cast // Swap with generated table - String generated_table = String::make_reserve(GlobalAllocator, kilobytes(32)); + StrBuilder generated_table = StrBuilder::make_reserve(GlobalAllocator, kilobytes(32)); { for (Code type : type_kinds) - generated_table.append(token_fmt("type", (StrC)type->Name, stringize( + generated_table.append(token_fmt("type", (Str)type->Name, stringize( { cast(u8 *) "", gb_size_of("") -1 }, ))); } - CodeVar swapped_table = code.cast().duplicate(); - swapped_table->Value = code_fmt( "types", (StrC)generated_table, stringize( + CodeVar swapped_table = ((CodeVar)code).duplicate(); + swapped_table->Value = code_fmt( "types", (Str)generated_table, stringize( { - {cast(u8 *)"invalid node", gb_size_of("invalid node")},\n + { cast(u8 *)"invalid node", gb_size_of("invalid node")},\n })); + #pragma pop_macro("cast") body.append(swapped_table); body.append(fmt_newline); @@ -486,7 +488,7 @@ int gen_main() for (Code type : type_kinds) { Code def = type.duplicate(); - def->Name = get_cached_string( String::fmt_buf(GlobalAllocator, "Type%S", type->Name)); + def->Name = get_cached_string( StrBuilder::fmt_buf(GlobalAllocator, "Type%S", type->Name)); body.append( def ); body.append(fmt_newline); } @@ -497,21 +499,21 @@ int gen_main() } continue; - case ECode::Struct: + case CT_Struct: { - CodeStruct code_struct = code.cast(); - if ( String::are_equal(code->Name, txt("Type"))) + CodeStruct code_struct = cast(CodeStruct, code); + if ( str_are_equal(code->Name, txt("Type"))) for (Code type_code : code_struct->Body) switch (type_code->Type) { - case ECode::Union: + case CT_Union: { // Swap out the union's contents with the generated member definitions - CodeBody body_swap = def_body(ECode::Union_Body); + CodeBody body_swap = def_body(CT_Union_Body); for (Code type : type_kinds) - body_swap.append( parse_variable( token_fmt( "name", (StrC)type->Name, stringize( + body_swap.append( parse_variable( token_fmt( "name", (Str)type->Name, stringize( Type ; )))); - type_code->Body = rcast(AST*, body_swap.ast); + type_code->Body = body_swap; } break; default: diff --git a/codegen/gencpp/AST_Design.md b/codegen/gencpp/AST_Design.md new file mode 100644 index 000000000..cff0b1401 --- /dev/null +++ b/codegen/gencpp/AST_Design.md @@ -0,0 +1,58 @@ +## Navigation + +[Top](../Readme.md) + +<- [docs - General](Readme.md) + +## Current Design + +`AST` is the actual managed node object for the library. +Its raw and really not meant to be used directly. + +All user interaction must be with its pointer so the type they deal with is `AST*`. +In order to abstract away constant use of `AST*` its wrapped in a Code type which can be either: + +When its the [C generated variant of the library](../gen_c_library/) +```c +typedef AST* Code; +tyepdef AST_* Code; +... +``` + +**or** + +For C++: +```cpp +struct Code { + AST* ast; +}; +struct Code { + ... + + AST_* ast; +}; +``` + +The full definitions of all asts are within: + +* [`ast.hpp`](../base/components/ast.hpp) +* [`ast_types.hpp`](../base/components/ast_types.hpp) +* [`code_types.hpp`](../base/components/code_types.hpp) + +The C/C++ interface procedures are located with `ast.hpp` (for the Code type), and `code_types.hpp` for all others. + +## Serialization + +All code types can either serialize using a function of the pattern: + +```c +StrBuilder _to_string(Code code); +// or +_to_string(Code code, StrBuilder& result); +``` + +Where the first generates strings allocated using Allocator_StringArena and the other appends an existing strings with their backed allocator. + +Serialization of for the AST is defined for `Code` in [`ast.chpp`](../base/components/ast.cpp) with `code_to_strbuilder_ptr` & `code_to_string`. +Serializtion for the rest of the code types is within [`code_serialization.cpp`](../base/components/code_serialization.cpp). +Gencpp's serialization does not provide coherent formatting of the code. The user should use a formatter after serializing. diff --git a/codegen/gencpp/AST_Types.md b/codegen/gencpp/AST_Types.md new file mode 100644 index 000000000..ccc35c7a8 --- /dev/null +++ b/codegen/gencpp/AST_Types.md @@ -0,0 +1,765 @@ +## Navigation + +[Top](../Readme.md) + +<- [docs - General](Readme.md) + +# AST Types Documentation + +While the Readme for docs covers the data layout per AST, this will focus on the AST types avaialble, and their nuances. + +## Body + +These are containers representing a scope body of a definition that can be of the following `CodeType` type: + +* Class_Body +* Enum_Body +* Export_Body +* Extern_Linkage_Body +* Function_Body +* Global_Body +* Namespace_Body +* Struct_Body +* Union_Body + +Fields: + +```cpp +StringCached Name; +Code Front; +Code Back; +Token* Tok; +Code Parent; +CodeT Type; +s32 NumEntries; +``` + +The `Front` member represents the start of the link list and `Back` the end. +NumEntries is the number of entries in the body. + +Parent should have a compatible CodeType type for the type of defintion used. + +Serialization: + +Will output only the entries, the braces are handled by the parent. + +```cpp + +... + +``` + +## Attributes + +Represent standard or vendor specific C/C++ attributes. + +Fields: + +```cpp +StringCached Content; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp + +``` + +While the parser supports the `__declspec` and `__attribute__` syntax, the upfront constructor ( def_attributes ) must have the user specify the entire attribute, including the `[[]]`, `__declspec` or `__attribute__` parts. + +## Comment + +Stores a comment. + +Fields: + +```cpp +StringCached Content; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp + +``` + +The parser will perserve comments found if residing with a body or in accepted inline-to-definition locations. +Otherwise they will be skipped by the TokArray::__eat and TokArray::current( skip foramtting enabled ) functions. + +The upfront constructor: `def_comment` expects to recieve a comment without the `//` or `/* */` parts. It will add them during construction. + +## Class & Struct + +Fields: + +```cpp +CodeComment InlineCmt; // Only supported by forward declarations +CodeAttributes Attributes; +CodeType ParentType; +CodeBody Body; +StringCached Name; +CodeType Prev; +CodeType Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +AccessSpec ParentAccess; +``` + +Serialization: + +```cpp +// Class_Fwd + ; + +// Class + : , public Next>, ... +{ + +}; +``` + +You'll notice that only one parent type is supported only with parent access. This library only supports single inheritance, the rest are assumed to be interfaces and are given public acess specifiers. + +## Constructor + +Fields: + +```cpp +CodeComment InlineCmt; // Only supported by forward declarations +Code InitializerList; +CodeParams Params; +Code Body; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +// Constructor_Fwd + Name>( ); + +// Constructor + Name>( ) + : +{ + +} + +// Constructor Source Implementation + ::~Name>( ) +{ + +} +``` + +## Define + +Represents a preprocessor define + +Fields: + +```cpp +StringCached Content; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +#define +``` + +## Destructor + +Fields: + +```cpp +CodeComment InlineCmt; +CodeSpecifiers Specs; +Code Body; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +// Destructor_Fwd + ~Name>( ) ; + +// Destructor + ~Name>( ) +{ + +} + +// Destructor Source Implementation + ::~Name>( ) +{ + +} +``` + +## Enum + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeType UnderlyingType; +Code UnderlyingTypeMacro; +CodeBody Body; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +UnderlyingTypeMacro is a macro the library natively supports: `enum_underlying(type)` that is meant to behave as a wrapper for underlying type assignment. +The `enum_underlying_sig` is a `Str` global var that can be set which will be defined within `PreprocessorDefines` and used in `parser_parse_enum` to identify a valid macro. + +Serialization: + +```cpp +// Enum_Fwd + enum class : or ; + +// Enum + : or +{ + +}; +``` + +## Execution + +Just represents an execution body. Equivalent to an untyped body. +Will be obsolute when function body parsing is implemented. + +Fields: + +```cpp +StringCached Content; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp + +``` + +## External Linkage + +Fields: + +```cpp +CodeBody Body; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +extern "" +{ + +} +``` + +## Include + +Fields: + +```cpp +StringCached Content; +StringCached Name; +Code Prev; +Code Next; +Code Parent; +Token* Tok; +CodeT Type; +``` + +Serialization: + +```cpp +#include +``` + +## Friend + +This library (until its necessary become some third-party library to do otherwise) does not support friend declarations with in-statment function definitions. + +Fields: + +```cpp +CodeComment InlineCmt; +Code Declaration; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +friend ; +``` + +## Function + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeSpecifiers Specs; +CodeType ReturnType; +CodeParams Params; +CodeBody Body; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp +// Function_Fwd + ( ) ; + +// Function + ( ) +{ + +} +``` + +## Module + +Fields: + +```cpp +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp + module ; +``` + +## Namespace + +Fields: + +```cpp +CodeBody Body; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp + namespace +{ + +} +``` + +## Operator Overload (Operator) + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeSpecifiers Specs; +CodeType ReturnType; +CodeParams Params; +CodeBody Body; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +OperatorT Op; +``` + +Serialization: + +```cpp +// Operator_Fwd + operator ( ) ; + +// Operator + operator ( ) +{ + +} +``` + +## Operator Cast Overload ( User-Defined Type Conversion, OpCast ) + +Fields: + +```cpp +CodeComment InlineCmt; +CodeSpecifiers Specs; +CodeType ValueType; +CodeBody Body; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +// Operator_Cast_Fwd + operator () ; + +// Operator_Cast + operator () +{ + +} +``` + +## Parameters (AST_Params) + +Fields: + +```cpp +CodeType ValueType; +Code Macro; +Code Value; +Code PostNameMacro; +StringCached Name; +CodeParams Last; +CodeParams Next; +Token* Tok; +Code Parent; +CodeT Type; +s32 NumEntries; +``` + +Serialization: + +```cpp +, ... + + = , ... +``` + +## Pragma + +Fields: + +```cpp +StringCached Content; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +#pragma +``` + +## Preprocessor Conditional + +Fields: + +```cpp +StringCached Content; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +``` + +Serialization: + +```cpp +# +``` + +## Specifiers + +Fields: + +```cpp +SpecifierT ArrSpecs[ AST_ArrSpecs_Cap ]; +CodeSpecifiers NextSpecs; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +s32 NumEntries; +``` + +Serialization: + +```cpp +, ... +``` + +## Template + +Fields: + +```cpp +CodeParams Params; +Code Declaration; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp + +template< > + +``` + +## Typename + +Typenames represent the type "symbol". + +Fields: + +```cpp +CodeAttributes Attributes; +CodeSpecifiers Specs; +CodeReturnType ReturnType; +CodeParams Params; +Code ArrExpr; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +StringCached Name; +CodeT Type; +b32 IsParamPack; +ETypenameTag TypeTag; +``` + +Serialization: + +```cpp + +// Function + +``` + +`` currently has the full serialization of anything with + +*Note: ArrExpr is not used in serialization by `typename_to_strbuilder_ref` its instead handled by a parent AST's serailization (variable, typedef, using).* + +## Typedef + +Behave as usual except function or macro typedefs. +Those (macros) don't use the underlying type field as everything was serialized under the Name field. + +Fields: + +```cpp +CodeComment InlineCmt; +Code UnderlyingType; +StringCached Name; +Code Prev; +Code Next; +Token* Tok +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +b32 IsFunction; +``` + +Serialization: + +```cpp +// Regular + typedef ; + +// Functions + +// Currently: + typedef ; + +// Desired: Not handled yet + typedef ReturnType> UnderlyingType->Name> ( Parameters> ); + typedef ReturnType> ( Namespace> forhas(Spec_Ptr) ?: *> Name> ) ( Parameters> ); +``` + +## Union + +Fields: + +```cpp +CodeAttributes Attributes; +CodeBody Body; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp + union +{ + +} +``` + +## Using + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeType UnderlyingType; +StringCached Name; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +``` + +Serialization: + +```cpp +// Regular + using = ; + +// Namespace + using namespace ; +``` + +## Variable + +[Algo](./Parser_Algo.md:) + +Fields: + +```cpp +CodeComment InlineCmt; +CodeAttributes Attributes; +CodeSpecifiers Specs; +CodeType ValueType; +Code BitfieldSize; +Code Value; +StringCached Name; +CodeVar NextVar; +Code Prev; +Code Next; +Token* Tok; +Code Parent; +CodeT Type; +ModuleFlag ModuleFlags; +s32 VarParenthesizedInit; +``` + +Serialization: + +```cpp +// Regular + = , NextVar ...; + +// Bitfield + : = , NextVar ...; + +// VarParenthesizedInit + ( , NextVar ... ); +``` diff --git a/codegen/gencpp/Parser_Algo.md b/codegen/gencpp/Parser_Algo.md new file mode 100644 index 000000000..81801682d --- /dev/null +++ b/codegen/gencpp/Parser_Algo.md @@ -0,0 +1,708 @@ +## Navigation + +[Top](../Readme.md) + +<- [docs - General](Readme.md) + +# Parser's Algorithim + +gencpp uses a hand-written recursive descent parser. Both the lexer and parser currently handle a full C/C++ file in a single pass. + +## Notable implementation background + +### Lexer + +The lex procedure does the lexical pass of content provided as a `Str` type. +The tokens are stored (for now) in `Lexer_Tokens`. + +Fields: + +```cpp +Array Arr; +s32 Idx; +``` + +What token types are supported can be found in [ETokType.csv](../base/enums/ETokType.csv) you can also find the token types in [ETokType.h](../base/components/gen/etoktype.cpp) , which is the generated enum from the csv file. + +Tokens are defined with the struct `gen::parser::Token`: + +Fields: + +```cpp +char const* Text; +sptr Length; +TokType Type; +s32 Line; +s32 Column; +u32 Flags; +``` + +Flags is a bitfield made up of TokFlags (Token Flags): + +* `TF_Operator` : Any operator token used in expressions +* `TF_Assign` + * Using statment assignment + * Parameter argument default value assignment + * Variable declaration initialization assignment +* `TF_Preprocess` : Related to a preprocessing directive +* `TF_Preprocess_Cond` : A preprocess conditional +* `TF_Attribute` : An attribute token +* `TF_AccessSpecifier` : An accesor operation token +* `TF_Specifier` : One of the specifier tokens +* `TF_EndDefinition` : Can be interpreted as an end definition for a scope. +* `TF_Formatting` : Considered a part of the formatting +* `TF_Literal` : Anything considered a literal by C++. + +I plan to replace IsAssign with a general flags field and properly keep track of all operator types instead of abstracting it away to `ETokType::Operator`. + +Traversing the tokens is done with the following interface macros: + +| Macro | Description | +| --- | --- | +| `currtok_noskip` | Get the current token without skipping whitespace | +| `currtok` | Get the current token, skip any whitespace tokens | +| `prevtok` | Get the previous token (does not skip whitespace) | +| `nexttok` | Get the next token (does not skip whitespace) | +| `eat( Token Type )` | Check to see if the current token is of the given type, if so, advance Token's index to the next token | +| `left` | Get the number of tokens left in the token array | +| `check_noskip` | Check to see if the current token is of the given type, without skipping whitespace | +| `check` | Check to see if the current token is of the given type, skip any whitespace tokens | + +### Parser + +The parser has a limited user interface, only specific types of definitions or statements are expected to be provided by the user directly when using to construct an AST dynamically (See SOA for example). It however does attempt to provide capability to parse a full C/C++ from production codebases. + +Each public user interface procedure has the following format: + +```cpp + parse_( Str def ) +{ + check_parse_args( def ); + using namespace Parser; + + TokArray toks = lex( def ); + if ( toks.Arr == nullptr ) + return CodeInvalid; + + // Parse the tokens and return a constructed AST using internal procedures + ... +} +``` + +The most top-level parsing procedure used for C/C++ file parsing is `parse_global_body`: + +It uses a helper procedure called `parse_global_nspace`. + +Each internal procedure will have the following format: + +```cpp +internal + parse_( ) +{ + push_scope(); + + ... + + result = () make_code(); + ... + + Context.pop(); + return result; +} +``` + +Below is an outline of the general alogirithim used for these internal procedures. The intention is to provide a basic briefing to aid the user in traversing the actual code definitions. These appear in the same order as they are in the `parser.cpp` file + +***NOTE: This is still heavily in an alpha state. A large swaph of this can change, make sure these docs are up to date before considering them 1:1 with the repo commit your considering.*** + +## `parse_array_decl` + +1. Check if its an array declaration with no expression. + 1. Consume and return empty array declaration +2. Opening square bracket +3. Consume expression +4. Closing square bracket +5. If adjacent opening bracket + 1. Repeat array declaration parse until no brackets remain + +## `parse_assignment_expression` + +1. Eat the assignment operator +2. Make sure there is content or at least an end statement after. +3. Flatten the assignment expression to an untyped Code string. + +## `parse_attributes` + +1. Check for standard attribute +2. Check for GNU attribute +3. Check for MSVC attribute +4. Check for a token registered as an attribute + a. Check and grab the arguments of a token registered of an attribute if it has any. +5. Repeat for chained attributes. Flatten them to a single attribute AST node. + +## `parse_class_struct` + +1. Check for export module specifier +2. class or struct keyword +3. `parse_attributes` +4. If identifier : `parse_identifier` +5. Parse inherited parent or interfaces +6. If opening curly brace : `parse_class_struct_body` +7. If not an inplace definition + 1. End statement + 2. Check for inline comment + +## `parse_class_struct_body` + +1. Opening curly brace +2. Parse the body (Possible options): + 1. Ignore dangling end statements + 2. Newline : ast constant + 3. Comment : `parse_comment` + 4. Access_Public : ast constant + 5. Access_Protected : ast constant + 6. Access_Private : ast constant + 7. Decl_Class : `parse_complicated_definition` + 8. Decl_Enum : `parse_complicated_definition` + 9. Decl_Friend : `parse_friend` + 10. Decl_Operator : `parse_operator_cast` + 11. Decl_Struct : `parse_complicated_definition` + 12. Decl_Template : `parse_template` + 13. Decl_Typedef : `parse_typedef` + 14. Decl_Union : `parse_complicated_definition` + 15. Decl_Using : `parse_using` + 16. Operator == '~' + 1. `parse_destructor` + 17. Preprocess_Define : `parse_define` + 18. Preprocess_Include : `parse_include` + 19. Preprocess_Conditional (if, ifdef, ifndef, elif, else, endif) : `parse_preprocess_cond` or else/endif ast constant + 20. Preprocess_Macro : `parse_simple_preprocess` + 21. Preprocess_Pragma : `parse_pragma` + 22. Preprocess_Unsupported : `parse_simple_preprocess` + 23. StaticAssert : `parse_static_assert` + 24. The following compound into a resolved definition or declaration: + 1. Attributes (Standard, GNU, MSVC) : `parse_attributes` + 2. Specifiers (consteval, constexpr, constinit, explicit, forceinline, inline, mutable, neverinline, static, volatile, virtual) + 3. Possible Destructor : `parse_destructor` + 4. Possible User defined operator cast : `parse_operator_cast` + 5. Possible Constructor : `parse_constructor` + 6. Something that has the following: (identifier, const, unsigned, signed, short, long, bool, char, int, double) + 1. Possible Constructor `parse_constructor` + 2. Possible Operator, Function, or varaible : `parse_operator_function_or_variable` + 25. Something completely unknown (will just make untyped...) : `parse_untyped` + +## `parse_comment` + +1. Just wrap the token into a cached string ( the lexer did the processing ) + +## `parse_compilcated_definition` + +This is a helper function used by the following functions to help resolve a declaration or definition: + +* `parse_class_struct_body` +* `parse_global_nspace` +* `parse_union` + +A portion of the code in `parse_typedef` is very similar to this as both have to resolve a similar issue. + +1. Look ahead to the termination token (End statement) +2. Check to see if it fits the pattern for a forward declare +3. If the previous token was an identifier ( `token[-1]` ): + 1. Look back one more token : `[-2]` + 2. If the token has a closing brace its an inplace definition + 3. If the `token[-2]` is an identifier & `token[-3]` is the declaration type, its a variable using a namespaced type. + 4. If the `token[-2]` is an indirection, then its a variable using a namespaced/forwarded type. + 5. If the `token[-2]` is an assign classifier, and the starting tokens were the which type with possible `class` token after, its an enum forward declaration. + 6. If any of the above is the case, `parse_operator_function_or_variable` +4. If the `token[2]` is a vendor fundamental type (builtin) then it is an enum forward declaration. +5. If the previous token was a closing curly brace, its a definition : `parse_forward_or_definition` +6. If the previous token was a closing square brace, its an array definition : `parse_operator_function_or_variable` + +## `parse_define` + +1. Define directive +2. Get identifier +3. Get Content (Optional) + +## `parse_forward_or_definition` + +* Parse any of the following for either a forward declaration or definition: + 1. Decl_Class : `parse_class` + 2. Decl_Enum : `parse_enum` + 3. Decl_Struct : `parse_struct` + 4. Decl_Union : `parse_union` + +## `parse_function_after_name` + +This is needed as a function defintion is not easily resolvable early on, as such this function handles resolving a function +after its been made ceratin that the type of declaration or definition is indeed for a function signature. + +By the point this function is called the following are known : export module flag, attributes, specifiers, return type, & name + +1. `parse_parameters` +2. parse postfix specifiers (we do not check if the specifier here is correct or not to be here... yet) +3. If there is a body : `parse_body` +4. Otherwise : + 1. Statment end + 2. Check for inline comment + +## `parse_function_body` + +Currently there is no actual parsing of the function body. Any content with the braces is shoved into an execution AST node. +In the future statements and expressions will be parsed. + +1. Open curly brace +2. Grab all tokens between the brace and the closing brace, shove them in a execution AST node. +3. Closing curly brace + +## `parse_global_nspace` + +1. Make sure this is being called for a valid type (namespace, global body, export body, linkage body) +2. If its not a global body, consume the opening curly brace +3. Parse the body (Possible options): + 1. Ignore dangling end statements + 2. NewLine : ast constant + 3. Comment : `parse_comment` + 4. Decl_Cass : `parse_complicated_definition` + 5. Decl_Enum : `parse_complicated_definition` + 6. Decl_Extern_Linkage : `parse_extern_link` + 7. Decl_Namespace : `parse_namespace` + 8. Decl_Struct : `parse_complicated_definition` + 9. Decl_Template : `parse_template` + 10. Decl_Typedef : `parse_typedef` + 11. Decl_Union : `parse_complicated_definition` + 12. Decl_Using : `parse_using` + 13. Preprocess_Define : `parse_define` + 14. Preprocess_Include : `parse_include` + 15. Preprocess_If, IfDef, IfNotDef, Elif : `parse_preprocess_cond` + 16. Preprocess_Else : ast constant + 17. Preprocess_Endif : ast constant + 18. Preprocess_Macro : `parse_simple_preprocess` + 19. Preprocess_Pragma : `parse_pragma` + 20. Preprocess_Unsupported : `parse_simple_preprocess` + 21. StaticAssert : `parse_static_assert` + 22. Module_Export : `parse_export_body` + 23. Module_Import : NOT_IMPLEMENTED + 24. The following compound into a resolved definition or declaration: + 1. Attributes ( Standard, GNU, MSVC, Macro ) : `parse_attributes` + 2. Specifiers ( consteval, constexpr, constinit, extern, forceinline, global, inline, internal_linkage, neverinline, static ) + 3. Is either ( identifier, const specifier, long, short, signed, unsigned, bool, char, double, int) + 1. Attempt to parse as construtor or destructor : `parse_global_nspace_constructor_destructor` + 2. If its an operator cast (definition outside class) : `parse_operator_cast` + 3. Its an operator, function, or varaible : `parse_operator_function_or_varaible` +4. If its not a global body, consume the closing curly brace + +## `parse_global_nspace_constructor_destructor` + +1. Look ahead for the start of the arguments for a possible constructor/destructor +2. Go back past the identifier +3. Check to see if its a destructor by checking for the `~` +4. Continue the next token should be a `::` +5. Determine if the next valid identifier (ignoring possible template parameters) is the same as the first identifier of the function. +6. If it is we have either a constructor or destructor so parse using their respective functions (`parse_constructor`, `parse_destructor`). + +## `parse_identifier` + +This is going to get heavily changed down the line to have a more broken down "identifier expression" so that the qualifier, template args, etc, can be distinguished between the targeted identifier. +The function can parse all of them, however the AST node compresses them all into a string. + +1. Consume first identifier +2. `parse_template_args` +3. While there is a static symbol accessor ( `::` ) + 1. Consume `::` + 2. Consume member identifier + 3. `parse_template args` (for member identifier) + 4. If a `~` is encounted and the scope is for a destructor's identifier, do not consume it and return with what parsed. + +## `parse_include` + +1. Consume include directive +2. Consume the path + +## `parse_operator_after_ret_type` + +This is needed as a operator defintion is not easily resolvable early on, as such this function handles resolving a operator after its been made ceratin that the type of declaration or definition is indeed for a operator signature. + +By the point this function is called the following are known : export module flag, attributes, specifiers, return type + +1. If there is any qualifiers for the operator, parse them +2. Consume operator keyword +3. Determine the operator type (This will be offloaded to the lexer moreso than how it is now) & consume +4. `parse_params` +5. If there is no parameters this is operator is a member of pointer if its symbols is a *. +6. Parse postfix specifiers +7. If there is a opening curly brace, `parse function_body` +8. Otherwise: consume end statement, check for inline comment. + +## `parse_operator_function_or_variable` + +When this function is called, attribute and specifiers may have been resolved, however what comes next can still be either an operator, function, or varaible. + +1. Check for preprocessor macro, if there is one : `parse_simple_preprocess` +2. `parse_type` (Does the bulk of the work) +3. Begin lookahead to see if we get qualifiers or we eventually find the operator declaration +4. If we find an operator keyword : `parse_operator_after_ret_type` +5. otherwise : + 1. `parse_identifier` + 2. If we se a opening parenthesis (capture start), its a function : `parse_function_after_name` + 3. Its a variable : `parse_variable_after_name` + +## `parse_pragma` + +1. Consume pragma directive +2. Process the token content into cached string + +## `parse_params` + +1. Consume either a `(` or `<` based on `use_template_capture` arg +2. If the we immdiately find a closing token, consume it and finish. +3. If we encounter a varadic argument, consume it and return a `param_varadic` ast constant +4. `parse_type` +5. If we have a macro, parse it (Unreal has macros as tags to parameters and or as entire arguments). +6. So long as next token isn't a comma + a. If we have an identifier + 1. Consume it + 2. Check for assignment: + a. Consume assign operator + b. Parse the expression +7. While we continue to encounter commas + a. Consume them + b. Repeat steps 3 to 6.2.b +8. Consume the closing token + +## `parse_preprocess_cond` + +1. Parse conditional directive +2. Process directive's content expression + +## `parse_simple_preprocess` + +There is still decent room for improvement in this setup. Right now the entire macro's relevant tokens are shoved into an untyped AST. It would be better to store it instead in an `AST_Macro` node instead down the line. + +1. Consume the macro token +2. Check for an opening curly brace + 1. Consume opening curly brace + 2. Until the closing curly is encountered consume all tokens. + 3. If the parent context is a typedef + 1. Check for end stement + 1. Consume it + 2. Consume potential inline comment +3. Otherwise do steps 3 to 3.1.2 +4. Shove it all in an untyped string + +## `parse_static_assert` + +1. Consume static assert and opening curly brace +2. Consume all tokens until the the closing brace is reached. +3. Consume curly brace and end statement +4. Place all tokens within braces into a content for the assert. + +## `parse_template_args` + +This will get changed heavily once we have better support for typename expressions + +1. Consume opening angle bracket +2. Consume all tokens until closing angle bracket +3. Consme closing angle bracket +4. Return the currtok with the ammended length. + +## `parse_variable_after_name` + +This is needed as a variable defintion is not easily resolvable early on, it takes a long evaluation period before its known that the declaration or definition is a variable. As such this function handles resolving a variable. + +By the point this function is called the following are known : export module flag, attributes, specifiers, value type, name + +1. If its an assignment, parse the assignment expression (currently to an untyped string) +2. If its an opening curly brace, parse the expression within (currnelty to an untyped stirng). + 1. Consume the closing curly brace +3. If its a `:`, we're dealing with bitfield definition: + 1. Consume the assign classifier + 2. Consume the expression (currently to an untyped string) +4. If a comma is encountered : `parse_variable declaration_list` +5. Consume statement end +6. Check for inline comment + +## `parse_variable_declaration_list` + +1. Consume the comma +2. Parse specifiers +3. `parse_variable_after_name` + +## `parse_class` + +1. `parse_class_struct` + +## `parse_constructor` + +This currently doesn't support postfix specifiers (planning to in the future) + +1. `parse_identifier` +2. `parse_parameters` +3. If currtok is a `:` + 1. Consume `:` + 2. Parse the initializer list + 3. `parse_function_body` +4. If currtok is an opening curly brace + 1. `parse_function_body` +5. Otherwise: + 1. Consume statement end + 2. Check for inline comment + +## `parse_destructor` + +1. Check for and consume virtual specifier +2. Check for the `~` operator +3. `parse_identifier` +4. Consume opening and closing parenthesis +5. Check for assignment operator: + 1. Consume assignment op + 2. Consume pure specifier `0` +6. If not pure virtual & currtok is opening curly brace: + 1. `parse_function_body` +7. Otherwise: + 1. Consume end statement + 2. If currtok is comment : `parse_comment` + +## `parse_enum` + +1. Consume enum token +2. Check for and consume class token +3. `parse_attributes` +4. If there is an identifier consume it +5. Check for a `:` + 1. Consume `:` + 2. `parse_type` +6. If there is a body parse it (Consume `{`): + 1. Newline : ast constant + 2. Comment : `parse_comment` + 3. Preprocess_Define : `parse_define` + 4. Preprocess_Conditional (if, ifdef, ifndef, elif ) : `parse_preprocess_cond` + 5. Preprocess_Else : ast constant + 6. Preprocess_Endif : ast constant + 7. Preprocess_Macro : `parse_simple_preprocess` + 8. Preprocess_Pragma : `parse_pragma` + 9. Preprocess_Unsupported : `parse_smple_preprocess` + 10. An actual enum entry + 1. Consume identifier + 2. If there is an assignment operator: + 1. Consume operator + 2. Consume the expression (assigned to untyped string for now) + 3. If a macro is encountered consume it (Unreal UMETA macro support) + 3. If there is a comma, consume it + +## `parse_export_body` + +1. `parse_global_nspace` + +## `parse_extern_link_body` + +1. `parse_global_nspace` + +## `parse_extern_link` + +1. Consume Decl_Extern_Linkage +2. Consume the linkage identifier +3. `parse_extern_link_body` + +## `parse_friend` + +1. Consume `friend` +2. `parse_type` +3. If the currok is an identifier its a function declaration or definition + 1. `parse_function_after_name` +4. Consume end statement so long as its not a function definion +5. Check for inline comment, `parse_comment` if exists + +## `parse_function` + +1. Check and parse for `export` +2. `parse_attributes` +3. Parse specifiers +4. `parse_type` +5. `parse_identifier` +6. `parse_function_after_name` + +## `parse_namespace` + +1. Consume namespace declaration +2. Parse identifier +3. `parse_global_namespace` + +## `parse_operator` + +1. Check for and parse export declaration +2. `parse_attributes` +3. Parse specifiers +4. `parse_type` +5. `parse_operator_after_ret_type` + +## `parse_operator_cast` + +1. Look for and parse a qualifier namespace for the cast (in-case this is defined outside the class's scope) +2. Consume operator declaration +3. `parse_type` +4. Consume opening and closing parethesis +5. Check for a const qualifiying specifier +6. Check to see if this is a definition (`{`) + 1. Consume `{` + 2. Parse body to untyped string (parsing statement and expressions not supported yet) + 3. Consume `}` +7. Otherwise: + 1. Consume end statement + 2. Check for and consume comment : `parse_comment` + + +## `parse_struct` + +1. `parse_class_struct` + +## `parse_template` + +Note: This currently doesn't support templated operator casts (going to need to add support for it) + +1. Check for and parse export declaration +2. Consume template declaration +3. `parse_params` +4. Parse for any of the following: + 1. Decl_Class : `parse_class` + 2. Decl_Struct : `parse_struct` + 3. Decl_Union : `parse_union` + 4. Decl_Using : `parse_using` + 5. The following compound into a resolved definition or declaration: + 1. `parse_attributes` + 2. Parse specifiers + 3. Attempt to parse as constructor or destructor: `parse_global_nspace_constructor_destructor` + 4. Otherwise: `parse_operator_function_or_variable` + +## `parse_type` + +This function's implementation is awful and not done correctly. It will most likely be overhauled in the future as I plan to segement the AST_Type into several AST varaints along with sub-types to help produce robust type expressions. +Hopefully I won't need to make authentic type expressions as I was hopeing to avoid that... + +### Current Algorithim + +Anything that is in the qualifier capture of the function typename is treated as an expression abstracted as an untyped string + +1. `parse_attributes` +2. Parse specifiers +3. If the `parse_type` was called from a template parse, check to see if class was used instead of typname and consume as name. +4. This is where things get ugly for each of these depend on what the next token is. + 1. If its an in-place definition of a class, enum, struct, or union: + 2. If its a decltype (Not supported yet but draft impl there) + 3. If its a compound native type expression (unsigned, char, short, long, int, float, dobule, etc ) + 4. Ends up being a regular type alias of an identifier +5. Parse specifiers (postfix) +6. We need to now look ahead to see If we're dealing with a function typename +7. If wer're dealing with a function typename: + 1. Shove the specifiers, and identifier code we have so far into a return type typename's Name (untyped string) + 1. Reset the specifiers code for the top-level typeanme + 2. Check to see if the next token is an identifier: + 1. `parse_identifier` + 3. Check to see if the next token is capture start and is not the last capture ("qualifier capture"): + 1. Consume `(` + 2. Consume expresssion between capture + 3. Consume `)` + 4. `parse_params` + 5. Parse postfix specifiers +8. Check for varaidic argument (param pack) token: + 1. Consume varadic argument token + +### WIP - Alternative Algorithim + +Currently wrapped up via macro: `GEN_USE_NEW_TYPENAME_PARSING` +Anything that is in the qualifier capture of the function typename is treated as an expression abstracted as an untyped string + +1. `parse_attributes` +2. Parse specifiers (prefix) +3. This is where things get ugly for each of these depend on what the next token is. + 1. If its an in-place definition of a class, enum, struct, or union: + 2. If its a decltype (Not supported yet but draft impl there) + 3. If its a compound native type expression (unsigned, char, short, long, int, float, dobule, etc ) + 4. Ends up being a regular type alias of an identifier +4. Parse specifiers (postfix) + 1. If any specifiers are found populate specifiers code with them. +5. We need to now look ahead to see If we're dealing with a function typename +6. If wer're dealing with a function typename: + 1. Shove the specifiers, and identifier code we have so far into a return type typename's Name (untyped string) + 1. Reset the specifiers code for the top-level typename + 2. Check to see if the next token is an identifier: + 1. `parse_identifier` + 3. Check to see if the next token is capture start and is not the last capture ("qualifier capture"): + 1. Consume `(` + 2. Parse binding specifiers + 3. `parse_identifier` + 4. `parse_parameters` -> params_nested + 5. Consume `)` + 6. Construct a nested function typename definition for the qualifier `Name` + 4. `parse_params` - > params + 5. Parse postfix specifiers +7. Check for varaidic argument (param pack) token: + 1. Consume varadic argument token + +### **Later: Algorithim based on typename expressions** + +## `parse_typedef` + +1. Check for export module specifier +2. typedef keyword +3. If its a preprocess macro: Get the macro name +4. Else: + 1. Check to see if its a complicated definition (in-place enum, class, struct, union) + 2. If its a complicated definition: + 1. Perform the look ahead similar to `parse_complicated_definition`'s implementation + 2. Check to see if its a forward declaration : `parse_forward_declaration` + 3. If end[-1] is an identifier: + 1. Its either an in-place, varaible type qualified identifier, or indirection type: + 1. `parse_foward_or_definition` + 4. else if end[-1] is a closing curly brace + 1. Its a definition: `parse_forward_or_definition` + 5. else if end[-1] is a closing square brace + 2. Its an array definition: `parse_type` + 3. Else : `parse-type` + 4. Check for identifier : Consume the token + 5. `parse_array_decl` +5. Consume end statement +6. Check for inline comment : `parse_comment` + +## `parse_union` + +1. Check for export module specifier +2. union keyword +3. `parse_attributes` +4. Check for identifier +5. Parse the body (Possible options): + 1. Newline + 2. Comment + 3. Decl_Class + 4. Decl_Enum + 5. Decl_Struct + 6. Decl_Union + 7. Preprocess_Define + 8. Preprocess_Conditional (if, ifdef, ifndef, elif, else, endif) + 9. Preprocess_Macro + 10. Preprocess_Pragma + 11. Unsupported preprocess directive + 12. Variable +6. If its not an inplace definiton: End Statement + +## `parse_using` + +1. Check for export module specifier +2. using keyword +3. Check to see if its a using namespace +4. Get the identifier +5. If its a regular using declaration: + 1. `parse_attributes` + 2. `parse_type` + 3. `parse_array_decl` +6. End statement +7. Check for inline comment + +## `parse_variable` + +1. Check for export module specifier +2. `parse_attributes` +3. `parse specifiers` +4. `parse_type` +5. `parse_identifier` +6. `parse_variable_after_name` diff --git a/codegen/gencpp/Parsing.md b/codegen/gencpp/Parsing.md index 3c2348ac6..c29fc179e 100644 --- a/codegen/gencpp/Parsing.md +++ b/codegen/gencpp/Parsing.md @@ -1,33 +1,39 @@ +## Navigation + +[Top](../Readme.md) + +<- [docs - General](Readme.md) + # Parsing -The library features a naive parser tailored for only what the library needs to construct the supported syntax of C++ into its AST. +The library features a naive single-pass parser tailored for only what the library needs to construct the supported syntax of C++ into its AST for *"front-end"* meta-programming purposes. -This parser does not, and should not do the compiler's job. By only supporting this minimal set of features, the parser is kept (so far) around ~5600 loc. I hope to keep it under 10k loc worst case. +This parser does not, and should not do the compiler's job. By only supporting this minimal set of features, the parser is kept (so far) around ~7000 loc. I hope to keep it under 10k loc worst case. -You can think of this parser of a frontend parser vs a semantic parser. Its intuitively similar to WYSIWYG. What you precerive as the syntax from the user-side before the compiler gets a hold of it, is what you get. +You can think of this parser as *frontend parser* vs a *semantic parser*. Its intuitively similar to WYSIWYG. What you ***precerive*** as the syntax from the user-side before the compiler gets a hold of it, is what you get. User exposed interface: ```cpp -CodeClass parse_class ( StrC class_def ); -CodeConstructor parse_constructor ( StrC constructor_def ); -CodeDestructor parse_destructor ( StrC destructor_def ); -CodeEnum parse_enum ( StrC enum_def ); -CodeBody parse_export_body ( StrC export_def ); -CodeExtern parse_extern_link ( StrC exten_link_def ); -CodeFriend parse_friend ( StrC friend_def ); -CodeFn parse_function ( StrC fn_def ); -CodeBody parse_global_body ( StrC body_def ); -CodeNS parse_namespace ( StrC namespace_def ); -CodeOperator parse_operator ( StrC operator_def ); -CodeOpCast parse_operator_cast( StrC operator_def ); -CodeStruct parse_struct ( StrC struct_def ); -CodeTemplate parse_template ( StrC template_def ); -CodeType parse_type ( StrC type_def ); -CodeTypedef parse_typedef ( StrC typedef_def ); -CodeUnion parse_union ( StrC union_def ); -CodeUsing parse_using ( StrC using_def ); -CodeVar parse_variable ( StrC var_def ); +CodeClass parse_class ( Str class_def ); +CodeConstructor parse_constructor ( Str constructor_def ); +CodeDestructor parse_destructor ( Str destructor_def ); +CodeEnum parse_enum ( Str enum_def ); +CodeBody parse_export_body ( Str export_def ); +CodeExtern parse_extern_link ( Str exten_link_def ); +CodeFriend parse_friend ( Str friend_def ); +CodeFn parse_function ( Str fn_def ); +CodeBody parse_global_body ( Str body_def ); +CodeNS parse_namespace ( Str namespace_def ); +CodeOperator parse_operator ( Str operator_def ); +CodeOpCast parse_operator_cast( Str operator_def ); +CodeStruct parse_struct ( Str struct_def ); +CodeTemplate parse_template ( Str template_def ); +CodeType parse_type ( Str type_def ); +CodeTypedef parse_typedef ( Str typedef_def ); +CodeUnion parse_union ( Str union_def ); +CodeUsing parse_using ( Str using_def ); +CodeVar parse_variable ( Str var_def ); ``` To parse file buffers, use the `parse_global_body` function. @@ -47,10 +53,11 @@ The keywords supported for the preprocessor are: * endif * pragma -Each directive `#` line is considered one preproecessor unit, and will be treated as one Preprocessor AST. *These ASTs will be considered members or entries of braced scope they reside within*. +Each directive `#` line is considered one preproecessor unit, and will be treated as one Preprocessor AST. If a directive is used with an unsupported keyword its will be processed as an untyped AST. -The preprocessor lines are stored as members of their associated scope they are parsed within. ( Global, Namespace, Class/Struct ) +The preprocessor lines are stored as members of their associated scope they are parsed within. ( Global, Namespace, Class/Struct ) +***Again (Its not standard): These ASTs will be considered members or entries of braced scope they reside within*** Any preprocessor definition abuse that changes the syntax of the core language is unsupported and will fail to parse if not kept within an execution scope (function body, or expression assignment). Exceptions: @@ -59,6 +66,8 @@ Exceptions: * Disable with: `#define GEN_PARSER_DISABLE_MACRO_FUNCTION_SIGNATURES` * typedefs allow for a preprocessed macro: `typedef MACRO();` * Disable with: `#define GEN_PARSER_DISABLE_MACRO_TYPEDEF` +* Macros can behave as typenames +* There is some macro support in paramters for functions or templates *(Specifically added to support parsing Unreal Engine source)*. *(Exceptions are added on an on-demand basis)* *(See functions `parse_operator_function_or_variable` and `parse_typedef` )* @@ -67,15 +76,23 @@ Adding your own exceptions is possible by simply modifying the parser to allow f *Note: You could interpret this strictness as a feature. This would allow the user to see if their codebase or a third-party's codebase some some egregious preprocessor abuse.* +If a macro is not defined withint e scope of parsing a set of files, it can be defined beforehand by: + +* Appending the [`PreprocessorDefines`](https://github.com/Ed94/gencpp/blob/a18b5b97aa5cfd20242065cbf53462a623cd18fa/base/components/header_end.hpp#L137) array. + * For functional macros a "(" just needs to be added after the name like: `(` so that it will tokenize its arguments as part of the token during lexing. +* Defining a CodeDefine using `def_define`. The definition will be processed by the interface for user into `PreprocessorDefines`. + * This can be prevented by setting the optional prameter `dont_append_preprocess_defines`. + The lexing and parsing takes shortcuts from whats expected in the standard. * Numeric literals are not checked for validity. -* The parse API treats any execution scope definitions with no validation and are turned into untyped Code ASTs. +* The parse API treats any execution scope definitions with no validation and are turned into untyped Code ASTs. (There is a [todo](https://github.com/Ed94/gencpp/issues/49) to add support) * *This includes the assignment of variables.* * Attributes ( `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU) ) * Assumed to *come before specifiers* (`const`, `constexpr`, `extern`, `static`, etc) for a function or right afterthe return type. * Or in the usual spot for class, structs, (*right after the declaration keyword*) * typedefs have attributes with the type (`parse_type`) * Parsing attributes can be extended to support user defined macros by defining `GEN_DEFINE_ATTRIBUTE_TOKENS` (see `gen.hpp` for the formatting) + * This is useful for example: parsing Unreal `Module_API` macros. Empty lines used throughout the file are preserved for formatting purposes during ast serialization. diff --git a/codegen/gencpp/Readme.md b/codegen/gencpp/Readme.md index f9c0815f0..3effbf251 100644 --- a/codegen/gencpp/Readme.md +++ b/codegen/gencpp/Readme.md @@ -1,135 +1,18 @@ -# gencpp +## Navigation -An attempt at simple staged metaprogramming for c/c++. +# base -The library API is a composition of code element constructors. -These build up a code AST to then serialize with a file builder. +[Top](../Readme.md) -This code base attempts follow the [handmade philosophy](https://handmade.network/manifesto). -Its not meant to be a black box metaprogramming utility, it should be easy to intergrate into a user's project domain. +* [docs](../docs/Readme.md) -## Notes +# Singleheader -**On Partial Hiatus: Working on handmade hero for now. Only fixes will be pushed as I come across them until I get what I want done from the series** +Creates a single header file version of the library using `singleheader.cpp`. +Follows the same convention seen in the gb, stb, and zpl libraries. -This project is still in development (very much an alpha state), so expect bugs and missing features. -See [issues](https://github.com/Ed94/gencpp/issues) for a list of known bugs or todos. - -The library can already be used to generate code just fine, but the parser is where the most work is needed. If your C++ isn't "down to earth" expect issues. - -A `natvis` and `natstepfilter` are provided in the scripts directory (its outdated, I'll update this readme when its not). - -***The editor and scanner have not been implemented yet. The scanner will come first, then the editor.*** - -A C variant is hosted [here](https://github.com/Ed94/genc); I will complete it when this library is feature complete, it should be easier to make than this... - -## Usage - -A metaprogram is built to generate files before the main program is built. We'll term runtime for this program as `GEN_TIME`. The metaprogram's core implementation are within `gen.hpp` and `gen.cpp` in the project directory. - -`gen.cpp` \`s `main()` is defined as `gen_main()` which the user will have to define once for their program. There they will dictate everything that should be generated. - -In order to keep the locality of this code within the same files the following pattern may be used (although this pattern isn't required at all): - -Within `program.cpp` : - -```cpp -#ifdef GEN_TIME -#include "gen.hpp" - -... - -u32 gen_main() -{ - ... -} -#endif - -// "Stage" agnostic code. - -#ifndef GEN_TIME -#include "program.gen.cpp" - - // Regular runtime dependent on the generated code here. -#endif +If using the library's provided build scripts: +```ps1 +.\build.ps1 singleheader ``` - -The design uses a constructive builder API for the code to generate. -The user is provided `Code` objects that are used to build up the AST. - -Example using each construction interface: - -### Upfront - -Validation and construction through a functional interface. - -```cpp -Code t_uw = def_type( name(uw) ); -Code t_allocator = def_type( name(allocator) ); -Code t_string_const = def_type( name(char), def_specifiers( args( ESpecifier::Const, ESpecifier::Ptr ) )); - -Code header; -{ - Code num = def_variable( t_uw, name(Num) ); - Code cap = def_variable( t_uw, name(Capacity) ); - Code mem_alloc = def_variable( t_allocator, name(Allocator) ); - Code body = def_struct_body( args( num, cap, mem_alloc ) ); - - header = def_struct( name(ArrayHeader), __, __, body ); -} -``` - -### Parse - -Validation through ast construction. - -```cpp -Code header = parse_struct( code( - struct ArrayHeader - { - uw Num; - uw Capacity; - allocator Allocator; - }; -)); - -``` - -### Untyped - -No validation, just glorified text injection. - -```cpp -Code header = code_str( - struct ArrayHeader - { - uw Num; - uw Capacity; - allocator Allocator; - }; -); -``` - -`name` is a helper macro for providing a string literal with its size, intended for the name parameter of functions. -`code` is a helper macro for providing a string literal with its size, but intended for code string parameters. -`args` is a helper macro for providing the number of arguments to varadic constructors. -`code_str` is a helper macro for writting `untyped_str( code( ))` - -All three constrcuton interfaces will generate the following C code: - -```cpp -struct ArrayHeader -{ - uw Num; - uw Capacity; - allocator Allocator; -}; -``` - -**Note: The formatting shown here is not how it will look. For your desired formatting its recommended to run a pass through the files with an auto-formatter.** -*(The library currently uses clang-format for formatting, beware its pretty slow...)* - -## Building - -See the [scripts directory](scripts/). diff --git a/codegen/gencpp/Readme_Docs.md b/codegen/gencpp/Readme_Docs.md index a0874d1fd..3eb1db2b8 100644 --- a/codegen/gencpp/Readme_Docs.md +++ b/codegen/gencpp/Readme_Docs.md @@ -1,62 +1,42 @@ -## Documentation +# General Docs -The project has no external dependencies beyond: +[Top](../Readme.md) -* `errno.h` -* `stat.h` -* `stdarg.h` -* `stddef.h` -* `stdio.h` -* `copyfile.h` (Mac) -* `types.h` (Linux) -* `unistd.h` (Linux/Mac) -* `intrin.h` (Windows) -* `io.h` (Windows with gcc) -* `windows.h` (Windows) +Contains: -Dependencies for the project are wrapped within `GENCPP_ROLL_OWN_DEPENDENCIES` (Defining it will disable them). -The majority of the dependency's implementation was derived from the [c-zpl library](https://github.com/zpl-c/zpl). +* [AST_Design](./AST_Design.md): Overview of ASTs +* [AST Types](./AST_Types.md): Listing of all AST types along with their Code type interface. +* [Parsing](./Parsing.md): Overview of the parsing interface. +* [Parser Algo](./Parser_Algo.md): In-depth breakdown of the parser's implementation. -This library was written in a subset of C++ where the following are not used at all: - -* RAII (Constructors/Destructors), lifetimes are managed using named static or regular functions. -* Language provide dynamic dispatch, RTTI -* Object-Oriented Inheritance -* Exceptions - -Polymorphic & Member-functions are used as an ergonomic choice, along with a conserative use of operator overloads. -There are only 4 template definitions in the entire library. (`Array`, `Hashtable`, `swap`, and `AST/Code::cast`) - -Two generic templated containers are used throughout the library: - -* `template< class Type> struct Array` -* `template< class Type> struct HashTable` - -Both Code and AST definitions have a `template< class Type> Code/AST :: cast()`. Its just an alternative way to explicitly cast to each other. - -`template< class Type> swap( Type& a, Type& b)` is used over a macro. - -Otherwise the library is free of any templates. - -### *WHAT IS NOT PROVIDED* +### *CURRENTLY UNSUPPORTED* **There is no support for validating expressions.** -Its difficult to parse without enough benefits (At the metaprogramming level). -I plan to add this only at the tail of the project parsing milestone. +Its a [todo](https://github.com/Ed94/gencpp/issues/49) -**Only trivial template support is provided.** -The intention is for only simple, non-recursive substitution. -The parameters of the template are treated like regular parameter AST entries. +**Only trivial template support is provided.** +The intention is for only simple, non-recursive substitution. +The parameters of the template are treated like regular parameter AST entries. This means that the typename entry for the parameter AST would be either: * `class` * `typename` * A fundamental type, function, or pointer type. -Anything beyond this usage is not supported by parse_template for arguments (at least not intentionally). -Use at your own mental peril. +***Concepts and Constraints are not supported*** +Its a [todo](https://github.com/Ed94/gencpp/issues/21) -*Concepts and Constraints are not supported, its usage is non-trivial substitution.* +### Feature Macros: + +* `GEN_DEFINE_ATTRIBUTE_TOKENS` : Allows user to define their own attribute macros for use in parsing. + * This can be generated using base.cpp. +* `GEN_DEFINE_LIBRARY_CORE_CONSTANTS` : Optional typename codes as they are non-standard to C/C++ and not necessary to library usage +* `GEN_DONT_ENFORCE_GEN_TIME_GUARD` : By default, the library ( gen.hpp/ gen.cpp ) expects the macro `GEN_TIME` to be defined, this disables that. +* `GEN_ENFORCE_STRONG_CODE_TYPES` : Enforces casts to filtered code types. +* `GEN_EXPOSE_BACKEND` : Will expose symbols meant for internal use only. +* `GEN_ROLL_OWN_DEPENDENCIES` : Optional override so that user may define the dependencies themselves. +* `GEN_DONT_ALLOW_INVALID_CODE` (Not implemented yet) : Will fail when an invalid code is constructed, parsed, or serialized. +* `GEN_C_LIKE_PP` : Setting to `` Will prevent usage of function defnitions using references and structs with member functions. Structs will still have user-defined operator conversions, for-range support, and other operator overloads ### The Data & Interface @@ -65,87 +45,33 @@ As mentioned in root readme, the user is provided Code objects by calling the co The AST is managed by the library and provided to the user via its interface. However, the user may specifiy memory configuration. -Data layout of AST struct (Subject to heavily change with upcoming redesign): +[Data layout of AST struct (Subject to heavily change with upcoming todos)](../base/components/ast.hpp#L396-461) -```cpp -union { - struct - { - AST* InlineCmt; // Class, Constructor, Destructor, Enum, Friend, Functon, Operator, OpCast, Struct, Typedef, Using, Variable - AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable - AST* Specs; // Destructor, Function, Operator, Typename, Variable - union { - AST* InitializerList; // Constructor - AST* ParentType; // Class, Struct, ParentType->Next has a possible list of interfaces. - AST* ReturnType; // Function, Operator, Typename - AST* UnderlyingType; // Enum, Typedef - AST* ValueType; // Parameter, Variable - }; - union { - AST* Macro; // Parameters - AST* BitfieldSize; // Variable (Class/Struct Data Member) - AST* Params; // Constructor, Function, Operator, Template, Typename - }; - union { - AST* ArrExpr; // Typename - AST* Body; // Class, Constructr, Destructor, Enum, Function, Namespace, Struct, Union - AST* Declaration; // Friend, Template - AST* Value; // Parameter, Variable - }; - union { - AST* NextVar; // Variable; Possible way to handle comma separated variables declarations. ( , NextVar->Specs NextVar->Name NextVar->ArrExpr = NextVar->Value ) - AST* SpecsFuncSuffix; // Only used with typenames, to store the function suffix if typename is function signature. - }; - }; - StringCached Content; // Attributes, Comment, Execution, Include - struct { - SpecifierT ArrSpecs[AST::ArrSpecs_Cap]; // Specifiers - AST* NextSpecs; // Specifiers - }; -}; -union { - AST* Prev; - AST* Front; - AST* Last; -}; -union { - AST* Next; - AST* Back; -}; -AST* Parent; -StringCached Name; -CodeT Type; -ModuleFlag ModuleFlags; -union { - b32 IsFunction; // Used by typedef to not serialize the name field. - b32 IsParamPack; // Used by typename to know if type should be considered a parameter pack. - OperatorT Op; - AccessSpec ParentAccess; - s32 NumEntries; -}; -s32 Token; // Handle to the token, stored in the CodeFile (Otherwise unretrivable) -``` +https://github.com/Ed94/gencpp/blob/eea4ebf5c40d5d87baa465abfb1be30845b2377e/base/components/ast.hpp#L396-L461 -*`CodeT` is a typedef for `ECode::Type` which has an underlying type of `u32`* +*`StringCahced` is a typedef for `Str` (a string slice), to denote it is an interned string* +*`CodeType` is enum taggin the type of code. Has an underlying type of `u32`* *`OperatorT` is a typedef for `EOperator::Type` which has an underlying type of `u32`* -*`StringCahced` is a typedef for `String const`, to denote it is an interned string* -*`String` is the dynamically allocated string type for the library* +*`StrBuilder` is the dynamically allocated string type for the library* -AST widths are setup to be AST_POD_Size. +AST widths are setup to be AST_POD_Size. The width dictates how much the static array can hold before it must give way to using an allocated array: ```cpp constexpr static -uw ArrSpecs_Cap = +int AST_ArrSpecs_Cap = ( AST_POD_Size - - sizeof(AST*) * 3 + - sizeof(Code) - sizeof(StringCached) - - sizeof(CodeT) + - sizeof(Code) * 2 + - sizeof(Token*) + - sizeof(Code) + - sizeof(CodeType) - sizeof(ModuleFlag) - sizeof(u32) ) -/ sizeof(SpecifierT) -1; // -1 for 4 extra bytes (Odd num of AST*) +/ sizeof(Specifier) - 1; ``` *Ex: If the AST_POD_Size is 128 the capacity of the static array is 20.* @@ -153,39 +79,39 @@ uw ArrSpecs_Cap = Data Notes: * The allocator definitions used are exposed to the user incase they want to dictate memory usage - * You'll find the memory handling in `init`, `deinit`, `reset`, `gen_string_allocator`, `get_cached_string`, `make_code`. - * Allocators are defined with the `AllocatorInfo` structure found in `dependencies\memory.hpp` + * You'll find the memory handling in `init`, `deinit`, `reset`, `gen_strbuilder_allocator`, `get_cached_string`, `make_code`. + * Allocators are defined with the `AllocatorInfo` structure found in [`memory.hpp`](../base/dependencies/memory.hpp) * Most of the work is just defining the allocation procedure: ```cpp - void* ( void* allocator_data, AllocType type, sw size, sw alignment, void* old_memory, sw old_size, u64 flags ); + void* ( void* allocator_data, AllocType type, ssize size, ssize alignment, void* old_memory, ssize old_size, u64 flags ); ``` * ASTs are wrapped for the user in a Code struct which is a wrapper for a AST* type. -* Both AST and Code have member symbols but their data layout is enforced to be POD types. +* Code types have member symbols but their data layout is enforced to be POD types. * This library treats memory failures as fatal. * Cached Strings are stored in their own set of arenas. AST constructors use cached strings for names, and content. * `StringArenas`, `StringCache`, `Allocator_StringArena`, and `Allocator_StringTable` are the associated containers or allocators. * Strings used for serialization and file buffers are not contained by those used for cached strings. * They are currently using `GlobalAllocator`, which are tracked array of arenas that grows as needed (adds buckets when one runs out). * Memory within the buckets is not reused, so its inherently wasteful. - * I will be augmenting the single arena with a simple slag allocator. -* Linked lists used children nodes on bodies, and parameters. + * I will be augmenting the default allocator with virtual memory & a slab allocator in the [future](https://github.com/Ed94/gencpp/issues/12) +* Intrusive linked lists used children nodes on bodies, and parameters. * Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup. -* Allocations can be tuned by defining the folloiwng macros: +* Allocations can be tuned by defining the folloiwng macros (will be moved to runtime configuration in the future): * `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator * `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator * `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching. * `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line. * `GEN_MAX_NAME_LENGTH` : Max length of any identifier. * `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code. - * `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable. + * `TokenMap_FixedArena` : token_fmt_va uses local_persit memory of this arena type for the hashtable. * `GEN_LEX_ALLOCATOR_SIZE` * `GEN_BUILDER_STR_BUFFER_RESERVE` The following CodeTypes are used which the user may optionally use strong typing with if they enable: `GEN_ENFORCE_STRONG_CODE_TYPES` -* CodeBody : Has support for `for-range` iterating across Code objects. +* CodeBody : Has support for `for : range` iterating across Code objects. * CodeAttributes * CodeComment * CodeClass @@ -202,13 +128,13 @@ The following CodeTypes are used which the user may optionally use strong typing * CodeNS * CodeOperator * CodeOpCast -* CodeParam : Has support for `for-range` iterating across parameters. +* CodeParams : Has support for `for : range` iterating across parameters. * CodePreprocessCond * CodePragma -* CodeSpecifiers : Has support for `for-range` iterating across specifiers. +* CodeSpecifiers : Has support for `for : range` iterating across specifiers. * CodeStruct * CodeTemplate -* CodeType +* CodeTypename * CodeTypedef * CodeUnion * CodeUsing @@ -293,6 +219,7 @@ Code ``` When using the body functions, its recommended to use the args macro to auto determine the number of arguments for the varadic: + ```cpp def_global_body( args( ht_entry, array_ht_entry, hashtable )); @@ -300,7 +227,7 @@ def_global_body( args( ht_entry, array_ht_entry, hashtable )); def_global_body( 3, ht_entry, array_ht_entry, hashtable ); ``` -If a more incremental approach is desired for the body ASTs, `Code def_body( CodeT type )` can be used to create an empty body. +If a more incremental approach is desired for the body ASTs, `Code def_body( CodeT type )` can be used to create an empty body. When the members have been populated use: `AST::validate_body` to verify that the members are valid entires for that type. ### Parse construction @@ -352,7 +279,7 @@ Interface : * untyped_fmt * untyped_token_fmt -During serialization any untyped Code AST has its string value directly injected inline of whatever context the content existed as an entry within. +During serialization any untyped Code AST has its string value directly injected inline of whatever context the content existed as an entry within. Even though these are not validated from somewhat correct c/c++ syntax or components, it doesn't mean that Untyped code can be added as any component of a Code AST: * Untyped code cannot have children, thus there cannot be recursive injection this way. @@ -373,6 +300,7 @@ Code = untyped_str( code( ``` Optionally, `code_str`, and `code_fmt` macros can be used so that the code macro doesn't have to be used: + ```cpp Code = code_str( ) ``` @@ -380,7 +308,7 @@ Code = code_str( ) Template metaprogramming in the traditional sense becomes possible with the use of `token_fmt` and parse constructors: ```cpp -StrC value = txt("Something"); +Str value = txt("Something"); char const* template_str = txt( Code with to replace with token_values @@ -402,8 +330,8 @@ The following are provided predefined by the library as they are commonly used: * `module_global_fragment` * `module_private_fragment` * `fmt_newline` -* `param_varaidc` (Used for varadic definitions) * `pragma_once` +* `param_varaidc` (Used for varadic definitions) * `preprocess_else` * `preprocess_endif` * `spec_const` @@ -412,13 +340,14 @@ The following are provided predefined by the library as they are commonly used: * `spec_constinit` * `spec_extern_linkage` (extern) * `spec_final` -* `Spec_gb_inline` +* `spec_forceinline` * `spec_global` (global macro) * `spec_inline` * `spec_internal_linkage` (internal macro) * `spec_local_persist` (local_persist macro) * `spec_mutable` * `spec_neverinline` +* `spec_noexcept` * `spec_override` * `spec_ptr` * `spec_pure` @@ -450,8 +379,8 @@ Optionally the following may be defined if `GEN_DEFINE_LIBRARY_CODE_CONSTANTS` i * `t_u16` * `t_u32` * `t_u64` -* `t_sw` (ssize_t) -* `t_uw` (size_t) +* `t_ssize` (ssize_t) +* `t_usize` (size_t) * `t_f32` * `t_f64` @@ -469,15 +398,12 @@ and have the desired specifiers assigned to them beforehand. ## Code generation and modification -There are three provided auxillary interfaces: +There are two provided auxillary interfaces: * Builder -* Editor * Scanner -Editor and Scanner are disabled by default, use `GEN_FEATURE_EDITOR` and `GEN_FEATURE_SCANNER` to enable them. - -### Builder is a similar object to the jai language's string_builder +### Builder is a similar object to the jai language's strbuilder_builder * The purpose of it is to generate a file. * A file is specified and opened for writing using the open( file_path) function. @@ -486,4 +412,8 @@ Editor and Scanner are disabled by default, use `GEN_FEATURE_EDITOR` and `GEN_FE ### Scanner Auxillary Interface -Provides *(eventually)* `scan_file` to automatically populate a CodeFile which contains a parsed AST (`Code`) of the file, with any contextual failures that are reported from the parser. +* The purpose is to scan or parse files +* Some with two basic functions to convert a fil to code: `scan_file` and `parse_file` + * `scan_file`: Merely grabs the file and stores it in an untyped Code. + * `parse_file`: Will parse the file using `parse_global_body` and return a `CodeBody`. +* Two basic functions for grabbing columns from a CSV: `parse_csv_one_column` and `parse_csv_two_columns` diff --git a/codegen/gencpp/gen.builder.cpp b/codegen/gencpp/gen.builder.cpp deleted file mode 100644 index 64d4751a7..000000000 --- a/codegen/gencpp/gen.builder.cpp +++ /dev/null @@ -1,62 +0,0 @@ -// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) - -#include "gen.builder.hpp" - -GEN_NS_BEGIN - -Builder Builder::open( char const* path ) -{ - Builder result; - - FileError error = file_open_mode( &result.File, EFileMode_WRITE, path ); - if ( error != EFileError_NONE ) - { - log_failure( "gen::File::open - Could not open file: %s", path ); - return result; - } - result.Buffer = String::make_reserve( GlobalAllocator, Builder_StrBufferReserve ); - - // log_fmt("$Builder - Opened file: %s\n", result.File.filename ); - return result; -} - -void Builder::pad_lines( s32 num ) -{ - Buffer.append( "\n" ); -} - -void Builder::print( Code code ) -{ - String str = code->to_string(); - // const sw len = str.length(); - // log_fmt( "%s - print: %.*s\n", File.filename, len > 80 ? 80 : len, str.Data ); - Buffer.append( str ); -} - -void Builder::print_fmt( char const* fmt, ... ) -{ - sw res; - char buf[GEN_PRINTF_MAXLEN] = { 0 }; - - va_list va; - va_start( va, fmt ); - res = str_fmt_va( buf, count_of( buf ) - 1, fmt, va ) - 1; - va_end( va ); - - // log_fmt( "$%s - print_fmt: %.*s\n", File.filename, res > 80 ? 80 : res, buf ); - Buffer.append( buf, res ); -} - -void Builder::write() -{ - bool result = file_write( &File, Buffer, Buffer.length() ); - - if ( result == false ) - log_failure( "gen::File::write - Failed to write to file: %s\n", file_name( &File ) ); - - log_fmt( "Generated: %s\n", File.filename ); - file_close( &File ); - Buffer.free(); -} - -GEN_NS_END diff --git a/codegen/gencpp/gen.builder.hpp b/codegen/gencpp/gen.builder.hpp deleted file mode 100644 index b5e202538..000000000 --- a/codegen/gencpp/gen.builder.hpp +++ /dev/null @@ -1,24 +0,0 @@ -// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) - -#pragma once - -#include "gen.hpp" - -GEN_NS_BEGIN - -struct Builder -{ - FileInfo File; - String Buffer; - - static Builder open( char const* path ); - - void pad_lines( s32 num ); - - void print( Code ); - void print_fmt( char const* fmt, ... ); - - void write(); -}; - -GEN_NS_END diff --git a/codegen/gencpp/gen.cpp b/codegen/gencpp/gen.cpp deleted file mode 100644 index fd61f690d..000000000 --- a/codegen/gencpp/gen.cpp +++ /dev/null @@ -1,12844 +0,0 @@ -// This file was generated automatially by gencpp's bootstrap.cpp (See: https://github.com/Ed94/gencpp) - -#if __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-const-variable" -#pragma clang diagnostic ignored "-Wunused-but-set-variable" -#pragma clang diagnostic ignored "-Wswitch" -#pragma clang diagnostic ignored "-Wunused-variable" -#pragma clang diagnostic ignored "-Wunknown-pragmas" -#pragma clang diagnostic ignored "-Wvarargs" -#pragma clang diagnostic ignored "-Wunused-function" -#endif - -#if __GNUC__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunknown-pragmas" -#pragma GCC diagnostic ignored "-Wcomment" -#pragma GCC diagnostic ignored "-Wswitch" -#pragma GCC diagnostic ignored "-Wunused-variable" -#endif -#if ! defined( GEN_DONT_ENFORCE_GEN_TIME_GUARD ) && ! defined( GEN_TIME ) -#error Gen.hpp : GEN_TIME not defined -#endif - -#include "gen.hpp" - -//! If its desired to roll your own dependencies, define GEN_ROLL_OWN_DEPENDENCIES before including this file. -//! Dependencies are derived from the c-zpl library: https://github.com/zpl-c/zpl -#ifndef GEN_ROLL_OWN_DEPENDENCIES -#include "gen.dep.cpp" -#endif - -GEN_NS_BEGIN - -#pragma region StaticData - -// TODO : Convert global allocation strategy to use a slab allocation strategy. -global AllocatorInfo GlobalAllocator; -global Array Global_AllocatorBuckets; - -// TODO(Ed) : Make the code pool a dynamic arena -global Array CodePools = { nullptr }; -global Array StringArenas = { nullptr }; - -global StringTable StringCache; - -global Arena LexArena; - -global AllocatorInfo Allocator_DataArrays = heap(); -global AllocatorInfo Allocator_CodePool = heap(); -global AllocatorInfo Allocator_Lexer = heap(); -global AllocatorInfo Allocator_StringArena = heap(); -global AllocatorInfo Allocator_StringTable = heap(); -global AllocatorInfo Allocator_TypeTable = heap(); - -#pragma endregion StaticData - -#pragma region Constants - -global Code access_public; -global Code access_protected; -global Code access_private; - -global CodeAttributes attrib_api_export; -global CodeAttributes attrib_api_import; - -global Code module_global_fragment; -global Code module_private_fragment; - -global Code fmt_newline; - -global CodeParam param_varadic; - -global CodePragma pragma_once; - -global CodePreprocessCond preprocess_else; -global CodePreprocessCond preprocess_endif; - -global CodeSpecifiers spec_const; -global CodeSpecifiers spec_consteval; -global CodeSpecifiers spec_constexpr; -global CodeSpecifiers spec_constinit; -global CodeSpecifiers spec_extern_linkage; -global CodeSpecifiers spec_final; -global CodeSpecifiers spec_gb_inline; -global CodeSpecifiers spec_gb_global; -global CodeSpecifiers spec_inline; -global CodeSpecifiers spec_gb_internal; -global CodeSpecifiers spec_local_persist; -global CodeSpecifiers spec_mutable; -global CodeSpecifiers spec_noexcept; -global CodeSpecifiers spec_neverinline; -global CodeSpecifiers spec_override; -global CodeSpecifiers spec_ptr; -global CodeSpecifiers spec_pure; -global CodeSpecifiers spec_ref; -global CodeSpecifiers spec_register; -global CodeSpecifiers spec_rvalue; -global CodeSpecifiers spec_static_member; -global CodeSpecifiers spec_gb_thread_local; -global CodeSpecifiers spec_virtual; -global CodeSpecifiers spec_volatile; - -global CodeType t_empty; -global CodeType t_auto; -global CodeType t_void; -global CodeType t_int; -global CodeType t_bool; -global CodeType t_char; -global CodeType t_wchar_t; -global CodeType t_class; -global CodeType t_typename; - -global Array PreprocessorDefines; - -#ifdef GEN_DEFINE_LIBRARY_CODE_CONSTANTS -global CodeType t_b32; - -global CodeType t_s8; -global CodeType t_s16; -global CodeType t_s32; -global CodeType t_s64; - -global CodeType t_u8; -global CodeType t_u16; -global CodeType t_u32; -global CodeType t_u64; - -global CodeType t_sw; -global CodeType t_uw; - -global CodeType t_f32; -global CodeType t_f64; -#endif - -#pragma endregion Constants - -#pragma region AST - -#define GEN_AST_BODY_CLASS_UNALLOWED_TYPES \ - case PlatformAttributes : \ - case Class_Body : \ - case Enum_Body : \ - case Extern_Linkage : \ - case Function_Body : \ - case Function_Fwd : \ - case Global_Body : \ - case Namespace : \ - case Namespace_Body : \ - case Operator : \ - case Operator_Fwd : \ - case Parameters : \ - case Specifiers : \ - case Struct_Body : \ - case Typename : -#define GEN_AST_BODY_STRUCT_UNALLOWED_TYPES GEN_AST_BODY_CLASS_UNALLOWED_TYPES - -#define GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES \ - case Access_Public : \ - case Access_Protected : \ - case Access_Private : \ - case PlatformAttributes : \ - case Class_Body : \ - case Enum_Body : \ - case Extern_Linkage : \ - case Friend : \ - case Function_Body : \ - case Function_Fwd : \ - case Global_Body : \ - case Namespace : \ - case Namespace_Body : \ - case Operator : \ - case Operator_Fwd : \ - case Operator_Member : \ - case Operator_Member_Fwd : \ - case Parameters : \ - case Specifiers : \ - case Struct_Body : \ - case Typename : - -#define GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES \ - case Access_Public : \ - case Access_Protected : \ - case Access_Private : \ - case PlatformAttributes : \ - case Class_Body : \ - case Enum_Body : \ - case Execution : \ - case Friend : \ - case Function_Body : \ - case Namespace_Body : \ - case Operator_Member : \ - case Operator_Member_Fwd : \ - case Parameters : \ - case Specifiers : \ - case Struct_Body : \ - case Typename : -#define GEN_AST_BODY_EXPORT_UNALLOWED_TYPES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES -#define GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES - -#define GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES \ - case Access_Public : \ - case Access_Protected : \ - case Access_Private : \ - case PlatformAttributes : \ - case Class_Body : \ - case Enum_Body : \ - case Execution : \ - case Friend : \ - case Function_Body : \ - case Namespace_Body : \ - case Operator_Member : \ - case Operator_Member_Fwd : \ - case Parameters : \ - case Specifiers : \ - case Struct_Body : \ - case Typename : - -Code Code::Global; -Code Code::Invalid; - -// This serializes all the data-members in a "debug" format, where each member is printed with its associated value. -char const* AST::debug_str() -{ - String result = String::make_reserve( GlobalAllocator, kilobytes( 1 ) ); - - if ( Parent ) - result.append_fmt( "\n\tParent : %S %S", Parent->type_str(), Name ? Name : "" ); - else - result.append_fmt( "\n\tParent : %S", "Null" ); - - result.append_fmt( "\n\tName : %S", Name ? Name : "Null" ); - result.append_fmt( "\n\tType : %S", type_str() ); - result.append_fmt( "\n\tModule Flags : %S", to_str( ModuleFlags ) ); - - switch ( Type ) - { - using namespace ECode; - - case Invalid : - case NewLine : - case Access_Private : - case Access_Protected : - case Access_Public : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - break; - - case Untyped : - case Execution : - case Comment : - case PlatformAttributes : - case Preprocess_Define : - case Preprocess_Include : - case Preprocess_Pragma : - case Preprocess_If : - case Preprocess_ElIf : - case Preprocess_Else : - case Preprocess_IfDef : - case Preprocess_IfNotDef : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tContent: %S", Content ); - break; - - case Class : - case Struct : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmd : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes : %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tParentAccess: %s", ParentType ? to_str( ParentAccess ) : "No Parent" ); - result.append_fmt( "\n\tParentType : %s", ParentType ? ParentType->type_str() : "Null" ); - result.append_fmt( "\n\tBody : %S", Body ? Body->debug_str() : "Null" ); - break; - - case Class_Fwd : - case Struct_Fwd : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmd : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes : %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tParentAccess: %s", ParentType ? to_str( ParentAccess ) : "No Parent" ); - result.append_fmt( "\n\tParentType : %s", ParentType ? ParentType->type_str() : "Null" ); - break; - - case Constructor : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tInitializerList: %S", InitializerList ? InitializerList->to_string() : "Null" ); - result.append_fmt( "\n\tParams : %S", Params ? Params->to_string() : "Null" ); - result.append_fmt( "\n\tBody : %S", Body ? Body->debug_str() : "Null" ); - break; - - case Constructor_Fwd : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tInitializerList: %S", InitializerList ? InitializerList->to_string() : "Null" ); - result.append_fmt( "\n\tParams : %S", Params ? Params->to_string() : "Null" ); - break; - - case Destructor : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tBody : %S", Body ? Body->debug_str() : "Null" ); - break; - - case Destructor_Fwd : - break; - - case Enum : - case Enum_Class : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes : %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tUnderlying Type : %S", UnderlyingType ? UnderlyingType->to_string() : "Null" ); - result.append_fmt( "\n\tBody : %S", Body ? Body->debug_str() : "Null" ); - break; - - case Enum_Fwd : - case Enum_Class_Fwd : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes : %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tUnderlying Type : %S", UnderlyingType ? UnderlyingType->to_string() : "Null" ); - break; - - case Extern_Linkage : - case Namespace : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tBody: %S", Body ? Body->debug_str() : "Null" ); - break; - - case Friend : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tDeclaration: %S", Declaration ? Declaration->to_string() : "Null" ); - break; - - case Function : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes: %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tReturnType: %S", ReturnType ? ReturnType->to_string() : "Null" ); - result.append_fmt( "\n\tParams : %S", Params ? Params->to_string() : "Null" ); - result.append_fmt( "\n\tBody : %S", Body ? Body->debug_str() : "Null" ); - break; - - case Function_Fwd : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes: %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tReturnType: %S", ReturnType ? ReturnType->to_string() : "Null" ); - result.append_fmt( "\n\tParams : %S", Params ? Params->to_string() : "Null" ); - break; - - case Module : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - break; - - case Operator : - case Operator_Member : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes: %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tReturnType: %S", ReturnType ? ReturnType->to_string() : "Null" ); - result.append_fmt( "\n\tParams : %S", Params ? Params->to_string() : "Null" ); - result.append_fmt( "\n\tBody : %S", Body ? Body->debug_str() : "Null" ); - result.append_fmt( "\n\tOp : %S", to_str( Op ) ); - break; - - case Operator_Fwd : - case Operator_Member_Fwd : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes: %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tReturnType: %S", ReturnType ? ReturnType->to_string() : "Null" ); - result.append_fmt( "\n\tParams : %S", Params ? Params->to_string() : "Null" ); - result.append_fmt( "\n\tOp : %S", to_str( Op ) ); - break; - - case Operator_Cast : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tValueType : %S", ValueType ? ValueType->to_string() : "Null" ); - result.append_fmt( "\n\tBody : %S", Body ? Body->debug_str() : "Null" ); - break; - - case Operator_Cast_Fwd : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tValueType : %S", ValueType ? ValueType->to_string() : "Null" ); - break; - - case Parameters : - result.append_fmt( "\n\tNumEntries: %d", NumEntries ); - result.append_fmt( "\n\tLast : %S", Last->Name ); - result.append_fmt( "\n\tNext : %S", Next->Name ); - result.append_fmt( "\n\tValueType : %S", ValueType ? ValueType->to_string() : "Null" ); - result.append_fmt( "\n\tValue : %S", Value ? Value->to_string() : "Null" ); - break; - - case Specifiers : - { - result.append_fmt( "\n\tNumEntries: %d", NumEntries ); - result.append( "\n\tArrSpecs: " ); - - s32 idx = 0; - s32 left = NumEntries; - while ( left-- ) - { - StrC spec = ESpecifier::to_str( ArrSpecs[idx] ); - result.append_fmt( "%.*s, ", spec.Len, spec.Ptr ); - idx++; - } - result.append_fmt( "\n\tNextSpecs: %S", NextSpecs ? NextSpecs->debug_str() : "Null" ); - } - break; - - case Template : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tParams : %S", Params ? Params->to_string() : "Null" ); - result.append_fmt( "\n\tDeclaration: %S", Declaration ? Declaration->to_string() : "Null" ); - break; - - case Typedef : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tUnderlyingType: %S", UnderlyingType ? UnderlyingType->to_string() : "Null" ); - break; - - case Typename : - result.append_fmt( "\n\tAttributes : %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tReturnType : %S", ReturnType ? ReturnType->to_string() : "Null" ); - result.append_fmt( "\n\tParams : %S", Params ? Params->to_string() : "Null" ); - result.append_fmt( "\n\tArrExpr : %S", ArrExpr ? ArrExpr->to_string() : "Null" ); - break; - - case Union : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tAttributes: %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tBody : %S", Body ? Body->debug_str() : "Null" ); - break; - - case Using : - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes : %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tUnderlyingType: %S", UnderlyingType ? UnderlyingType->to_string() : "Null" ); - break; - - case Variable : - - if ( Parent && Parent->Type == Variable ) - { - // Its a NextVar - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tValue : %S", Value ? Value->to_string() : "Null" ); - result.append_fmt( "\n\tBitfieldSize: %S", BitfieldSize ? BitfieldSize->to_string() : "Null" ); - result.append_fmt( "\n\tNextVar : %S", NextVar ? NextVar->debug_str() : "Null" ); - break; - } - - if ( Prev ) - result.append_fmt( "\n\tPrev: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - if ( Next ) - result.append_fmt( "\n\tNext: %S %S", Prev->type_str(), Prev->Name ? Prev->Name : "Null" ); - - result.append_fmt( "\n\tInlineCmt : %S", InlineCmt ? InlineCmt->Content : "Null" ); - result.append_fmt( "\n\tAttributes : %S", Attributes ? Attributes->to_string() : "Null" ); - result.append_fmt( "\n\tSpecs : %S", Specs ? Specs->to_string() : "Null" ); - result.append_fmt( "\n\tValueType : %S", ValueType ? ValueType->to_string() : "Null" ); - result.append_fmt( "\n\tBitfieldSize: %S", BitfieldSize ? BitfieldSize->to_string() : "Null" ); - result.append_fmt( "\n\tValue : %S", Value ? Value->to_string() : "Null" ); - result.append_fmt( "\n\tNextVar : %S", NextVar ? NextVar->debug_str() : "Null" ); - break; - } - - return result; -} - -AST* AST::duplicate() -{ - using namespace ECode; - - AST* result = make_code().ast; - - mem_copy( result, this, sizeof( AST ) ); - - result->Parent = nullptr; - return result; -} - -String AST::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void AST::to_string( String& result ) -{ - local_persist thread_local char SerializationLevel = 0; - - switch ( Type ) - { - using namespace ECode; - - case Invalid : -#ifdef GEN_DONT_ALLOW_INVALID_CODE - log_failure( "Attempted to serialize invalid code! - %S", Parent ? Parent->debug_str() : Name ); -#else - result.append_fmt( "Invalid Code!" ); -#endif - break; - - case NewLine : - result.append( "\n" ); - break; - - case Untyped : - case Execution : - case Comment : - case PlatformAttributes : - result.append( Content ); - break; - - case Access_Private : - case Access_Protected : - case Access_Public : - result.append( Name ); - break; - - case Class : - cast().to_string_def( result ); - break; - - case Class_Fwd : - cast().to_string_fwd( result ); - break; - - case Constructor : - cast().to_string_def( result ); - break; - - case Constructor_Fwd : - cast().to_string_fwd( result ); - break; - - case Destructor : - cast().to_string_def( result ); - break; - - case Destructor_Fwd : - cast().to_string_fwd( result ); - break; - - case Enum : - cast().to_string_def( result ); - break; - - case Enum_Fwd : - cast().to_string_fwd( result ); - break; - - case Enum_Class : - cast().to_string_class_def( result ); - break; - - case Enum_Class_Fwd : - cast().to_string_class_fwd( result ); - break; - - case Export_Body : - cast().to_string_export( result ); - break; - - case Extern_Linkage : - cast().to_string( result ); - break; - - case Friend : - cast().to_string( result ); - break; - - case Function : - cast().to_string_def( result ); - break; - - case Function_Fwd : - cast().to_string_fwd( result ); - break; - - case Module : - cast().to_string( result ); - break; - - case Namespace : - cast().to_string( result ); - break; - - case Operator : - case Operator_Member : - cast().to_string_def( result ); - break; - - case Operator_Fwd : - case Operator_Member_Fwd : - cast().to_string_fwd( result ); - break; - - case Operator_Cast : - cast().to_string_def( result ); - break; - - case Operator_Cast_Fwd : - cast().to_string_fwd( result ); - break; - - case Parameters : - cast().to_string( result ); - break; - - case Preprocess_Define : - cast().to_string( result ); - break; - - case Preprocess_If : - cast().to_string_if( result ); - break; - - case Preprocess_IfDef : - cast().to_string_ifdef( result ); - break; - - case Preprocess_IfNotDef : - cast().to_string_ifndef( result ); - break; - - case Preprocess_Include : - cast().to_string( result ); - break; - - case Preprocess_ElIf : - cast().to_string_elif( result ); - break; - - case Preprocess_Else : - cast().to_string_else( result ); - break; - - case Preprocess_EndIf : - cast().to_string_endif( result ); - break; - - case Preprocess_Pragma : - cast().to_string( result ); - break; - - case Specifiers : - cast().to_string( result ); - break; - - case Struct : - cast().to_string_def( result ); - break; - - case Struct_Fwd : - cast().to_string_fwd( result ); - break; - - case Template : - cast().to_string( result ); - break; - - case Typedef : - cast().to_string( result ); - break; - - case Typename : - cast().to_string( result ); - break; - - case Union : - cast().to_string( result ); - break; - - case Using : - cast().to_string( result ); - break; - - case Using_Namespace : - cast().to_string_ns( result ); - break; - - case Variable : - cast().to_string( result ); - break; - - case Enum_Body : - case Class_Body : - case Extern_Linkage_Body : - case Function_Body : - case Global_Body : - case Namespace_Body : - case Struct_Body : - case Union_Body : - cast().to_string( result ); - break; - } -} - -bool AST::is_equal( AST* other ) -{ - /* - AST values are either some u32 value, a cached string, or a pointer to another AST. - - u32 values are compared by value. - Cached strings are compared by pointer. - AST nodes are compared with AST::is_equal. - */ - if ( other == nullptr ) - { - log_fmt( "AST::is_equal: other is null\nAST: %S", debug_str() ); - return false; - } - - if ( Type != other->Type ) - { - log_fmt( "AST::is_equal: Type check failure with other\nAST: %S\nOther: %S", debug_str(), other->debug_str() ); - - return false; - } - - switch ( Type ) - { - using namespace ECode; - -#define check_member_val( val ) \ - if ( val != other->val ) \ - { \ - log_fmt( \ - "\nAST::is_equal: Member - " #val \ - " failed\n" \ - "AST : %S\n" \ - "Other: %S\n", \ - debug_str(), \ - other->debug_str() \ - ); \ - \ - return false; \ - } - -#define check_member_str( str ) \ - if ( str != other->str ) \ - { \ - log_fmt( \ - "\nAST::is_equal: Member string - " #str \ - " failed\n" \ - "AST : %S\n" \ - "Other: %S\n", \ - debug_str(), \ - other->debug_str() \ - ); \ - \ - return false; \ - } - -#define check_member_content( content ) \ - if ( content != other->content ) \ - { \ - log_fmt( \ - "\nAST::is_equal: Member content - " #content \ - " failed\n" \ - "AST : %S\n" \ - "Other: %S\n", \ - debug_str(), \ - other->debug_str() \ - ); \ - \ - log_fmt( \ - "Content cannot be trusted to be unique with this check " \ - "so it must be verified by eye for now\n" \ - "AST Content:\n%S\n" \ - "Other Content:\n%S\n", \ - content.visualize_whitespace(), \ - other->content.visualize_whitespace() \ - ); \ - } - -#define check_member_ast( ast ) \ - if ( ast ) \ - { \ - if ( other->ast == nullptr ) \ - { \ - log_fmt( \ - "\nAST::is_equal: Failed for member " #ast \ - " other equivalent param is null\n" \ - "AST : %s\n" \ - "Other: %s\n" \ - "For ast member: %s\n", \ - debug_str(), \ - other->debug_str(), \ - ast->debug_str() \ - ); \ - \ - return false; \ - } \ - \ - if ( ! ast->is_equal( other->ast ) ) \ - { \ - log_fmt( \ - "\nAST::is_equal: Failed for " #ast \ - "\n" \ - "AST : %S\n" \ - "Other: %S\n" \ - "For ast member: %S\n" \ - "other's ast member: %S\n", \ - debug_str(), \ - other->debug_str(), \ - ast->debug_str(), \ - other->ast->debug_str() \ - ); \ - \ - return false; \ - } \ - } - - case NewLine : - case Access_Public : - case Access_Protected : - case Access_Private : - case Preprocess_Else : - case Preprocess_EndIf : - return true; - - - // Comments are not validated. - case Comment : - return true; - - case Execution : - case PlatformAttributes : - case Untyped : - { - check_member_content( Content ); - - return true; - } - - case Class_Fwd : - case Struct_Fwd : - { - check_member_str( Name ); - check_member_ast( ParentType ); - check_member_val( ParentAccess ); - check_member_ast( Attributes ); - - return true; - } - - case Class : - case Struct : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( ParentType ); - check_member_val( ParentAccess ); - check_member_ast( Attributes ); - check_member_ast( Body ); - - return true; - } - - case Constructor : - { - check_member_ast( InitializerList ); - check_member_ast( Params ); - check_member_ast( Body ); - - return true; - } - - case Constructor_Fwd : - { - check_member_ast( InitializerList ); - check_member_ast( Params ); - - return true; - } - - case Destructor : - { - check_member_ast( Specs ); - check_member_ast( Body ); - - return true; - } - - case Destructor_Fwd : - { - check_member_ast( Specs ); - - return true; - } - - case Enum : - case Enum_Class : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( Attributes ); - check_member_ast( UnderlyingType ); - check_member_ast( Body ); - - return true; - } - - case Enum_Fwd : - case Enum_Class_Fwd : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( Attributes ); - check_member_ast( UnderlyingType ); - - return true; - } - - case Extern_Linkage : - { - check_member_str( Name ); - check_member_ast( Body ); - - return true; - } - - case Friend : - { - check_member_str( Name ); - check_member_ast( Declaration ); - - return true; - } - - case Function : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( ReturnType ); - check_member_ast( Attributes ); - check_member_ast( Specs ); - check_member_ast( Params ); - check_member_ast( Body ); - - return true; - } - - case Function_Fwd : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( ReturnType ); - check_member_ast( Attributes ); - check_member_ast( Specs ); - check_member_ast( Params ); - - return true; - } - - case Module : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - - return true; - } - - case Namespace : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( Body ); - - return true; - } - - case Operator : - case Operator_Member : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( ReturnType ); - check_member_ast( Attributes ); - check_member_ast( Specs ); - check_member_ast( Params ); - check_member_ast( Body ); - - return true; - } - - case Operator_Fwd : - case Operator_Member_Fwd : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( ReturnType ); - check_member_ast( Attributes ); - check_member_ast( Specs ); - check_member_ast( Params ); - - return true; - } - - case Operator_Cast : - { - check_member_str( Name ); - check_member_ast( Specs ); - check_member_ast( ValueType ); - check_member_ast( Body ); - - return true; - } - - case Operator_Cast_Fwd : - { - check_member_str( Name ); - check_member_ast( Specs ); - check_member_ast( ValueType ); - - return true; - } - - case Parameters : - { - if ( NumEntries > 1 ) - { - AST* curr = this; - AST* curr_other = other; - while ( curr != nullptr ) - { - if ( curr ) - { - if ( curr_other == nullptr ) - { - log_fmt( - "\nAST::is_equal: Failed for parameter, other equivalent param is null\n" - "AST : %S\n" - "Other: %S\n" - "For ast member: %S\n", - curr->debug_str() - ); - - return false; - } - - if ( curr->Name != curr_other->Name ) - { - log_fmt( - "\nAST::is_equal: Failed for parameter name check\n" - "AST : %S\n" - "Other: %S\n" - "For ast member: %S\n" - "other's ast member: %S\n", - debug_str(), - other->debug_str(), - curr->debug_str(), - curr_other->debug_str() - ); - return false; - } - - if ( curr->ValueType && ! curr->ValueType->is_equal( curr_other->ValueType ) ) - { - log_fmt( - "\nAST::is_equal: Failed for parameter value type check\n" - "AST : %S\n" - "Other: %S\n" - "For ast member: %S\n" - "other's ast member: %S\n", - debug_str(), - other->debug_str(), - curr->debug_str(), - curr_other->debug_str() - ); - return false; - } - - if ( curr->Value && ! curr->Value->is_equal( curr_other->Value ) ) - { - log_fmt( - "\nAST::is_equal: Failed for parameter value check\n" - "AST : %S\n" - "Other: %S\n" - "For ast member: %S\n" - "other's ast member: %S\n", - debug_str(), - other->debug_str(), - curr->debug_str(), - curr_other->debug_str() - ); - return false; - } - } - - curr = curr->Next; - curr_other = curr_other->Next; - } - - check_member_val( NumEntries ); - - return true; - } - - check_member_str( Name ); - check_member_ast( ValueType ); - check_member_ast( Value ); - check_member_ast( ArrExpr ); - - return true; - } - - case Preprocess_Define : - { - check_member_str( Name ); - check_member_content( Content ); - - return true; - } - - case Preprocess_If : - case Preprocess_IfDef : - case Preprocess_IfNotDef : - case Preprocess_ElIf : - { - check_member_content( Content ); - - return true; - } - - case Preprocess_Include : - case Preprocess_Pragma : - { - check_member_content( Content ); - - return true; - } - - case Specifiers : - { - check_member_val( NumEntries ); - check_member_str( Name ); - for ( s32 idx = 0; idx < NumEntries; ++idx ) - { - check_member_val( ArrSpecs[idx] ); - } - return true; - } - - case Template : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( Params ); - check_member_ast( Declaration ); - - return true; - } - - case Typedef : - { - check_member_val( IsFunction ); - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( Specs ); - check_member_ast( UnderlyingType ); - - return true; - } - case Typename : - { - check_member_val( IsParamPack ); - check_member_str( Name ); - check_member_ast( Specs ); - check_member_ast( ArrExpr ); - - return true; - } - - case Union : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( Attributes ); - check_member_ast( Body ); - - return true; - } - - case Using : - case Using_Namespace : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( UnderlyingType ); - check_member_ast( Attributes ); - - return true; - } - - case Variable : - { - check_member_val( ModuleFlags ); - check_member_str( Name ); - check_member_ast( ValueType ); - check_member_ast( BitfieldSize ); - check_member_ast( Value ); - check_member_ast( Attributes ); - check_member_ast( Specs ); - check_member_ast( NextVar ); - - return true; - } - - case Class_Body : - case Enum_Body : - case Export_Body : - case Global_Body : - case Namespace_Body : - case Struct_Body : - case Union_Body : - { - check_member_ast( Front ); - check_member_ast( Back ); - - AST* curr = Front; - AST* curr_other = other->Front; - while ( curr != nullptr ) - { - if ( curr_other == nullptr ) - { - log_fmt( - "\nAST::is_equal: Failed for body, other equivalent param is null\n" - "AST : %S\n" - "Other: %S\n" - "For ast member: %S\n", - curr->debug_str() - ); - - return false; - } - - if ( ! curr->is_equal( curr_other ) ) - { - log_fmt( - "\nAST::is_equal: Failed for body\n" - "AST : %S\n" - "Other: %S\n" - "For ast member: %S\n" - "other's ast member: %S\n", - debug_str(), - other->debug_str(), - curr->debug_str(), - curr_other->debug_str() - ); - - return false; - } - - curr = curr->Next; - curr_other = curr_other->Next; - } - - check_member_val( NumEntries ); - - return true; - } - -#undef check_member_val -#undef check_member_str -#undef check_member_ast - } - - return true; -} - -bool AST::validate_body() -{ - using namespace ECode; - -#define CheckEntries( Unallowed_Types ) \ - do \ - { \ - for ( Code entry : cast() ) \ - { \ - switch ( entry->Type ) \ - { \ - Unallowed_Types log_failure( "AST::validate_body: Invalid entry in body %s", entry.debug_str() ); \ - return false; \ - } \ - } \ - } while ( 0 ); - - switch ( Type ) - { - case Class_Body : - CheckEntries( GEN_AST_BODY_CLASS_UNALLOWED_TYPES ); - break; - case Enum_Body : - for ( Code entry : cast() ) - { - if ( entry->Type != Untyped ) - { - log_failure( "AST::validate_body: Invalid entry in enum body (needs to be untyped or comment) %s", entry.debug_str() ); - return false; - } - } - break; - case Export_Body : - CheckEntries( GEN_AST_BODY_CLASS_UNALLOWED_TYPES ); - break; - case Extern_Linkage : - CheckEntries( GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES ); - break; - case Function_Body : - CheckEntries( GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES ); - break; - case Global_Body : - for ( Code entry : cast() ) - { - switch ( entry->Type ) - { - case Access_Public : - case Access_Protected : - case Access_Private : - case PlatformAttributes : - case Class_Body : - case Enum_Body : - case Execution : - case Friend : - case Function_Body : - case Global_Body : - case Namespace_Body : - case Operator_Member : - case Operator_Member_Fwd : - case Parameters : - case Specifiers : - case Struct_Body : - case Typename : - log_failure( "AST::validate_body: Invalid entry in body %s", entry.debug_str() ); - return false; - } - } - break; - case Namespace_Body : - CheckEntries( GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES ); - break; - case Struct_Body : - CheckEntries( GEN_AST_BODY_STRUCT_UNALLOWED_TYPES ); - break; - case Union_Body : - for ( Code entry : Body->cast() ) - { - if ( entry->Type != Untyped ) - { - log_failure( "AST::validate_body: Invalid entry in union body (needs to be untyped or comment) %s", entry.debug_str() ); - return false; - } - } - break; - - default : - log_failure( "AST::validate_body: Invalid this AST does not have a body %s", debug_str() ); - return false; - } - - return false; - -#undef CheckEntries -} - -String Code::to_string() -{ - if ( ast == nullptr ) - { - log_failure( "Code::to_string: Cannot convert code to string, AST is null!" ); - return { nullptr }; - } - return rcast( AST*, ast )->to_string(); -} - -String CodeAttributes::to_string() -{ - return ast->Content.duplicate( GlobalAllocator ); -} - -String CodeBody::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Untyped : - case Execution : - result.append( raw()->Content ); - break; - - case Enum_Body : - case Class_Body : - case Extern_Linkage_Body : - case Function_Body : - case Global_Body : - case Namespace_Body : - case Struct_Body : - case Union_Body : - to_string( result ); - break; - - case Export_Body : - to_string_export( result ); - break; - } - return result; -} - -void CodeBody::to_string( String& result ) -{ - Code curr = ast->Front; - s32 left = ast->NumEntries; - while ( left-- ) - { - result.append_fmt( "%S", curr.to_string() ); - ++curr; - } -} - -void CodeBody::to_string_export( String& result ) -{ - result.append_fmt( "export\n{\n" ); - - Code curr = *this; - s32 left = ast->NumEntries; - while ( left-- ) - { - result.append_fmt( "%S", curr.to_string() ); - ++curr; - } - - result.append_fmt( "};\n" ); -} - -String CodeComment::to_string() -{ - return ast->Content.duplicate( GlobalAllocator ); -} - -String CodeConstructor::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Constructor : - to_string_def( result ); - break; - case Constructor_Fwd : - to_string_fwd( result ); - break; - } - return result; -} - -void CodeConstructor::to_string_def( String& result ) -{ - AST* ClassStructParent = ast->Parent->Parent; - if ( ClassStructParent ) - { - result.append( ClassStructParent->Name ); - } - else - { - result.append( ast->Name ); - } - - if ( ast->Params ) - result.append_fmt( "( %S )", ast->Params.to_string() ); - else - result.append( "()" ); - - if ( ast->InitializerList ) - result.append_fmt( " : %S", ast->InitializerList.to_string() ); - - if ( ast->InlineCmt ) - result.append_fmt( " // %S", ast->InlineCmt->Content ); - - result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() ); -} - -void CodeConstructor::to_string_fwd( String& result ) -{ - AST* ClassStructParent = ast->Parent->Parent; - if ( ClassStructParent ) - { - result.append( ClassStructParent->Name ); - } - else - { - result.append( ast->Name ); - } - - if ( ast->Params ) - result.append_fmt( "( %S )", ast->Params.to_string() ); - else - result.append_fmt( "()" ); - - if ( ast->Body ) - result.append_fmt( " = %S", ast->Body.to_string() ); - - if ( ast->InlineCmt ) - result.append_fmt( "; // %S\n", ast->InlineCmt->Content ); - else - result.append( ";\n" ); -} - -String CodeClass::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Class : - to_string_def( result ); - break; - case Class_Fwd : - to_string_fwd( result ); - break; - } - return result; -} - -void CodeClass::to_string_def( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - result.append( "class " ); - - if ( ast->Attributes ) - { - result.append_fmt( "%S ", ast->Attributes.to_string() ); - } - - if ( ast->ParentType ) - { - char const* access_level = to_str( ast->ParentAccess ); - - result.append_fmt( "%S : %s %S", ast->Name, access_level, ast->ParentType.to_string() ); - - CodeType interface = ast->ParentType->Next->cast(); - if ( interface ) - result.append( "\n" ); - - while ( interface ) - { - result.append_fmt( ", %S", interface.to_string() ); - interface = interface->Next ? interface->Next->cast() : CodeType { nullptr }; - } - } - else if ( ast->Name ) - { - result.append( ast->Name ); - } - - if ( ast->InlineCmt ) - { - result.append_fmt( " // %S", ast->InlineCmt->Content ); - } - - result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() ); - - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - result.append( ";\n" ); -} - -void CodeClass::to_string_fwd( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes ) - result.append_fmt( "class %S %S", ast->Attributes.to_string(), ast->Name ); - - else - result.append_fmt( "class %S", ast->Name ); - - // Check if it can have an end-statement - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - { - if ( ast->InlineCmt ) - result.append_fmt( "; // %S\n", ast->InlineCmt->Content ); - else - result.append( ";\n" ); - } -} - -String CodeDefine::to_string() -{ - return String::fmt_buf( GlobalAllocator, "#define %S %S\n", ast->Name, ast->Content ); -} - -void CodeDefine::to_string( String& result ) -{ - result.append_fmt( "#define %S %S\n", ast->Name, ast->Content ); -} - -String CodeDestructor::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Destructor : - to_string_def( result ); - break; - case Destructor_Fwd : - to_string_fwd( result ); - break; - } - return result; -} - -void CodeDestructor::to_string_def( String& result ) -{ - if ( ast->Name ) - { - result.append_fmt( "%S()", ast->Name ); - } - else if ( ast->Specs ) - { - if ( ast->Specs.has( ESpecifier::Virtual ) ) - result.append_fmt( "virtual ~%S()", ast->Parent->Name ); - else - result.append_fmt( "~%S()", ast->Parent->Name ); - } - else - result.append_fmt( "~%S()", ast->Parent->Name ); - - result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() ); -} - -void CodeDestructor::to_string_fwd( String& result ) -{ - if ( ast->Specs ) - { - if ( ast->Specs.has( ESpecifier::Virtual ) ) - result.append_fmt( "virtual ~%S();\n", ast->Parent->Name ); - else - result.append_fmt( "~%S()", ast->Parent->Name ); - - if ( ast->Specs.has( ESpecifier::Pure ) ) - result.append( " = 0;" ); - else if ( ast->Body ) - result.append_fmt( " = %S;", ast->Body.to_string() ); - } - else - result.append_fmt( "~%S();", ast->Parent->Name ); - - if ( ast->InlineCmt ) - result.append_fmt( " %S", ast->InlineCmt->Content ); - else - result.append( "\n" ); -} - -String CodeEnum::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Enum : - to_string_def( result ); - break; - case Enum_Fwd : - to_string_fwd( result ); - break; - case Enum_Class : - to_string_class_def( result ); - break; - case Enum_Class_Fwd : - to_string_class_fwd( result ); - break; - } - return result; -} - -void CodeEnum::to_string_def( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes || ast->UnderlyingType ) - { - result.append( "enum " ); - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - if ( ast->UnderlyingType ) - result.append_fmt( "%S : %S\n{\n%S\n}", ast->Name, ast->UnderlyingType.to_string(), ast->Body.to_string() ); - - else - result.append_fmt( "%S\n{\n%S\n}", ast->Name, ast->Body.to_string() ); - } - else - result.append_fmt( "enum %S\n{\n%S\n}", ast->Name, ast->Body.to_string() ); - - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - result.append( ";\n" ); -} - -void CodeEnum::to_string_fwd( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - result.append_fmt( "enum %S : %S", ast->Name, ast->UnderlyingType.to_string() ); - - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - { - if ( ast->InlineCmt ) - result.append_fmt( "; %S", ast->InlineCmt->Content ); - else - result.append( ";\n" ); - } -} - -void CodeEnum::to_string_class_def( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes || ast->UnderlyingType ) - { - result.append( "enum class " ); - - if ( ast->Attributes ) - { - result.append_fmt( "%S ", ast->Attributes.to_string() ); - } - - if ( ast->UnderlyingType ) - { - result.append_fmt( "%S : %S\n{\n%S\n}", ast->Name, ast->UnderlyingType.to_string(), ast->Body.to_string() ); - } - else - { - result.append_fmt( "%S\n{\n%S\n}", ast->Name, ast->Body.to_string() ); - } - } - else - { - result.append_fmt( "enum class %S\n{\n%S\n}", ast->Body.to_string() ); - } - - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - result.append( ";\n" ); -} - -void CodeEnum::to_string_class_fwd( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - result.append( "enum class " ); - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - result.append_fmt( "%S : %S", ast->Name, ast->UnderlyingType.to_string() ); - - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - { - if ( ast->InlineCmt ) - result.append_fmt( "; %S", ast->InlineCmt->Content ); - else - result.append( ";\n" ); - } -} - -String CodeExec::to_string() -{ - return ast->Content.duplicate( GlobalAllocator ); -} - -void CodeExtern::to_string( String& result ) -{ - if ( ast->Body ) - result.append_fmt( "extern \"%S\"\n{\n%S\n}\n", ast->Name, ast->Body.to_string() ); - else - result.append_fmt( "extern \"%S\"\n{}\n", ast->Name ); -} - -String CodeInclude::to_string() -{ - return String::fmt_buf( GlobalAllocator, "#include %S\n", ast->Content ); -} - -void CodeInclude::to_string( String& result ) -{ - result.append_fmt( "#include %S\n", ast->Content ); -} - -String CodeFriend::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeFriend::to_string( String& result ) -{ - result.append_fmt( "friend %S", ast->Declaration->to_string() ); - - if ( ast->Declaration->Type != ECode::Function && result[result.length() - 1] != ';' ) - { - result.append( ";" ); - } - - if ( ast->InlineCmt ) - result.append_fmt( " %S", ast->InlineCmt->Content ); - else - result.append( "\n" ); -} - -String CodeFn::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Function : - to_string_def( result ); - break; - case Function_Fwd : - to_string_fwd( result ); - break; - } - return result; -} - -void CodeFn::to_string_def( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export" ); - - if ( ast->Attributes ) - result.append_fmt( " %S ", ast->Attributes.to_string() ); - - b32 prefix_specs = false; - if ( ast->Specs ) - { - for ( SpecifierT spec : ast->Specs ) - { - if ( ! ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - - prefix_specs = true; - } - } - } - - if ( ast->Attributes || prefix_specs ) - result.append( "\n" ); - - if ( ast->ReturnType ) - result.append_fmt( "%S %S(", ast->ReturnType.to_string(), ast->Name ); - - else - result.append_fmt( "%S(", ast->Name ); - - if ( ast->Params ) - result.append_fmt( "%S)", ast->Params.to_string() ); - - else - result.append( ")" ); - - if ( ast->Specs ) - { - for ( SpecifierT spec : ast->Specs ) - { - if ( ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - } - } - } - - result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() ); -} - -void CodeFn::to_string_fwd( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - bool prefix_specs = false; - if ( ast->Specs ) - { - for ( SpecifierT spec : ast->Specs ) - { - if ( ! ESpecifier::is_trailing( spec ) || ! ( spec != ESpecifier::Pure ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - - prefix_specs = true; - } - } - } - - if ( ast->Attributes || prefix_specs ) - { - result.append( "\n" ); - } - - if ( ast->ReturnType ) - result.append_fmt( "%S %S(", ast->ReturnType.to_string(), ast->Name ); - - else - result.append_fmt( "%S(", ast->Name ); - - if ( ast->Params ) - result.append_fmt( "%S)", ast->Params.to_string() ); - - else - result.append( ")" ); - - if ( ast->Specs ) - { - for ( SpecifierT spec : ast->Specs ) - { - if ( ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - } - } - } - - if ( ast->Specs && ast->Specs.has( ESpecifier::Pure ) >= 0 ) - result.append( " = 0;" ); - else if ( ast->Body ) - result.append_fmt( " = %S;", ast->Body.to_string() ); - - if ( ast->InlineCmt ) - result.append_fmt( "; %S", ast->InlineCmt->Content ); - else - result.append( ";\n" ); -} - -String CodeModule::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeModule::to_string( String& result ) -{ - if ( ( ( u32( ModuleFlag::Export ) & u32( ast->ModuleFlags ) ) == u32( ModuleFlag::Export ) ) ) - result.append( "export " ); - - if ( ( ( u32( ModuleFlag::Import ) & u32( ast->ModuleFlags ) ) == u32( ModuleFlag::Import ) ) ) - result.append( "import " ); - - result.append_fmt( "%S;\n", ast->Name ); -} - -String CodeNS::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeNS::to_string( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - result.append_fmt( "namespace %S\n{\n%S\n}\n", ast->Name, ast->Body.to_string() ); -} - -String CodeOperator::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Operator : - case Operator_Member : - to_string_def( result ); - break; - case Operator_Fwd : - case Operator_Member_Fwd : - to_string_fwd( result ); - break; - } - return result; -} - -void CodeOperator::to_string_def( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - if ( ast->Specs ) - { - for ( SpecifierT spec : ast->Specs ) - { - if ( ! ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - } - } - } - - if ( ast->Attributes || ast->Specs ) - { - result.append( "\n" ); - } - - if ( ast->ReturnType ) - result.append_fmt( "%S %S (", ast->ReturnType.to_string(), ast->Name ); - - if ( ast->Params ) - result.append_fmt( "%S)", ast->Params.to_string() ); - - else - result.append( ")" ); - - if ( ast->Specs ) - { - for ( SpecifierT spec : ast->Specs ) - { - if ( ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - } - } - } - - result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() ); -} - -void CodeOperator::to_string_fwd( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes ) - result.append_fmt( "%S\n", ast->Attributes.to_string() ); - - if ( ast->Specs ) - { - for ( SpecifierT spec : ast->Specs ) - { - if ( ! ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - } - } - } - - if ( ast->Attributes || ast->Specs ) - { - result.append( "\n" ); - } - - result.append_fmt( "%S %S (", ast->ReturnType.to_string(), ast->Name ); - - if ( ast->Params ) - result.append_fmt( "%S)", ast->Params.to_string() ); - - else - result.append_fmt( ")" ); - - if ( ast->Specs ) - { - for ( SpecifierT spec : ast->Specs ) - { - if ( ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - } - } - } - - if ( ast->InlineCmt ) - result.append_fmt( "; %S", ast->InlineCmt->Content ); - else - result.append( ";\n" ); -} - -String CodeOpCast::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Operator_Cast : - to_string_def( result ); - break; - case Operator_Cast_Fwd : - to_string_fwd( result ); - break; - } - return result; -} - -void CodeOpCast::to_string_def( String& result ) -{ - if ( ast->Specs ) - { - // TODO : Add support for specifies before the operator keyword - - if ( ast->Name && ast->Name.length() ) - result.append_fmt( "%Soperator %S()", ast->Name, ast->ValueType.to_string() ); - else - result.append_fmt( "operator %S()", ast->ValueType.to_string() ); - - for ( SpecifierT spec : ast->Specs ) - { - if ( ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %.*s", spec_str.Len, spec_str.Ptr ); - } - } - - result.append_fmt( "\n{\n%S\n}\n", ast->Body.to_string() ); - return; - } - - if ( ast->Name && ast->Name.length() ) - result.append_fmt( "%Soperator %S()\n{\n%S\n}\n", ast->Name, ast->ValueType.to_string(), ast->Body.to_string() ); - else - result.append_fmt( "operator %S()\n{\n%S\n}\n", ast->ValueType.to_string(), ast->Body.to_string() ); -} - -void CodeOpCast::to_string_fwd( String& result ) -{ - if ( ast->Specs ) - { - // TODO : Add support for specifies before the operator keyword - - result.append_fmt( "operator %S()", ast->ValueType.to_string() ); - - for ( SpecifierT spec : ast->Specs ) - { - if ( ESpecifier::is_trailing( spec ) ) - { - StrC spec_str = ESpecifier::to_str( spec ); - result.append_fmt( " %*s", spec_str.Len, spec_str.Ptr ); - } - } - - if ( ast->InlineCmt ) - result.append_fmt( "; %S", ast->InlineCmt->Content ); - else - result.append( ";\n" ); - return; - } - - if ( ast->InlineCmt ) - result.append_fmt( "operator %S(); %S", ast->ValueType.to_string() ); - else - result.append_fmt( "operator %S();\n", ast->ValueType.to_string() ); -} - -String CodeParam::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeParam::to_string( String& result ) -{ - if ( ast->Macro ) - { - // Related to parsing: ( , ... ) - result.append( ast->Macro.ast->Content ); - // Could also be: ( , ... ) - } - - if ( ast->Name ) - { - if ( ast->ValueType.ast == nullptr ) - result.append_fmt( " %S", ast->Name ); - else - result.append_fmt( " %S %S", ast->ValueType.to_string(), ast->Name ); - } - else if ( ast->ValueType ) - result.append_fmt( " %S", ast->ValueType.to_string() ); - - if ( ast->Value ) - result.append_fmt( " = %S", ast->Value.to_string() ); - - if ( ast->NumEntries - 1 > 0 ) - { - for ( CodeParam param : ast->Next ) - { - result.append_fmt( ", %S", param.to_string() ); - } - } -} - -String CodePreprocessCond::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Preprocess_If : - to_string_if( result ); - break; - case Preprocess_IfDef : - to_string_ifdef( result ); - break; - case Preprocess_IfNotDef : - to_string_ifndef( result ); - break; - case Preprocess_ElIf : - to_string_elif( result ); - break; - case Preprocess_Else : - to_string_else( result ); - break; - case Preprocess_EndIf : - to_string_endif( result ); - break; - } - return result; -} - -void CodePreprocessCond::to_string_if( String& result ) -{ - result.append_fmt( "#if %S\n", ast->Content ); -} - -void CodePreprocessCond::to_string_ifdef( String& result ) -{ - result.append_fmt( "#ifdef %S\n", ast->Content ); -} - -void CodePreprocessCond::to_string_ifndef( String& result ) -{ - result.append_fmt( "#ifndef %S\n", ast->Content ); -} - -void CodePreprocessCond::to_string_elif( String& result ) -{ - result.append_fmt( "#elif %S\n", ast->Content ); -} - -void CodePreprocessCond::to_string_else( String& result ) -{ - result.append_fmt( "#else\n" ); -} - -void CodePreprocessCond::to_string_endif( String& result ) -{ - result.append_fmt( "#endif\n" ); -} - -String CodePragma::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodePragma::to_string( String& result ) -{ - result.append_fmt( "#pragma %S\n", ast->Content ); -} - -String CodeSpecifiers::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeSpecifiers::to_string( String& result ) -{ - s32 idx = 0; - s32 left = ast->NumEntries; - while ( left-- ) - { - StrC spec = ESpecifier::to_str( ast->ArrSpecs[idx] ); - result.append_fmt( "%.*s ", spec.Len, spec.Ptr ); - idx++; - } -} - -String CodeStruct::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Struct : - to_string_def( result ); - break; - case Struct_Fwd : - to_string_fwd( result ); - break; - } - return result; -} - -void CodeStruct::to_string_def( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - result.append( "struct " ); - - if ( ast->Attributes ) - { - result.append_fmt( "%S ", ast->Attributes.to_string() ); - } - - if ( ast->ParentType ) - { - char const* access_level = to_str( ast->ParentAccess ); - - result.append_fmt( "%S : %s %S", ast->Name, access_level, ast->ParentType.to_string() ); - - CodeType interface = ast->ParentType->Next->cast(); - if ( interface ) - result.append( "\n" ); - - while ( interface ) - { - result.append_fmt( ", %S", interface.to_string() ); - interface = interface->Next ? interface->Next->cast() : CodeType { nullptr }; - } - } - else if ( ast->Name ) - { - result.append( ast->Name ); - } - - if ( ast->InlineCmt ) - { - result.append_fmt( " // %S", ast->InlineCmt->Content ); - } - - result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() ); - - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - result.append( ";\n" ); -} - -void CodeStruct::to_string_fwd( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes ) - result.append_fmt( "struct %S %S", ast->Attributes.to_string(), ast->Name ); - - else - result.append_fmt( "struct %S", ast->Name ); - - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - { - if ( ast->InlineCmt ) - result.append_fmt( "; %S", ast->InlineCmt->Content ); - else - result.append( ";\n" ); - } -} - -String CodeTemplate::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeTemplate::to_string( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Params ) - result.append_fmt( "template< %S >\n%S", ast->Params.to_string(), ast->Declaration.to_string() ); - else - result.append_fmt( "template<>\n%S", ast->Declaration.to_string() ); -} - -String CodeTypedef::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeTypedef::to_string( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - result.append( "typedef " ); - - // Determines if the typedef is a function typename - if ( ast->UnderlyingType->ReturnType ) - result.append( ast->UnderlyingType.to_string() ); - else - result.append_fmt( "%S %S", ast->UnderlyingType.to_string(), ast->Name ); - - if ( ast->UnderlyingType->Type == ECode::Typename && ast->UnderlyingType->ArrExpr ) - { - result.append_fmt( "[ %S ];", ast->UnderlyingType->ArrExpr->to_string() ); - - AST* next_arr_expr = ast->UnderlyingType->ArrExpr->Next; - while ( next_arr_expr ) - { - result.append_fmt( "[ %S ];", next_arr_expr->to_string() ); - next_arr_expr = next_arr_expr->Next; - } - } - else - { - result.append( ";" ); - } - - if ( ast->InlineCmt ) - result.append_fmt( " %S", ast->InlineCmt->Content ); - else - result.append( "\n" ); -} - -String CodeType::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeType::to_string( String& result ) -{ -#if GEN_USE_NEW_TYPENAME_PARSING - if ( ast->ReturnType && ast->Params ) - { - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - else - { - if ( ast->Specs ) - result.append_fmt( "%S ( %S ) ( %S ) %S", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string(), ast->Specs.to_string() ); - else - result.append_fmt( "%S ( %S ) ( %S )", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string() ); - } - - break; - } -#else - if ( ast->ReturnType && ast->Params ) - { - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - else - { - if ( ast->Specs ) - result.append_fmt( "%S %S ( %S ) %S", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string(), ast->Specs.to_string() ); - else - result.append_fmt( "%S %S ( %S )", ast->ReturnType.to_string(), ast->Name, ast->Params.to_string() ); - } - - return; - } -#endif - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - if ( ast->Specs ) - result.append_fmt( "%S %S", ast->Name, ast->Specs.to_string() ); - else - result.append_fmt( "%S", ast->Name ); - - if ( ast->IsParamPack ) - result.append( "..." ); -} - -String CodeUnion::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeUnion::to_string( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - result.append( "union " ); - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - if ( ast->Name ) - { - result.append_fmt( "%S\n{\n%S\n}", ast->Name, ast->Body.to_string() ); - } - else - { - // Anonymous union - result.append_fmt( "\n{\n%S\n}", ast->Body.to_string() ); - } - - if ( ast->Parent.ast == nullptr || ( ast->Parent->Type != ECode::Typedef && ast->Parent->Type != ECode::Variable ) ) - result.append( ";\n" ); -} - -String CodeUsing::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - switch ( ast->Type ) - { - using namespace ECode; - case Using : - to_string( result ); - break; - case Using_Namespace : - to_string_ns( result ); - break; - } - return result; -} - -void CodeUsing::to_string( String& result ) -{ - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Attributes.to_string() ); - - if ( ast->UnderlyingType ) - { - result.append_fmt( "using %S = %S", ast->Name, ast->UnderlyingType.to_string() ); - - if ( ast->UnderlyingType->ArrExpr ) - { - result.append_fmt( "[ %S ]", ast->UnderlyingType->ArrExpr.to_string() ); - - AST* next_arr_expr = ast->UnderlyingType->ArrExpr->Next; - while ( next_arr_expr ) - { - result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); - next_arr_expr = next_arr_expr->Next; - } - } - - result.append( ";" ); - } - else - result.append_fmt( "using %S;", ast->Name ); - - if ( ast->InlineCmt ) - result.append_fmt( " %S\n", ast->InlineCmt->Content ); - else - result.append( "\n" ); -} - -void CodeUsing::to_string_ns( String& result ) -{ - if ( ast->InlineCmt ) - result.append_fmt( "using namespace $S; %S", ast->Name, ast->InlineCmt->Content ); - else - result.append_fmt( "using namespace %s;\n", ast->Name ); -} - -String CodeVar::to_string() -{ - String result = String::make( GlobalAllocator, "" ); - to_string( result ); - return result; -} - -void CodeVar::to_string( String& result ) -{ - if ( ast->Parent && ast->Parent->Type == ECode::Variable ) - { - // Its a comma-separated variable ( a NextVar ) - - if ( ast->Specs ) - result.append_fmt( "%S ", ast->Specs.to_string() ); - - result.append( ast->Name ); - - if ( ast->ValueType && ast->ValueType->ArrExpr ) - { - result.append_fmt( "[ %S ]", ast->ValueType->ArrExpr.to_string() ); - - AST* next_arr_expr = ast->ValueType->ArrExpr->Next; - while ( next_arr_expr ) - { - result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); - next_arr_expr = next_arr_expr->Next; - } - } - - if ( ast->Value ) - result.append_fmt( " = %S", ast->Value.to_string() ); - - // Keep the chain going... - if ( ast->NextVar ) - result.append_fmt( ", %S", ast->NextVar.to_string() ); - - return; - } - - if ( bitfield_is_equal( u32, ast->ModuleFlags, ModuleFlag::Export ) ) - result.append( "export " ); - - if ( ast->Attributes || ast->Specs ) - { - if ( ast->Attributes ) - result.append_fmt( "%S ", ast->Specs.to_string() ); - - if ( ast->Specs ) - result.append_fmt( "%S\n", ast->Specs.to_string() ); - - result.append_fmt( "%S %S", ast->ValueType.to_string(), ast->Name ); - - if ( ast->ValueType->ArrExpr ) - { - result.append_fmt( "[ %S ]", ast->ValueType->ArrExpr.to_string() ); - - AST* next_arr_expr = ast->ValueType->ArrExpr->Next; - while ( next_arr_expr ) - { - result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); - next_arr_expr = next_arr_expr->Next; - } - } - - if ( ast->BitfieldSize ) - result.append_fmt( " : %S", ast->BitfieldSize.to_string() ); - - if ( ast->Value ) - result.append_fmt( " = %S", ast->Value.to_string() ); - - if ( ast->NextVar ) - result.append_fmt( ", %S", ast->NextVar.to_string() ); - - if ( ast->InlineCmt ) - result.append_fmt( "; %S", ast->InlineCmt->Content ); - else - result.append( ";\n" ); - - return; - } - - if ( ast->BitfieldSize ) - result.append_fmt( "%S %S : %S", ast->ValueType.to_string(), ast->Name, ast->BitfieldSize.to_string() ); - - else if ( ast->ValueType->ArrExpr ) - { - result.append_fmt( "%S %S[ %S ]", ast->ValueType.to_string(), ast->Name, ast->ValueType->ArrExpr.to_string() ); - - AST* next_arr_expr = ast->ValueType->ArrExpr->Next; - while ( next_arr_expr ) - { - result.append_fmt( "[ %S ]", next_arr_expr->to_string() ); - next_arr_expr = next_arr_expr->Next; - } - } - - else - result.append_fmt( "%S %S", ast->ValueType.to_string(), ast->Name ); - - if ( ast->Value ) - result.append_fmt( " = %S", ast->Value.to_string() ); - - if ( ast->NextVar ) - result.append_fmt( ", %S", ast->NextVar.to_string() ); - - result.append( ";" ); - - if ( ast->InlineCmt ) - result.append_fmt( " %S", ast->InlineCmt->Content ); - else - result.append( "\n" ); -} - -#pragma endregion AST - -#pragma region Interface - -namespace parser -{ - internal void init(); - internal void deinit(); -} - -internal void* Global_Allocator_Proc( void* allocator_data, AllocType type, sw size, sw alignment, void* old_memory, sw old_size, u64 flags ) -{ - Arena* last = &Global_AllocatorBuckets.back(); - - switch ( type ) - { - case EAllocation_ALLOC : - { - if ( ( last->TotalUsed + size ) > last->TotalSize ) - { - Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize ); - - if ( bucket.PhysicalStart == nullptr ) - GEN_FATAL( "Failed to create bucket for Global_AllocatorBuckets" ); - - if ( ! Global_AllocatorBuckets.append( bucket ) ) - GEN_FATAL( "Failed to append bucket to Global_AllocatorBuckets" ); - - last = &Global_AllocatorBuckets.back(); - } - - return alloc_align( *last, size, alignment ); - } - case EAllocation_FREE : - { - // Doesn't recycle. - } - break; - case EAllocation_FREE_ALL : - { - // Memory::cleanup instead. - } - break; - case EAllocation_RESIZE : - { - if ( last->TotalUsed + size > last->TotalSize ) - { - Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize ); - - if ( bucket.PhysicalStart == nullptr ) - GEN_FATAL( "Failed to create bucket for Global_AllocatorBuckets" ); - - if ( ! Global_AllocatorBuckets.append( bucket ) ) - GEN_FATAL( "Failed to append bucket to Global_AllocatorBuckets" ); - - last = &Global_AllocatorBuckets.back(); - } - - void* result = alloc_align( last->Backing, size, alignment ); - - if ( result != nullptr && old_memory != nullptr ) - { - mem_copy( result, old_memory, old_size ); - } - - return result; - } - } - - return nullptr; -} - -internal void define_constants() -{ - Code::Global = make_code(); - Code::Global->Name = get_cached_string( txt( "Global Code" ) ); - Code::Global->Content = Code::Global->Name; - - Code::Invalid = make_code(); - Code::Invalid.set_global(); - - t_empty = (CodeType)make_code(); - t_empty->Type = ECode::Typename; - t_empty->Name = get_cached_string( txt( "" ) ); - t_empty.set_global(); - - access_private = make_code(); - access_private->Type = ECode::Access_Private; - access_private->Name = get_cached_string( txt( "private:\n" ) ); - access_private.set_global(); - - access_protected = make_code(); - access_protected->Type = ECode::Access_Protected; - access_protected->Name = get_cached_string( txt( "protected:\n" ) ); - access_protected.set_global(); - - access_public = make_code(); - access_public->Type = ECode::Access_Public; - access_public->Name = get_cached_string( txt( "public:\n" ) ); - access_public.set_global(); - - attrib_api_export = def_attributes( code( GEN_API_Export_Code ) ); - attrib_api_export.set_global(); - - attrib_api_import = def_attributes( code( GEN_API_Import_Code ) ); - attrib_api_import.set_global(); - - module_global_fragment = make_code(); - module_global_fragment->Type = ECode::Untyped; - module_global_fragment->Name = get_cached_string( txt( "module;" ) ); - module_global_fragment->Content = module_global_fragment->Name; - module_global_fragment.set_global(); - - module_private_fragment = make_code(); - module_private_fragment->Type = ECode::Untyped; - module_private_fragment->Name = get_cached_string( txt( "module : private;" ) ); - module_private_fragment->Content = module_private_fragment->Name; - module_private_fragment.set_global(); - - fmt_newline = make_code(); - fmt_newline->Type = ECode::NewLine; - fmt_newline.set_global(); - - pragma_once = (CodePragma)make_code(); - pragma_once->Type = ECode::Preprocess_Pragma; - pragma_once->Name = get_cached_string( txt( "once" ) ); - pragma_once->Content = pragma_once->Name; - pragma_once.set_global(); - - param_varadic = (CodeType)make_code(); - param_varadic->Type = ECode::Parameters; - param_varadic->Name = get_cached_string( txt( "..." ) ); - param_varadic->ValueType = t_empty; - param_varadic.set_global(); - - preprocess_else = (CodePreprocessCond)make_code(); - preprocess_else->Type = ECode::Preprocess_Else; - preprocess_else.set_global(); - - preprocess_endif = (CodePreprocessCond)make_code(); - preprocess_endif->Type = ECode::Preprocess_EndIf; - preprocess_endif.set_global(); - -#define def_constant_code_type( Type_ ) \ - t_##Type_ = def_type( name( Type_ ) ); \ - t_##Type_.set_global(); - - def_constant_code_type( auto ); - def_constant_code_type( void ); - def_constant_code_type( int ); - def_constant_code_type( bool ); - def_constant_code_type( char ); - def_constant_code_type( wchar_t ); - def_constant_code_type( class ); - def_constant_code_type( typename ); - -#ifdef GEN_DEFINE_LIBRARY_CODE_CONSTANTS - t_b32 = def_type( name( b32 ) ); - - def_constant_code_type( s8 ); - def_constant_code_type( s16 ); - def_constant_code_type( s32 ); - def_constant_code_type( s64 ); - - def_constant_code_type( u8 ); - def_constant_code_type( u16 ); - def_constant_code_type( u32 ); - def_constant_code_type( u64 ); - - def_constant_code_type( sw ); - def_constant_code_type( uw ); - - def_constant_code_type( f32 ); - def_constant_code_type( f64 ); -#endif -#undef def_constant_code_type - - -#define def_constant_spec( Type_, ... ) \ - spec_##Type_ = def_specifiers( num_args( __VA_ARGS__ ), __VA_ARGS__ ); \ - spec_##Type_.set_global(); - -#pragma push_macro( "internal" ) -#pragma push_macro( "local_persist" ) -#pragma push_macro( "neverinline" ) -#undef internal -#undef local_persist -#undef neverinline - def_constant_spec( const, ESpecifier::Const ); - def_constant_spec( consteval, ESpecifier::Consteval ); - def_constant_spec( constexpr, ESpecifier::Constexpr ); - def_constant_spec( constinit, ESpecifier::Constinit ); - def_constant_spec( extern_linkage, ESpecifier::External_Linkage ); - def_constant_spec( final, ESpecifier::Final ); - def_constant_spec( gb_inline, ESpecifier::gb_inline ); - def_constant_spec( gb_global, ESpecifier::gb_global ); - def_constant_spec( inline, ESpecifier::Inline ); - def_constant_spec( gb_internal, ESpecifier::gb_internal ); - def_constant_spec( local_persist, ESpecifier::Local_Persist ); - def_constant_spec( mutable, ESpecifier::Mutable ); - def_constant_spec( neverinline, ESpecifier::NeverInline ); - def_constant_spec( noexcept, ESpecifier::NoExceptions ); - def_constant_spec( override, ESpecifier::Override ); - def_constant_spec( ptr, ESpecifier::Ptr ); - def_constant_spec( pure, ESpecifier::Pure ) def_constant_spec( ref, ESpecifier::Ref ); - def_constant_spec( register, ESpecifier::Register ); - def_constant_spec( rvalue, ESpecifier::RValue ); - def_constant_spec( static_member, ESpecifier::Static ); - def_constant_spec( gb_thread_local, ESpecifier::gb_thread_local ); - def_constant_spec( virtual, ESpecifier::Virtual ); - def_constant_spec( volatile, ESpecifier::Volatile ) - - spec_local_persist = def_specifiers( 1, ESpecifier::Local_Persist ); - spec_local_persist.set_global(); - -#pragma pop_macro( "internal" ) -#pragma pop_macro( "local_persist" ) -#pragma pop_macro( "neverinline" ) - -#undef def_constant_spec -} - -void init() -{ - // Setup global allocator - { - GlobalAllocator = AllocatorInfo { &Global_Allocator_Proc, nullptr }; - - Global_AllocatorBuckets = Array::init_reserve( heap(), 128 ); - - if ( Global_AllocatorBuckets == nullptr ) - GEN_FATAL( "Failed to reserve memory for Global_AllocatorBuckets" ); - - Arena bucket = Arena::init_from_allocator( heap(), Global_BucketSize ); - - if ( bucket.PhysicalStart == nullptr ) - GEN_FATAL( "Failed to create first bucket for Global_AllocatorBuckets" ); - - Global_AllocatorBuckets.append( bucket ); - } - - // Setup the arrays - { - CodePools = Array::init_reserve( Allocator_DataArrays, InitSize_DataArrays ); - - if ( CodePools == nullptr ) - GEN_FATAL( "gen::init: Failed to initialize the CodePools array" ); - - StringArenas = Array::init_reserve( Allocator_DataArrays, InitSize_DataArrays ); - - if ( StringArenas == nullptr ) - GEN_FATAL( "gen::init: Failed to initialize the StringArenas array" ); - } - - // Setup the code pool and code entries arena. - { - Pool code_pool = Pool::init( Allocator_CodePool, CodePool_NumBlocks, sizeof( AST ) ); - - if ( code_pool.PhysicalStart == nullptr ) - GEN_FATAL( "gen::init: Failed to initialize the code pool" ); - - CodePools.append( code_pool ); - - LexArena = Arena::init_from_allocator( Allocator_Lexer, LexAllocator_Size ); - - Arena string_arena = Arena::init_from_allocator( Allocator_StringArena, SizePer_StringArena ); - - if ( string_arena.PhysicalStart == nullptr ) - GEN_FATAL( "gen::init: Failed to initialize the string arena" ); - - StringArenas.append( string_arena ); - } - - // Setup the hash tables - { - StringCache = StringTable::init( Allocator_StringTable ); - - if ( StringCache.Entries == nullptr ) - GEN_FATAL( "gen::init: Failed to initialize the StringCache" ); - } - - // Preprocessor Defines - PreprocessorDefines = Array::init_reserve( GlobalAllocator, kilobytes( 1 ) ); - - define_constants(); - parser::init(); -} - -void deinit() -{ - uw index = 0; - uw left = CodePools.num(); - do - { - Pool* code_pool = &CodePools[index]; - code_pool->free(); - index++; - } while ( left--, left ); - - index = 0; - left = StringArenas.num(); - do - { - Arena* string_arena = &StringArenas[index]; - string_arena->free(); - index++; - } while ( left--, left ); - - StringCache.destroy(); - - CodePools.free(); - StringArenas.free(); - - LexArena.free(); - - PreprocessorDefines.free(); - - index = 0; - left = Global_AllocatorBuckets.num(); - do - { - Arena* bucket = &Global_AllocatorBuckets[index]; - bucket->free(); - index++; - } while ( left--, left ); - - Global_AllocatorBuckets.free(); - parser::deinit(); -} - -void reset() -{ - s32 index = 0; - s32 left = CodePools.num(); - do - { - Pool* code_pool = &CodePools[index]; - code_pool->clear(); - index++; - } while ( left--, left ); - - index = 0; - left = StringArenas.num(); - do - { - Arena* string_arena = &StringArenas[index]; - string_arena->TotalUsed = 0; - ; - index++; - } while ( left--, left ); - - StringCache.clear(); - - define_constants(); -} - -AllocatorInfo get_string_allocator( s32 str_length ) -{ - Arena* last = &StringArenas.back(); - - uw size_req = str_length + sizeof( String::Header ) + sizeof( char* ); - - if ( last->TotalUsed + size_req > last->TotalSize ) - { - Arena new_arena = Arena::init_from_allocator( Allocator_StringArena, SizePer_StringArena ); - - if ( ! StringArenas.append( new_arena ) ) - GEN_FATAL( "gen::get_string_allocator: Failed to allocate a new string arena" ); - - last = &StringArenas.back(); - } - - return *last; -} - -// Will either make or retrive a code string. -StringCached get_cached_string( StrC str ) -{ - s32 hash_length = str.Len > kilobytes( 1 ) ? kilobytes( 1 ) : str.Len; - u64 key = crc32( str.Ptr, hash_length ); - { - StringCached* result = StringCache.get( key ); - - if ( result ) - return *result; - } - - String result = String::make( get_string_allocator( str.Len ), str ); - StringCache.set( key, result ); - - return result; -} - -// Used internally to retireve a Code object form the CodePool. -Code make_code() -{ - Pool* allocator = &CodePools.back(); - if ( allocator->FreeList == nullptr ) - { - Pool code_pool = Pool::init( Allocator_CodePool, CodePool_NumBlocks, sizeof( AST ) ); - - if ( code_pool.PhysicalStart == nullptr ) - GEN_FATAL( "gen::make_code: Failed to allocate a new code pool - CodePool allcoator returned nullptr." ); - - if ( ! CodePools.append( code_pool ) ) - GEN_FATAL( "gen::make_code: Failed to allocate a new code pool - CodePools failed to append new pool." ); - - allocator = &CodePools.back(); - } - - Code result { rcast( AST*, alloc( *allocator, sizeof( AST ) ) ) }; - mem_set( result.ast, 0, sizeof( AST ) ); - // result->Type = ECode::Invalid; - - // result->Content = { nullptr }; - // result->Prev = { nullptr }; - // result->Next = { nullptr }; - // result->Token = nullptr; - // result->Parent = { nullptr }; - // result->Name = { nullptr }; - // result->Type = ECode::Invalid; - // result->ModuleFlags = ModuleFlag::Invalid; - // result->NumEntries = 0; - - return result; -} - -void set_allocator_data_arrays( AllocatorInfo allocator ) -{ - Allocator_DataArrays = allocator; -} - -void set_allocator_code_pool( AllocatorInfo allocator ) -{ - Allocator_CodePool = allocator; -} - -void set_allocator_lexer( AllocatorInfo allocator ) -{ - Allocator_Lexer = allocator; -} - -void set_allocator_string_arena( AllocatorInfo allocator ) -{ - Allocator_StringArena = allocator; -} - -void set_allocator_string_table( AllocatorInfo allocator ) -{ - Allocator_StringArena = allocator; -} - -#pragma region Upfront - -enum class OpValidateResult : u32 -{ - Fail, - Global, - Member -}; - -OpValidateResult operator__validate( OperatorT op, CodeParam params_code, CodeType ret_type, CodeSpecifiers specifier ) -{ - using namespace EOperator; - - if ( op == EOperator::Invalid ) - { - log_failure( "gen::def_operator: op cannot be invalid" ); - return OpValidateResult::Fail; - } - -#pragma region Helper Macros -#define check_params() \ - if ( ! params_code ) \ - { \ - log_failure( "gen::def_operator: params is null and operator%s requires it", to_str( op ) ); \ - return OpValidateResult::Fail; \ - } \ - if ( params_code->Type != ECode::Parameters ) \ - { \ - log_failure( "gen::def_operator: params is not of Parameters type - %s", params_code.debug_str() ); \ - return OpValidateResult::Fail; \ - } - -#define check_param_eq_ret() \ - if ( ! is_member_symbol && ! params_code->ValueType.is_equal( ret_type ) ) \ - { \ - log_failure( \ - "gen::def_operator: operator%s requires first parameter to equal return type\n" \ - "param types: %s\n" \ - "return type: %s", \ - to_str( op ).Ptr, \ - params_code.debug_str(), \ - ret_type.debug_str() \ - ); \ - return OpValidateResult::Fail; \ - } -#pragma endregion Helper Macros - - if ( ! ret_type ) - { - log_failure( "gen::def_operator: ret_type is null but is required by operator%s", to_str( op ) ); - } - - if ( ret_type->Type != ECode::Typename ) - { - log_failure( "gen::def_operator: ret_type is not of typename type - %s", ret_type.debug_str() ); - return OpValidateResult::Fail; - } - - bool is_member_symbol = false; - - switch ( op ) - { -#define specs( ... ) num_args( __VA_ARGS__ ), __VA_ARGS__ - case Assign : - check_params(); - - if ( params_code->NumEntries > 1 ) - { - log_failure( - "gen::def_operator: " - "operator%s does not support non-member definition (more than one parameter provided) - %s", - to_str( op ), - params_code.debug_str() - ); - return OpValidateResult::Fail; - } - - is_member_symbol = true; - break; - - case Assign_Add : - case Assign_Subtract : - case Assign_Multiply : - case Assign_Divide : - case Assign_Modulo : - case Assign_BAnd : - case Assign_BOr : - case Assign_BXOr : - case Assign_LShift : - case Assign_RShift : - check_params(); - - if ( params_code->NumEntries == 1 ) - is_member_symbol = true; - - else - check_param_eq_ret(); - - if ( params_code->NumEntries > 2 ) - { - log_failure( - "gen::def_operator: operator%s may not be defined with more than two parametes - param count; %d\n%s", - to_str( op ), - params_code->NumEntries, - params_code.debug_str() - ); - return OpValidateResult::Fail; - } - break; - - case Increment : - case Decrement : - // If its not set, it just means its a prefix member op. - if ( params_code ) - { - if ( params_code->Type != ECode::Parameters ) - { - log_failure( "gen::def_operator: operator%s params code provided is not of Parameters type - %s", to_str( op ), params_code.debug_str() ); - return OpValidateResult::Fail; - } - - switch ( params_code->NumEntries ) - { - case 1 : - if ( params_code->ValueType.is_equal( t_int ) ) - is_member_symbol = true; - - else - check_param_eq_ret(); - break; - - case 2 : - check_param_eq_ret(); - - if ( ! params_code.get( 1 ).is_equal( t_int ) ) - { - log_failure( - "gen::def_operator: " - "operator%s requires second parameter of non-member definition to be int for post-decrement", - to_str( op ) - ); - return OpValidateResult::Fail; - } - break; - - default : - log_failure( - "gen::def_operator: operator%s recieved unexpected number of parameters recived %d instead of 0-2", - to_str( op ), - params_code->NumEntries - ); - return OpValidateResult::Fail; - } - } - break; - - case Unary_Plus : - case Unary_Minus : - if ( ! params_code ) - is_member_symbol = true; - - else - { - if ( params_code->Type != ECode::Parameters ) - { - log_failure( "gen::def_operator: params is not of Parameters type - %s", params_code.debug_str() ); - return OpValidateResult::Fail; - } - - if ( params_code->ValueType.is_equal( ret_type ) ) - { - log_failure( - "gen::def_operator: " - "operator%s is non-member symbol yet first paramter does not equal return type\n" - "param type: %s\n" - "return type: %s\n", - params_code.debug_str(), - ret_type.debug_str() - ); - return OpValidateResult::Fail; - } - - if ( params_code->NumEntries > 1 ) - { - log_failure( - "gen::def_operator: operator%s may not have more than one parameter - param count: %d", - to_str( op ), - params_code->NumEntries - ); - return OpValidateResult::Fail; - } - } - break; - - case BNot : - { - // Some compilers let you do this... -#if 0 - if ( ! ret_type.is_equal( t_bool) ) - { - log_failure( "gen::def_operator: return type is not a boolean - %s", params_code.debug_str() ); - return OpValidateResult::Fail; - } -#endif - - if ( ! params_code ) - is_member_symbol = true; - - else - { - if ( params_code->Type != ECode::Parameters ) - { - log_failure( "gen::def_operator: params is not of Parameters type - %s", params_code.debug_str() ); - return OpValidateResult::Fail; - } - - if ( params_code->NumEntries > 1 ) - { - log_failure( - "gen::def_operator: operator%s may not have more than one parameter - param count: %d", - to_str( op ), - params_code->NumEntries - ); - return OpValidateResult::Fail; - } - } - break; - } - - case Add : - case Subtract : - case Multiply : - case Divide : - case Modulo : - case BAnd : - case BOr : - case BXOr : - case LShift : - case RShift : - check_params(); - - switch ( params_code->NumEntries ) - { - case 1 : - is_member_symbol = true; - break; - - case 2 : - if ( ! params_code->ValueType.is_equal( ret_type ) ) - { - log_failure( - "gen::def_operator: " - "operator%s is non-member symbol yet first paramter does not equal return type\n" - "param type: %s\n" - "return type: %s\n", - params_code.debug_str(), - ret_type.debug_str() - ); - return OpValidateResult::Fail; - } - break; - - default : - log_failure( - "gen::def_operator: operator%s recieved unexpected number of paramters recived %d instead of 0-2", - to_str( op ), - params_code->NumEntries - ); - return OpValidateResult::Fail; - } - break; - - case UnaryNot : - if ( ! params_code ) - is_member_symbol = true; - - else - { - if ( params_code->Type != ECode::Parameters ) - { - log_failure( "gen::def_operator: params is not of Parameters type - %s", params_code.debug_str() ); - return OpValidateResult::Fail; - } - - if ( params_code->NumEntries != 1 ) - { - log_failure( - "gen::def_operator: operator%s recieved unexpected number of paramters recived %d instead of 0-1", - to_str( op ), - params_code->NumEntries - ); - return OpValidateResult::Fail; - } - } - - if ( ! ret_type.is_equal( t_bool ) ) - { - log_failure( "gen::def_operator: operator%s return type must be of type bool - %s", to_str( op ), ret_type.debug_str() ); - return OpValidateResult::Fail; - } - break; - - case LAnd : - case LOr : - case LEqual : - case LNot : - case Lesser : - case Greater : - case LesserEqual : - case GreaterEqual : - check_params(); - - switch ( params_code->NumEntries ) - { - case 1 : - is_member_symbol = true; - break; - - case 2 : - break; - - default : - log_failure( - "gen::def_operator: operator%s recieved unexpected number of paramters recived %d instead of 1-2", - to_str( op ), - params_code->NumEntries - ); - return OpValidateResult::Fail; - } - break; - - case Indirection : - case AddressOf : - case MemberOfPointer : - if ( params_code && params_code->NumEntries > 1 ) - { - log_failure( - "gen::def_operator: operator%s recieved unexpected number of paramters recived %d instead of 0-1", - to_str( op ), - params_code->NumEntries - ); - return OpValidateResult::Fail; - } - else - { - is_member_symbol = true; - } - break; - - case PtrToMemOfPtr : - if ( params_code ) - { - log_failure( "gen::def_operator: operator%s expects no paramters - %s", to_str( op ), params_code.debug_str() ); - return OpValidateResult::Fail; - } - break; - - case Subscript : - case FunctionCall : - case Comma : - check_params(); - break; - - case New : - case Delete : - // This library doesn't support validating new and delete yet. - break; -#undef specs - } - - return is_member_symbol ? OpValidateResult::Member : OpValidateResult::Global; -#undef check_params -#undef check_ret_type -#undef check_param_eq_ret -} - -#pragma region Helper Marcos -// This snippet is used in nearly all the functions. -#define name_check( Context_, Name_ ) \ - { \ - if ( Name_.Len <= 0 ) \ - { \ - log_failure( "gen::" stringize( Context_ ) ": Invalid name length provided - %d", Name_.Len ); \ - return CodeInvalid; \ - } \ - \ - if ( Name_.Ptr == nullptr ) \ - { \ - log_failure( "gen::" stringize( Context_ ) ": name is null" ); \ - return CodeInvalid; \ - } \ - } - -#define null_check( Context_, Code_ ) \ - if ( ! Code_ ) \ - { \ - log_failure( "gen::" stringize( Context_ ) ": " stringize( Code_ ) " provided is null" ); \ - return CodeInvalid; \ - } - -#define null_or_invalid_check( Context_, Code_ ) \ - { \ - if ( ! Code_ ) \ - { \ - log_failure( "gen::" stringize( Context_ ) ": " stringize( Code_ ) " provided is null" ); \ - return CodeInvalid; \ - } \ - \ - if ( Code_->is_invalid() ) \ - { \ - log_failure( "gen::" stringize( Context_ ) ": " stringize( Code_ ) " provided is invalid" ); \ - return CodeInvalid; \ - } \ - } - -#define not_implemented( Context_ ) \ - log_failure( "gen::%s: This function is not implemented" ); \ - return CodeInvalid; -#pragma endregion Helper Marcos - -/* -The implementaiton of the upfront constructors involves doing three things: -* Validate the arguments given to construct the intended type of AST is valid. -* Construct said AST type. -* Lock the AST (set to readonly) and return the valid object. - -If any of the validation fails, it triggers a call to log_failure with as much info the give the user so that they can hopefully -identify the issue without having to debug too much (at least they can debug though...) - -The largest of the functions is related to operator overload definitions. -The library validates a good protion of their form and thus the argument processing for is quite a bit. -*/ -CodeAttributes def_attributes( StrC content ) -{ - if ( content.Len <= 0 || content.Ptr == nullptr ) - { - log_failure( "gen::def_attributes: Invalid attributes provided" ); - return CodeInvalid; - } - - Code result = make_code(); - result->Type = ECode::PlatformAttributes; - result->Name = get_cached_string( content ); - result->Content = result->Name; - - return (CodeAttributes)result; -} - -CodeComment def_comment( StrC content ) -{ - if ( content.Len <= 0 || content.Ptr == nullptr ) - { - log_failure( "gen::def_comment: Invalid comment provided:" ); - return CodeInvalid; - } - - static char line[MaxCommentLineLength]; - - String cmt_formatted = String::make_reserve( GlobalAllocator, kilobytes( 1 ) ); - char const* end = content.Ptr + content.Len; - char const* scanner = content.Ptr; - s32 curr = 0; - do - { - char const* next = scanner; - s32 length = 0; - while ( next != end && scanner[length] != '\n' ) - { - next = scanner + length; - length++; - } - length++; - - str_copy( line, scanner, length ); - cmt_formatted.append_fmt( "//%.*s", length, line ); - mem_set( line, 0, MaxCommentLineLength ); - - scanner += length; - } while ( scanner <= end ); - - if ( cmt_formatted.back() != '\n' ) - cmt_formatted.append( "\n" ); - - Code result = make_code(); - result->Type = ECode::Comment; - result->Name = get_cached_string( cmt_formatted ); - result->Content = result->Name; - - cmt_formatted.free(); - - return (CodeComment)result; -} - -CodeConstructor def_constructor( CodeParam params, Code initializer_list, Code body ) -{ - using namespace ECode; - - if ( params && params->Type != Parameters ) - { - log_failure( "gen::def_constructor: params must be of Parameters type - %s", params.debug_str() ); - return CodeInvalid; - } - - CodeConstructor result = (CodeConstructor)make_code(); - - if ( params ) - { - result->Params = params; - } - - if ( initializer_list ) - { - result->InitializerList = initializer_list; - } - - if ( body ) - { - switch ( body->Type ) - { - case Function_Body : - case Untyped : - break; - - default : - log_failure( "gen::def_constructor: body must be either of Function_Body or Untyped type - %s", body.debug_str() ); - return CodeInvalid; - } - - result->Type = Constructor; - result->Body = body; - } - else - { - result->Type = Constructor_Fwd; - } - - return result; -} - -CodeClass def_class( - StrC name, - Code body, - CodeType parent, - AccessSpec parent_access, - CodeAttributes attributes, - ModuleFlag mflags, - CodeType* interfaces, - s32 num_interfaces -) -{ - using namespace ECode; - - name_check( def_class, name ); - - if ( attributes && attributes->Type != PlatformAttributes ) - { - log_failure( "gen::def_class: attributes was not a 'PlatformAttributes' type: %s", attributes.debug_str() ); - return CodeInvalid; - } - - if ( parent && ( parent->Type != Class && parent->Type != Struct && parent->Type != Typename && parent->Type != Untyped ) ) - { - log_failure( "gen::def_class: parent provided is not type 'Class', 'Struct', 'Typeanme', or 'Untyped': %s", parent.debug_str() ); - return CodeInvalid; - } - - CodeClass result = (CodeClass)make_code(); - result->Name = get_cached_string( name ); - result->ModuleFlags = mflags; - - if ( body ) - { - switch ( body->Type ) - { - case Class_Body : - case Untyped : - break; - - default : - log_failure( "gen::def_class: body must be either of Class_Body or Untyped type - %s", body.debug_str() ); - return CodeInvalid; - } - - result->Type = Class; - result->Body = body; - result->Body->Parent = result; // TODO(Ed): Review this? - } - else - { - result->Type = Class_Fwd; - } - - if ( attributes ) - result->Attributes = attributes; - - if ( parent ) - { - result->ParentAccess = parent_access; - result->ParentType = parent; - } - - if ( interfaces ) - { - for ( s32 idx = 0; idx < num_interfaces; idx++ ) - { - result.add_interface( interfaces[idx] ); - } - } - - return result; -} - -CodeDefine def_define( StrC name, StrC content ) -{ - using namespace ECode; - - name_check( def_define, name ); - - // Defines can be empty definitions -#if 0 - if ( content.Len <= 0 || content.Ptr == nullptr ) - { - log_failure( "gen::def_define: Invalid value provided" ); - return CodeInvalid; - } -#endif - - CodeDefine result = (CodeDefine)make_code(); - result->Type = Preprocess_Define; - result->Name = get_cached_string( name ); - if ( content.Len <= 0 || content.Ptr == nullptr ) - { - result->Content = get_cached_string( txt( "" ) ); - } - else - result->Content = get_cached_string( content ); - - return result; -} - -CodeDestructor def_destructor( Code body, CodeSpecifiers specifiers ) -{ - using namespace ECode; - - if ( specifiers && specifiers->Type != Specifiers ) - { - log_failure( "gen::def_destructor: specifiers was not a 'Specifiers' type: %s", specifiers.debug_str() ); - return CodeInvalid; - } - - CodeDestructor result = (CodeDestructor)make_code(); - - if ( specifiers ) - result->Specs = specifiers; - - if ( body ) - { - switch ( body->Type ) - { - case Function_Body : - case Untyped : - break; - - default : - log_failure( "gen::def_destructor: body must be either of Function_Body or Untyped type - %s", body.debug_str() ); - return CodeInvalid; - } - - result->Type = Destructor; - result->Body = body; - } - else - { - result->Type = Destructor_Fwd; - } - - return result; -} - -CodeEnum def_enum( StrC name, Code body, CodeType type, EnumT specifier, CodeAttributes attributes, ModuleFlag mflags ) -{ - using namespace ECode; - - name_check( def_enum, name ); - - if ( type && type->Type != Typename ) - { - log_failure( "gen::def_enum: enum underlying type provided was not of type Typename: %s", type.debug_str() ); - return CodeInvalid; - } - - if ( attributes && attributes->Type != PlatformAttributes ) - { - log_failure( "gen::def_enum: attributes was not a 'PlatformAttributes' type: %s", attributes.debug_str() ); - return CodeInvalid; - } - - CodeEnum result = (CodeEnum)make_code(); - result->Name = get_cached_string( name ); - result->ModuleFlags = mflags; - - if ( body ) - { - switch ( body->Type ) - { - case Enum_Body : - case Untyped : - break; - - default : - log_failure( "gen::def_enum: body must be of Enum_Body or Untyped type %s", body.debug_str() ); - return CodeInvalid; - } - - result->Type = specifier == EnumClass ? Enum_Class : Enum; - - result->Body = body; - } - else - { - result->Type = specifier == EnumClass ? Enum_Class_Fwd : Enum_Fwd; - } - - if ( attributes ) - result->Attributes = attributes; - - if ( type ) - { - result->UnderlyingType = type; - } - else if ( result->Type != Enum_Class_Fwd && result->Type != Enum_Fwd ) - { - log_failure( "gen::def_enum: enum forward declaration must have an underlying type" ); - return CodeInvalid; - } - - return result; -} - -CodeExec def_execution( StrC content ) -{ - if ( content.Len <= 0 || content.Ptr == nullptr ) - { - log_failure( "gen::def_execution: Invalid execution provided" ); - return CodeInvalid; - } - - Code result = make_code(); - result->Type = ECode::Execution; - result->Name = get_cached_string( content ); - result->Content = result->Name; - - return (CodeExec)result; -} - -CodeExtern def_extern_link( StrC name, Code body ) -{ - using namespace ECode; - - name_check( def_extern_linkage, name ); - null_check( def_extern_linkage, body ); - - if ( body->Type != Extern_Linkage_Body && body->Type != Untyped ) - { - log_failure( "gen::def_extern_linkage: body is not of extern_linkage or untyped type %s", body->debug_str() ); - return CodeInvalid; - } - - CodeExtern result = (CodeExtern)make_code(); - result->Type = Extern_Linkage; - result->Name = get_cached_string( name ); - result->Body = body; - - return (CodeExtern)result; -} - -CodeFriend def_friend( Code declaration ) -{ - using namespace ECode; - - null_check( def_friend, declaration ); - - switch ( declaration->Type ) - { - case Class_Fwd : - case Function_Fwd : - case Operator_Fwd : - case Struct_Fwd : - case Class : - case Function : - case Operator : - case Struct : - break; - - default : - log_failure( "gen::def_friend: requires declartion to have class, function, operator, or struct - %s", declaration->debug_str() ); - return CodeInvalid; - } - - CodeFriend result = (CodeFriend)make_code(); - result->Type = Friend; - - result->Declaration = declaration; - - return result; -} - -CodeFn def_function( StrC name, CodeParam params, CodeType ret_type, Code body, CodeSpecifiers specifiers, CodeAttributes attributes, ModuleFlag mflags ) -{ - using namespace ECode; - - name_check( def_function, name ); - - if ( params && params->Type != Parameters ) - { - log_failure( "gen::def_function: params was not a `Parameters` type: %s", params.debug_str() ); - return CodeInvalid; - } - - if ( ret_type && ret_type->Type != Typename ) - { - log_failure( "gen::def_function: ret_type was not a Typename: %s", ret_type.debug_str() ); - return CodeInvalid; - } - - if ( specifiers && specifiers->Type != Specifiers ) - { - log_failure( "gen::def_function: specifiers was not a `Specifiers` type: %s", specifiers.debug_str() ); - return CodeInvalid; - } - - if ( attributes && attributes->Type != PlatformAttributes ) - { - log_failure( "gen::def_function: attributes was not a `PlatformAttributes` type: %s", attributes.debug_str() ); - return CodeInvalid; - } - - CodeFn result = (CodeFn)make_code(); - result->Name = get_cached_string( name ); - result->ModuleFlags = mflags; - - if ( body ) - { - switch ( body->Type ) - { - case Function_Body : - case Execution : - case Untyped : - break; - - default : - { - log_failure( "gen::def_function: body must be either of Function_Body, Execution, or Untyped type. %s", body->debug_str() ); - return CodeInvalid; - } - } - - result->Type = Function; - result->Body = body; - } - else - { - result->Type = Function_Fwd; - } - - if ( attributes ) - result->Attributes = attributes; - - if ( specifiers ) - result->Specs = specifiers; - - if ( ret_type ) - { - result->ReturnType = ret_type; - } - else - { - result->ReturnType = t_void; - } - - if ( params ) - result->Params = params; - - return result; -} - -CodeInclude def_include( StrC path, bool foreign ) -{ - if ( path.Len <= 0 || path.Ptr == nullptr ) - { - log_failure( "gen::def_include: Invalid path provided - %d" ); - return CodeInvalid; - } - - StrC content = foreign ? to_str( str_fmt_buf( "<%.*s>", path.Len, path.Ptr ) ) : to_str( str_fmt_buf( "\"%.*s\"", path.Len, path.Ptr ) ); - - Code result = make_code(); - result->Type = ECode::Preprocess_Include; - result->Name = get_cached_string( content ); - result->Content = result->Name; - - return (CodeInclude)result; -} - -CodeModule def_module( StrC name, ModuleFlag mflags ) -{ - name_check( def_module, name ); - - Code result = make_code(); - result->Type = ECode::Module; - result->Name = get_cached_string( name ); - result->Content = result->Name; - result->ModuleFlags = mflags; - - return (CodeModule)result; -} - -CodeNS def_namespace( StrC name, Code body, ModuleFlag mflags ) -{ - using namespace ECode; - - name_check( def_namespace, name ); - null_check( def_namespace, body ); - - if ( body->Type != Namespace_Body && body->Type != Untyped ) - { - log_failure( "gen::def_namespace: body is not of namespace or untyped type %s", body.debug_str() ); - return CodeInvalid; - } - - CodeNS result = (CodeNS)make_code(); - result->Type = Namespace; - result->Name = get_cached_string( name ); - result->ModuleFlags = mflags; - result->Body = body; - - return result; -} - -CodeOperator def_operator( - OperatorT op, - StrC nspace, - CodeParam params_code, - CodeType ret_type, - Code body, - CodeSpecifiers specifiers, - CodeAttributes attributes, - ModuleFlag mflags -) -{ - using namespace ECode; - - if ( attributes && attributes->Type != PlatformAttributes ) - { - log_failure( "gen::def_operator: PlatformAttributes was provided but its not of attributes type: %s", attributes.debug_str() ); - return CodeInvalid; - } - - if ( specifiers && specifiers->Type != Specifiers ) - { - log_failure( "gen::def_operator: Specifiers was provided but its not of specifiers type: %s", specifiers.debug_str() ); - return CodeInvalid; - } - - OpValidateResult check_result = operator__validate( op, params_code, ret_type, specifiers ); - - if ( check_result == OpValidateResult::Fail ) - { - return CodeInvalid; - } - - char const* name = nullptr; - - StrC op_str = to_str( op ); - if ( nspace.Len > 0 ) - name = str_fmt_buf( "%.*soperator %.*s", nspace.Len, nspace.Ptr, op_str.Len, op_str.Ptr ); - else - name = str_fmt_buf( "operator %.*s", op_str.Len, op_str.Ptr ); - CodeOperator result = (CodeOperator)make_code(); - result->Name = get_cached_string( { str_len( name ), name } ); - result->ModuleFlags = mflags; - result->Op = op; - - if ( body ) - { - switch ( body->Type ) - { - case Function_Body : - case Execution : - case Untyped : - break; - - default : - { - log_failure( "gen::def_operator: body must be either of Function_Body, Execution, or Untyped type. %s", body->debug_str() ); - return CodeInvalid; - } - } - - result->Type = check_result == OpValidateResult::Global ? Operator : Operator_Member; - - result->Body = body; - } - else - { - result->Type = check_result == OpValidateResult::Global ? Operator_Fwd : Operator_Member_Fwd; - } - - if ( attributes ) - result->Attributes = attributes; - - if ( specifiers ) - result->Specs = specifiers; - - result->ReturnType = ret_type; - - if ( params_code ) - result->Params = params_code; - - return result; -} - -CodeOpCast def_operator_cast( CodeType type, Code body, CodeSpecifiers const_spec ) -{ - using namespace ECode; - null_check( def_operator_cast, type ); - - if ( type->Type != Typename ) - { - log_failure( "gen::def_operator_cast: type is not a typename - %s", type.debug_str() ); - return CodeInvalid; - } - - CodeOpCast result = (CodeOpCast)make_code(); - - if ( body ) - { - result->Type = Operator_Cast; - - if ( body->Type != Function_Body && body->Type != Execution ) - { - log_failure( "gen::def_operator_cast: body is not of function body or execution type - %s", body.debug_str() ); - return CodeInvalid; - } - - result->Body = body; - } - else - { - result->Type = Operator_Cast_Fwd; - } - - if ( const_spec ) - { - result->Specs = const_spec; - } - - result->ValueType = type; - return result; -} - -CodeParam def_param( CodeType type, StrC name, Code value ) -{ - using namespace ECode; - - name_check( def_param, name ); - null_check( def_param, type ); - - if ( type->Type != Typename ) - { - log_failure( "gen::def_param: type is not a typename - %s", type.debug_str() ); - return CodeInvalid; - } - - if ( value && value->Type != Untyped ) - { - log_failure( "gen::def_param: value is not untyped - %s", value.debug_str() ); - return CodeInvalid; - } - - CodeParam result = (CodeParam)make_code(); - result->Type = Parameters; - result->Name = get_cached_string( name ); - - result->ValueType = type; - - if ( value ) - result->Value = value; - - result->NumEntries++; - - return result; -} - -CodePragma def_pragma( StrC directive ) -{ - using namespace ECode; - - if ( directive.Len <= 0 || directive.Ptr == nullptr ) - { - log_failure( "gen::def_comment: Invalid comment provided:" ); - return CodeInvalid; - } - - CodePragma result = (CodePragma)make_code(); - result->Type = Preprocess_Pragma; - result->Content = get_cached_string( directive ); - - return result; -} - -CodePreprocessCond def_preprocess_cond( EPreprocessCond type, StrC expr ) -{ - using namespace ECode; - - if ( expr.Len <= 0 || expr.Ptr == nullptr ) - { - log_failure( "gen::def_comment: Invalid comment provided:" ); - return CodeInvalid; - } - - CodePreprocessCond result = (CodePreprocessCond)make_code(); - result->Content = get_cached_string( expr ); - - switch ( type ) - { - case EPreprocessCond::If : - result->Type = Preprocess_If; - break; - case EPreprocessCond::IfDef : - result->Type = Preprocess_IfDef; - break; - case EPreprocessCond::IfNotDef : - result->Type = Preprocess_IfNotDef; - break; - case EPreprocessCond::ElIf : - result->Type = Preprocess_ElIf; - break; - } - - return result; -} - -CodeSpecifiers def_specifier( SpecifierT spec ) -{ - CodeSpecifiers result = (CodeSpecifiers)make_code(); - result->Type = ECode::Specifiers; - result.append( spec ); - - return result; -} - -CodeStruct def_struct( - StrC name, - Code body, - CodeType parent, - AccessSpec parent_access, - CodeAttributes attributes, - ModuleFlag mflags, - CodeType* interfaces, - s32 num_interfaces -) -{ - using namespace ECode; - - if ( attributes && attributes->Type != PlatformAttributes ) - { - log_failure( "gen::def_struct: attributes was not a `PlatformAttributes` type - %s", attributes.debug_str() ); - return CodeInvalid; - } - - if ( parent && parent->Type != Typename ) - { - log_failure( "gen::def_struct: parent was not a `Struct` type - %s", parent.debug_str() ); - return CodeInvalid; - } - - if ( body && body->Type != Struct_Body ) - { - log_failure( "gen::def_struct: body was not a Struct_Body type - %s", body.debug_str() ); - return CodeInvalid; - } - - CodeStruct result = (CodeStruct)make_code(); - result->ModuleFlags = mflags; - - if ( name ) - result->Name = get_cached_string( name ); - - if ( body ) - { - result->Type = Struct; - result->Body = body; - } - else - { - result->Type = Struct_Fwd; - } - - if ( attributes ) - result->Attributes = attributes; - - if ( parent ) - { - result->ParentAccess = parent_access; - result->ParentType = parent; - } - - if ( interfaces ) - { - for ( s32 idx = 0; idx < num_interfaces; idx++ ) - { - result.add_interface( interfaces[idx] ); - } - } - - return result; -} - -CodeTemplate def_template( CodeParam params, Code declaration, ModuleFlag mflags ) -{ - null_check( def_template, declaration ); - - if ( params && params->Type != ECode::Parameters ) - { - log_failure( "gen::def_template: params is not of parameters type - %s", params.debug_str() ); - return CodeInvalid; - } - - switch ( declaration->Type ) - { - case ECode::Class : - case ECode::Function : - case ECode::Struct : - case ECode::Variable : - case ECode::Using : - break; - - default : - log_failure( "gen::def_template: declaration is not of class, function, struct, variable, or using type - %s", declaration.debug_str() ); - } - - CodeTemplate result = (CodeTemplate)make_code(); - result->Type = ECode::Template; - result->ModuleFlags = mflags; - result->Params = params; - result->Declaration = declaration; - - return result; -} - -CodeType def_type( StrC name, Code arrayexpr, CodeSpecifiers specifiers, CodeAttributes attributes ) -{ - name_check( def_type, name ); - - if ( attributes && attributes->Type != ECode::PlatformAttributes ) - { - log_failure( "gen::def_type: attributes is not of attributes type - %s", attributes.debug_str() ); - return CodeInvalid; - } - - if ( specifiers && specifiers->Type != ECode::Specifiers ) - { - log_failure( "gen::def_type: specifiers is not of specifiers type - %s", specifiers.debug_str() ); - return CodeInvalid; - } - - if ( arrayexpr && arrayexpr->Type != ECode::Untyped ) - { - log_failure( "gen::def_type: arrayexpr is not of untyped type - %s", arrayexpr->debug_str() ); - return CodeInvalid; - } - - CodeType result = (CodeType)make_code(); - result->Name = get_cached_string( name ); - result->Type = ECode::Typename; - - if ( attributes ) - result->Attributes = attributes; - - if ( specifiers ) - result->Specs = specifiers; - - if ( arrayexpr ) - result->ArrExpr = arrayexpr; - - return result; -} - -CodeTypedef def_typedef( StrC name, Code type, CodeAttributes attributes, ModuleFlag mflags ) -{ - using namespace ECode; - - null_check( def_typedef, type ); - - switch ( type->Type ) - { - case Class : - case Class_Fwd : - case Enum : - case Enum_Fwd : - case Enum_Class : - case Enum_Class_Fwd : - case Function_Fwd : - case Struct : - case Struct_Fwd : - case Union : - case Typename : - break; - default : - log_failure( "gen::def_typedef: type was not a Class, Enum, Function Forward, Struct, Typename, or Union - %s", type.debug_str() ); - return CodeInvalid; - } - - if ( attributes && attributes->Type != ECode::PlatformAttributes ) - { - log_failure( "gen::def_typedef: attributes was not a PlatformAttributes - %s", attributes.debug_str() ); - return CodeInvalid; - } - - // Registering the type. - Code registered_type = def_type( name ); - - if ( ! registered_type ) - { - log_failure( "gen::def_typedef: failed to register type" ); - return CodeInvalid; - } - - CodeTypedef result = (CodeTypedef)make_code(); - result->Type = ECode::Typedef; - result->ModuleFlags = mflags; - - result->UnderlyingType = type; - - if ( name.Len <= 0 ) - { - if ( type->Type != Untyped ) - { - log_failure( "gen::def_typedef: name was empty and type was not untyped (indicating its a function typedef) - %s", type.debug_str() ); - return CodeInvalid; - } - - result->Name = get_cached_string( type->Name ); - result->IsFunction = true; - } - else - { - result->Name = get_cached_string( name ); - result->IsFunction = false; - } - - return result; -} - -CodeUnion def_union( StrC name, Code body, CodeAttributes attributes, ModuleFlag mflags ) -{ - null_check( def_union, body ); - - if ( body->Type != ECode::Union_Body ) - { - log_failure( "gen::def_union: body was not a Union_Body type - %s", body.debug_str() ); - return CodeInvalid; - } - - if ( attributes && attributes->Type != ECode::PlatformAttributes ) - { - log_failure( "gen::def_union: attributes was not a PlatformAttributes type - %s", attributes.debug_str() ); - return CodeInvalid; - } - - CodeUnion result = (CodeUnion)make_code(); - result->ModuleFlags = mflags; - result->Type = ECode::Union; - - if ( name.Ptr ) - result->Name = get_cached_string( name ); - - result->Body = body; - - if ( attributes ) - result->Attributes = attributes; - - return result; -} - -CodeUsing def_using( StrC name, CodeType type, CodeAttributes attributes, ModuleFlag mflags ) -{ - name_check( def_using, name ); - null_check( def_using, type ); - - Code register_type = def_type( name ); - - if ( ! register_type ) - { - log_failure( "gen::def_using: failed to register type" ); - return CodeInvalid; - } - - if ( attributes && attributes->Type != ECode::PlatformAttributes ) - { - log_failure( "gen::def_using: attributes was not a PlatformAttributes type - %s", attributes.debug_str() ); - return CodeInvalid; - } - - CodeUsing result = (CodeUsing)make_code(); - result->Name = get_cached_string( name ); - result->ModuleFlags = mflags; - result->Type = ECode::Using; - - result->UnderlyingType = type; - - if ( attributes ) - result->Attributes = attributes; - - return result; -} - -CodeUsing def_using_namespace( StrC name ) -{ - name_check( def_using_namespace, name ); - - Code result = make_code(); - result->Name = get_cached_string( name ); - result->Content = result->Name; - result->Type = ECode::Using_Namespace; - - return (CodeUsing)result; -} - -CodeVar def_variable( CodeType type, StrC name, Code value, CodeSpecifiers specifiers, CodeAttributes attributes, ModuleFlag mflags ) -{ - name_check( def_variable, name ); - null_check( def_variable, type ); - - if ( attributes && attributes->Type != ECode::PlatformAttributes ) - { - log_failure( "gen::def_variable: attributes was not a `PlatformAttributes` type - %s", attributes.debug_str() ); - return CodeInvalid; - } - - if ( specifiers && specifiers->Type != ECode::Specifiers ) - { - log_failure( "gen::def_variable: specifiers was not a `Specifiers` type - %s", specifiers.debug_str() ); - return CodeInvalid; - } - - if ( type->Type != ECode::Typename ) - { - log_failure( "gen::def_variable: type was not a Typename - %s", type.debug_str() ); - return CodeInvalid; - } - - if ( value && value->Type != ECode::Untyped ) - { - log_failure( "gen::def_variable: value was not a `Untyped` type - %s", value.debug_str() ); - return CodeInvalid; - } - - CodeVar result = (CodeVar)make_code(); - result->Name = get_cached_string( name ); - result->Type = ECode::Variable; - result->ModuleFlags = mflags; - - result->ValueType = type; - - if ( attributes ) - result->Attributes = attributes; - - if ( specifiers ) - result->Specs = specifiers; - - if ( value ) - result->Value = value; - - return result; -} - -#pragma region Helper Macros for def_**_body functions -#define def_body_start( Name_ ) \ - using namespace ECode; \ - \ - if ( num <= 0 ) \ - { \ - log_failure( "gen::" stringize( Name_ ) ": num cannot be zero or negative" ); \ - return CodeInvalid; \ - } - -#define def_body_code_array_start( Name_ ) \ - using namespace ECode; \ - \ - if ( num <= 0 ) \ - { \ - log_failure( "gen::" stringize( Name_ ) ": num cannot be zero or negative" ); \ - return CodeInvalid; \ - } \ - \ - if ( codes == nullptr ) \ - { \ - log_failure( "gen::" stringize( Name_ ) " : Provided a null array of codes" ); \ - return CodeInvalid; \ - } - -#pragma endregion Helper Macros for def_** _body functions - -CodeBody def_class_body( s32 num, ... ) -{ - def_body_start( def_class_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Class_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( - "gen::" - "def_class_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_CLASS_UNALLOWED_TYPES - log_failure( - "gen::" - "def_class_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return result; -} - -CodeBody def_class_body( s32 num, Code* codes ) -{ - def_body_code_array_start( def_class_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Function_Body; - - do - { - Code entry = *codes; - codes++; - - if ( ! entry ) - { - log_failure( - "gen::" - "def_class_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_CLASS_UNALLOWED_TYPES - log_failure( - "gen::" - "def_class_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - - return result; -} - -CodeBody def_enum_body( s32 num, ... ) -{ - def_body_start( def_enum_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Enum_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( "gen::def_enum_body: Provided a null entry" ); - return CodeInvalid; - } - - if ( entry->Type != Untyped && entry->Type != Comment ) - { - log_failure( "gen::def_enum_body: Entry type is not allowed - %s. Must be of untyped or comment type.", entry.debug_str() ); - return CodeInvalid; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return (CodeBody)result; -} - -CodeBody def_enum_body( s32 num, Code* codes ) -{ - def_body_code_array_start( def_enum_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Enum_Body; - - do - { - Code entry = *codes; - - if ( ! entry ) - { - log_failure( "gen::def_enum_body: Provided a null entry" ); - return CodeInvalid; - } - - if ( entry->Type != Untyped && entry->Type != Comment ) - { - log_failure( "gen::def_enum_body: Entry type is not allowed: %s", entry.debug_str() ); - return CodeInvalid; - } - - result.append( entry ); - } while ( codes++, num--, num > 0 ); - - return result; -} - -CodeBody def_export_body( s32 num, ... ) -{ - def_body_start( def_export_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Export_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( - "gen::" - "def_export_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_EXPORT_UNALLOWED_TYPES - log_failure( - "gen::" - "def_export_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return result; -} - -CodeBody def_export_body( s32 num, Code* codes ) -{ - def_body_code_array_start( def_export_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Export_Body; - - do - { - Code entry = *codes; - codes++; - - if ( ! entry ) - { - log_failure( - "gen::" - "def_export_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_EXPORT_UNALLOWED_TYPES - log_failure( - "gen::" - "def_export_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - - return result; -} - -CodeBody def_extern_link_body( s32 num, ... ) -{ - def_body_start( def_extern_linkage_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Extern_Linkage_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( - "gen::" - "def_extern_linkage_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES - log_failure( - "gen::" - "def_extern_linkage_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return result; -} - -CodeBody def_extern_link_body( s32 num, Code* codes ) -{ - def_body_code_array_start( def_extern_linkage_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Extern_Linkage_Body; - - do - { - Code entry = *codes; - codes++; - - if ( ! entry ) - { - log_failure( - "gen::" - "def_extern_linkage_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_EXTERN_LINKAGE_UNALLOWED_TYPES - log_failure( - "gen::" - "def_extern_linkage_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - - } while ( num--, num > 0 ); - - return result; -} - -CodeBody def_function_body( s32 num, ... ) -{ - def_body_start( def_function_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Function_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( "gen::" stringize( def_function_body ) ": Provided an null entry" ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - - GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES - log_failure( "gen::" stringize( def_function_body ) ": Entry type is not allowed: %s", entry.debug_str() ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return result; -} - -CodeBody def_function_body( s32 num, Code* codes ) -{ - def_body_code_array_start( def_function_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Function_Body; - - do - { - Code entry = *codes; - codes++; - - if ( ! entry ) - { - log_failure( - "gen::" - "def_function_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_FUNCTION_UNALLOWED_TYPES - log_failure( - "gen::" - "def_function_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - result.append( entry ); - } while ( num--, num > 0 ); - - return result; -} - -CodeBody def_global_body( s32 num, ... ) -{ - def_body_start( def_global_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Global_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( - "gen::" - "def_global_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - case Global_Body : - result.append( entry.cast() ); - continue; - - GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES - log_failure( - "gen::" - "def_global_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return ( *Code::Invalid.ast ); - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return result; -} - -CodeBody def_global_body( s32 num, Code* codes ) -{ - def_body_code_array_start( def_global_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Global_Body; - - do - { - Code entry = *codes; - codes++; - - if ( ! entry ) - { - log_failure( - "gen::" - "def_global_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - case Global_Body : - result.append( entry.cast() ); - continue; - - GEN_AST_BODY_GLOBAL_UNALLOWED_TYPES - log_failure( - "gen::" - "def_global_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - - return result; -} - -CodeBody def_namespace_body( s32 num, ... ) -{ - def_body_start( def_namespace_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Namespace_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( - "gen::" - "def_namespace_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES - log_failure( - "gen::" - "def_namespace_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return result; -} - -CodeBody def_namespace_body( s32 num, Code* codes ) -{ - def_body_code_array_start( def_namespace_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Global_Body; - - do - { - Code entry = *codes; - codes++; - - if ( ! entry ) - { - log_failure( - "gen::" - "def_namespace_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_NAMESPACE_UNALLOWED_TYPES - log_failure( - "gen::" - "def_namespace_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - - return result; -} - -CodeParam def_params( s32 num, ... ) -{ - def_body_start( def_params ); - - va_list va; - va_start( va, num ); - - Code_POD pod = va_arg( va, Code_POD ); - CodeParam param = pcast( CodeParam, pod ); - - null_check( def_params, param ); - - if ( param->Type != Parameters ) - { - log_failure( "gen::def_params: param %d is not a Parameters", num - num + 1 ); - return CodeInvalid; - } - - CodeParam result = (CodeParam)param.duplicate(); - - while ( --num ) - { - pod = va_arg( va, Code_POD ); - param = pcast( CodeParam, pod ); - - if ( param->Type != Parameters ) - { - log_failure( "gen::def_params: param %d is not a Parameters", num - num + 1 ); - return CodeInvalid; - } - - result.append( param ); - } - va_end( va ); - - return result; -} - -CodeParam def_params( s32 num, CodeParam* codes ) -{ - def_body_code_array_start( def_params ); - -#define check_current() \ - if ( current.ast == nullptr ) \ - { \ - log_failure( "gen::def_params: Provide a null code in codes array" ); \ - return CodeInvalid; \ - } \ - \ - if ( current->Type != Parameters ) \ - { \ - log_failure( "gen::def_params: Code in coes array is not of paramter type - %s", current.debug_str() ); \ - return CodeInvalid; \ - } - - CodeParam current = (CodeParam)codes->duplicate(); - check_current(); - - CodeParam result = (CodeParam)make_code(); - result->Name = current->Name; - result->Type = current->Type; - result->ValueType = current->ValueType; - - while ( codes++, current = *codes, num--, num > 0 ) - { - check_current(); - result.append( current ); - } -#undef check_current - - return result; -} - -CodeSpecifiers def_specifiers( s32 num, ... ) -{ - if ( num <= 0 ) - { - log_failure( "gen::def_specifiers: num cannot be zero or less" ); - return CodeInvalid; - } - - if ( num > AST::ArrSpecs_Cap ) - { - log_failure( "gen::def_specifiers: num of speciifers to define AST larger than AST specicifier capacity - %d", num ); - return CodeInvalid; - } - - CodeSpecifiers result = (CodeSpecifiers)make_code(); - result->Type = ECode::Specifiers; - - va_list va; - va_start( va, num ); - do - { - SpecifierT type = (SpecifierT)va_arg( va, int ); - - result.append( type ); - } while ( --num, num ); - va_end( va ); - - return result; -} - -CodeSpecifiers def_specifiers( s32 num, SpecifierT* specs ) -{ - if ( num <= 0 ) - { - log_failure( "gen::def_specifiers: num cannot be zero or less" ); - return CodeInvalid; - } - - if ( num > AST::ArrSpecs_Cap ) - { - log_failure( "gen::def_specifiers: num of speciifers to define AST larger than AST specicifier capacity - %d", num ); - return CodeInvalid; - } - - CodeSpecifiers result = (CodeSpecifiers)make_code(); - result->Type = ECode::Specifiers; - - s32 idx = 0; - do - { - result.append( specs[idx] ); - idx++; - } while ( --num, num ); - - return result; -} - -CodeBody def_struct_body( s32 num, ... ) -{ - def_body_start( def_struct_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Struct_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( - "gen::" - "def_struct_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_STRUCT_UNALLOWED_TYPES - log_failure( - "gen::" - "def_struct_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return result; -} - -CodeBody def_struct_body( s32 num, Code* codes ) -{ - def_body_code_array_start( def_struct_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Struct_Body; - - do - { - Code entry = *codes; - codes++; - - if ( ! entry ) - { - log_failure( - "gen::" - "def_struct_body" - ": Provided an null entry" - ); - return CodeInvalid; - } - - switch ( entry->Type ) - { - GEN_AST_BODY_STRUCT_UNALLOWED_TYPES - log_failure( - "gen::" - "def_struct_body" - ": Entry type is not allowed: %s", - entry.debug_str() - ); - return CodeInvalid; - - default : - break; - } - - result.append( entry ); - } while ( num--, num > 0 ); - - return result; -} - -CodeBody def_union_body( s32 num, ... ) -{ - def_body_start( def_union_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Union_Body; - - va_list va; - va_start( va, num ); - do - { - Code_POD pod = va_arg( va, Code_POD ); - Code entry = pcast( Code, pod ); - - if ( ! entry ) - { - log_failure( "gen::def_union_body: Provided a null entry" ); - return CodeInvalid; - } - - if ( entry->Type != Untyped && entry->Type != Comment ) - { - log_failure( "gen::def_union_body: Entry type is not allowed - %s. Must be of untyped or comment type.", entry.debug_str() ); - return CodeInvalid; - } - - result.append( entry ); - } while ( num--, num > 0 ); - va_end( va ); - - return result; -} - -CodeBody def_union_body( s32 num, CodeUnion* codes ) -{ - def_body_code_array_start( def_union_body ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Union_Body; - - do - { - Code entry = *codes; - - if ( ! entry ) - { - log_failure( "gen::def_union_body: Provided a null entry" ); - return CodeInvalid; - } - - if ( entry->Type != Untyped && entry->Type != Comment ) - { - log_failure( "gen::def_union_body: Entry type is not allowed: %s", entry.debug_str() ); - return CodeInvalid; - } - - result.append( entry ); - } while ( codes++, num--, num > 0 ); - - return (CodeBody)result; -} - -#undef name_check -#undef null_check -#undef null_or_invalid_check -#undef def_body_start -#undef def_body_code_array_start - -#pragma endregion Upfront - -#pragma region Parsing - -namespace parser -{ - namespace ETokType - { -#define GEN_DEFINE_ATTRIBUTE_TOKENS \ - Entry( Attribute_API_Export, "GEN_API_Export_Code" ) Entry( Attribute_API_Import, "GEN_API_Import_Code" ) - - enum Type : u32 - { - Invalid, - Access_Private, - Access_Protected, - Access_Public, - Access_MemberSymbol, - Access_StaticSymbol, - Ampersand, - Ampersand_DBL, - Assign_Classifer, - Attribute_Open, - Attribute_Close, - BraceCurly_Open, - BraceCurly_Close, - BraceSquare_Open, - BraceSquare_Close, - Capture_Start, - Capture_End, - Comment, - Comment_End, - Comment_Start, - Char, - Comma, - Decl_Class, - Decl_GNU_Attribute, - Decl_MSVC_Attribute, - Decl_Enum, - Decl_Extern_Linkage, - Decl_Friend, - Decl_Module, - Decl_Namespace, - Decl_Operator, - Decl_Struct, - Decl_Template, - Decl_Typedef, - Decl_Using, - Decl_Union, - Identifier, - Module_Import, - Module_Export, - NewLine, - Number, - Operator, - Preprocess_Hash, - Preprocess_Define, - Preprocess_If, - Preprocess_IfDef, - Preprocess_IfNotDef, - Preprocess_ElIf, - Preprocess_Else, - Preprocess_EndIf, - Preprocess_Include, - Preprocess_Pragma, - Preprocess_Content, - Preprocess_Macro, - Preprocess_Unsupported, - Spec_Alignas, - Spec_Const, - Spec_Consteval, - Spec_Constexpr, - Spec_Constinit, - Spec_Explicit, - Spec_Extern, - Spec_Final, - Spec_gb_inline, - Spec_gb_global, - Spec_Inline, - Spec_gb_internal, - Spec_LocalPersist, - Spec_Mutable, - Spec_NeverInline, - Spec_Override, - Spec_Static, - Spec_gb_thread_local, - Spec_Volatile, - Spec_Virtual, - Star, - Statement_End, - StaticAssert, - String, - Type_Typename, - Type_Unsigned, - Type_Signed, - Type_Short, - Type_Long, - Type_bool, - Type_char, - Type_int, - Type_double, - Type_MS_int8, - Type_MS_int16, - Type_MS_int32, - Type_MS_int64, - Type_MS_W64, - Varadic_Argument, - __Attributes_Start, - Attribute_API_Export, - Attribute_API_Import, - NumTokens - }; - - StrC to_str( Type type ) - { - local_persist StrC lookup[] { - { sizeof( "__invalid__" ), "__invalid__" }, - { sizeof( "private" ), "private" }, - { sizeof( "protected" ), "protected" }, - { sizeof( "public" ), "public" }, - { sizeof( "." ), "." }, - { sizeof( "::" ), "::" }, - { sizeof( "&" ), "&" }, - { sizeof( "&&" ), "&&" }, - { sizeof( ":" ), ":" }, - { sizeof( "[[" ), "[[" }, - { sizeof( "]]" ), "]]" }, - { sizeof( "{" ), "{" }, - { sizeof( "}" ), "}" }, - { sizeof( "[" ), "[" }, - { sizeof( "]" ), "]" }, - { sizeof( "(" ), "(" }, - { sizeof( ")" ), ")" }, - { sizeof( "__comment__" ), "__comment__" }, - { sizeof( "__comment_end__" ), "__comment_end__" }, - { sizeof( "__comment_start__" ), "__comment_start__" }, - { sizeof( "__character__" ), "__character__" }, - { sizeof( "," ), "," }, - { sizeof( "class" ), "class" }, - { sizeof( "__attribute__" ), "__attribute__" }, - { sizeof( "__declspec" ), "__declspec" }, - { sizeof( "enum" ), "enum" }, - { sizeof( "extern" ), "extern" }, - { sizeof( "friend" ), "friend" }, - { sizeof( "module" ), "module" }, - { sizeof( "namespace" ), "namespace" }, - { sizeof( "operator" ), "operator" }, - { sizeof( "struct" ), "struct" }, - { sizeof( "template" ), "template" }, - { sizeof( "typedef" ), "typedef" }, - { sizeof( "using" ), "using" }, - { sizeof( "union" ), "union" }, - { sizeof( "__identifier__" ), "__identifier__" }, - { sizeof( "import" ), "import" }, - { sizeof( "export" ), "export" }, - { sizeof( "__new_line__" ), "__new_line__" }, - { sizeof( "__number__" ), "__number__" }, - { sizeof( "__operator__" ), "__operator__" }, - { sizeof( "#" ), "#" }, - { sizeof( "define" ), "define" }, - { sizeof( "if" ), "if" }, - { sizeof( "ifdef" ), "ifdef" }, - { sizeof( "ifndef" ), "ifndef" }, - { sizeof( "elif" ), "elif" }, - { sizeof( "else" ), "else" }, - { sizeof( "endif" ), "endif" }, - { sizeof( "include" ), "include" }, - { sizeof( "pragma" ), "pragma" }, - { sizeof( "__macro_content__" ), "__macro_content__" }, - { sizeof( "__macro__" ), "__macro__" }, - { sizeof( "__unsupported__" ), "__unsupported__" }, - { sizeof( "alignas" ), "alignas" }, - { sizeof( "const" ), "const" }, - { sizeof( "consteval" ), "consteval" }, - { sizeof( "constexpr" ), "constexpr" }, - { sizeof( "constinit" ), "constinit" }, - { sizeof( "explicit" ), "explicit" }, - { sizeof( "extern" ), "extern" }, - { sizeof( "final" ), "final" }, - { sizeof( "gb_inline" ), "gb_inline" }, - { sizeof( "gb_global" ), "gb_global" }, - { sizeof( "inline" ), "inline" }, - { sizeof( "gb_internal" ), "gb_internal" }, - { sizeof( "local_persist" ), "local_persist" }, - { sizeof( "mutable" ), "mutable" }, - { sizeof( "neverinline" ), "neverinline" }, - { sizeof( "override" ), "override" }, - { sizeof( "static" ), "static" }, - { sizeof( "gb_thread_local" ), "gb_thread_local" }, - { sizeof( "volatile" ), "volatile" }, - { sizeof( "virtual" ), "virtual" }, - { sizeof( "*" ), "*" }, - { sizeof( ";" ), ";" }, - { sizeof( "static_assert" ), "static_assert" }, - { sizeof( "__string__" ), "__string__" }, - { sizeof( "typename" ), "typename" }, - { sizeof( "unsigned" ), "unsigned" }, - { sizeof( "signed" ), "signed" }, - { sizeof( "short" ), "short" }, - { sizeof( "long" ), "long" }, - { sizeof( "bool" ), "bool" }, - { sizeof( "char" ), "char" }, - { sizeof( "int" ), "int" }, - { sizeof( "double" ), "double" }, - { sizeof( "__int8" ), "__int8" }, - { sizeof( "__int16" ), "__int16" }, - { sizeof( "__int32" ), "__int32" }, - { sizeof( "__int64" ), "__int64" }, - { sizeof( "_W64" ), "_W64" }, - { sizeof( "..." ), "..." }, - { sizeof( "__attrib_start__" ), "__attrib_start__" }, - { sizeof( "GEN_API_Export_Code" ), "GEN_API_Export_Code" }, - { sizeof( "GEN_API_Import_Code" ), "GEN_API_Import_Code" }, - }; - return lookup[type]; - } - - Type to_type( StrC str ) - { - local_persist u32 keymap[NumTokens]; - do_once_start for ( u32 index = 0; index < NumTokens; index++ ) - { - StrC enum_str = to_str( (Type)index ); - keymap[index] = crc32( enum_str.Ptr, enum_str.Len - 1 ); - } - do_once_end u32 hash = crc32( str.Ptr, str.Len ); - for ( u32 index = 0; index < NumTokens; index++ ) - { - if ( keymap[index] == hash ) - return (Type)index; - } - return Invalid; - } - - } // namespace ETokType - - using TokType = ETokType::Type; - -} // namespace parser - -namespace parser -{ - - enum TokFlags : u32 - { - TF_Operator = bit( 0 ), - TF_Assign = bit( 1 ), - TF_Preprocess = bit( 2 ), - TF_Preprocess_Cond = bit( 3 ), - TF_Attribute = bit( 6 ), - TF_AccessOperator = bit( 7 ), - TF_AccessSpecifier = bit( 8 ), - TF_Specifier = bit( 9 ), - TF_EndDefinition = bit( 10 ), // Either ; or } - TF_Formatting = bit( 11 ), - TF_Literal = bit( 12 ), - - TF_Null = 0, - }; - - struct Token - { - char const* Text; - sptr Length; - TokType Type; - s32 Line; - s32 Column; - u32 Flags; - - operator bool() - { - return Text && Length && Type != TokType::Invalid; - } - - operator StrC() - { - return { Length, Text }; - } - - bool is_access_operator() - { - return bitfield_is_equal( u32, Flags, TF_AccessOperator ); - } - - bool is_access_specifier() - { - return bitfield_is_equal( u32, Flags, TF_AccessSpecifier ); - } - - bool is_attribute() - { - return bitfield_is_equal( u32, Flags, TF_Attribute ); - } - - bool is_operator() - { - return bitfield_is_equal( u32, Flags, TF_Operator ); - } - - bool is_preprocessor() - { - return bitfield_is_equal( u32, Flags, TF_Preprocess ); - } - - bool is_preprocess_cond() - { - return bitfield_is_equal( u32, Flags, TF_Preprocess_Cond ); - } - - bool is_specifier() - { - return bitfield_is_equal( u32, Flags, TF_Specifier ); - } - - bool is_end_definition() - { - return bitfield_is_equal( u32, Flags, TF_EndDefinition ); - } - - AccessSpec to_access_specifier() - { - return scast( AccessSpec, Type ); - } - - String to_string() - { - String result = String::make_reserve( GlobalAllocator, kilobytes( 4 ) ); - - StrC type_str = ETokType::to_str( Type ); - - result.append_fmt( "Line: %d Column: %d, Type: %.*s Content: %.*s", Line, Column, type_str.Len, type_str.Ptr, Length, Text ); - - return result; - } - }; - - constexpr Token NullToken { nullptr, 0, TokType::Invalid, false, 0, TF_Null }; - - struct TokArray - { - Array Arr; - s32 Idx; - - bool __eat( TokType type ); - - Token& current( bool skip_formatting = true ) - { - if ( skip_formatting ) - { - while ( Arr[Idx].Type == TokType::NewLine || Arr[Idx].Type == TokType::Comment ) - Idx++; - } - - return Arr[Idx]; - } - - Token& previous( bool skip_formatting = false ) - { - s32 idx = this->Idx; - - if ( skip_formatting ) - { - while ( Arr[idx].Type == TokType::NewLine ) - idx--; - - return Arr[idx]; - } - - return Arr[idx - 1]; - } - - Token& next( bool skip_formatting = false ) - { - s32 idx = this->Idx; - - if ( skip_formatting ) - { - while ( Arr[idx].Type == TokType::NewLine ) - idx++; - - return Arr[idx + 1]; - } - - return Arr[idx + 1]; - } - - Token& operator[]( s32 idx ) - { - return Arr[idx]; - } - }; - - global Arena_256KB defines_map_arena; - global HashTable defines; - global Array Tokens; - -#define current ( *scanner ) - -#define move_forward() \ - { \ - if ( current == '\n' ) \ - { \ - line++; \ - column = 1; \ - } \ - else \ - { \ - column++; \ - } \ - left--; \ - scanner++; \ - } - -#define SkipWhitespace() \ - while ( left && char_is_space( current ) ) \ - { \ - move_forward(); \ - } - -#define end_line() \ - do \ - { \ - while ( left && current == ' ' ) \ - { \ - move_forward(); \ - } \ - if ( left && current == '\r' ) \ - { \ - move_forward(); \ - move_forward(); \ - } \ - else if ( left && current == '\n' ) \ - { \ - move_forward(); \ - } \ - } while ( 0 ) - - enum - { - Lex_Continue, - Lex_ReturnNull, - }; - - FORCEINLINE s32 lex_preprocessor_directive( StrC& content, s32& left, char const*& scanner, s32& line, s32& column, HashTable& defines, Token& token ) - { - char const* hash = scanner; - Tokens.append( { hash, 1, TokType::Preprocess_Hash, line, column, TF_Preprocess } ); - - move_forward(); - SkipWhitespace(); - - token.Text = scanner; - while ( left && ! char_is_space( current ) ) - { - move_forward(); - token.Length++; - } - - token.Type = ETokType::to_type( token ); - - bool is_preprocessor = token.Type >= TokType::Preprocess_Define && token.Type <= TokType::Preprocess_Pragma; - if ( ! is_preprocessor ) - { - token.Type = TokType::Preprocess_Unsupported; - - // Its an unsupported directive, skip it - s32 within_string = false; - s32 within_char = false; - while ( left ) - { - if ( current == '"' && ! within_char ) - within_string ^= true; - - if ( current == '\'' && ! within_string ) - within_char ^= true; - - if ( current == '\\' && ! within_string && ! within_char ) - { - move_forward(); - token.Length++; - - if ( current == '\r' ) - { - move_forward(); - token.Length++; - } - - if ( current == '\n' ) - { - move_forward(); - token.Length++; - continue; - } - else - { - log_failure( - "gen::Parser::lex: Invalid escape sequence '\\%c' (%d, %d)" - " in preprocessor directive (%d, %d)\n%.100s", - current, - line, - column, - token.Line, - token.Column, - token.Text - ); - break; - } - } - - if ( current == '\r' ) - { - move_forward(); - token.Length++; - } - - if ( current == '\n' ) - { - move_forward(); - token.Length++; - break; - } - - move_forward(); - token.Length++; - } - - token.Length = token.Length + token.Text - hash; - token.Text = hash; - Tokens.append( token ); - return Lex_Continue; // Skip found token, its all handled here. - } - - if ( token.Type == TokType::Preprocess_Else || token.Type == TokType::Preprocess_EndIf ) - { - token.Flags |= TF_Preprocess_Cond; - Tokens.append( token ); - end_line(); - return Lex_Continue; - } - else if ( token.Type >= TokType::Preprocess_If && token.Type <= TokType::Preprocess_ElIf ) - { - token.Flags |= TF_Preprocess_Cond; - } - - Tokens.append( token ); - - SkipWhitespace(); - - if ( token.Type == TokType::Preprocess_Define ) - { - Token name = { scanner, 0, TokType::Identifier, line, column, TF_Preprocess }; - - name.Text = scanner; - name.Length = 1; - move_forward(); - - while ( left && ( char_is_alphanumeric( current ) || current == '_' ) ) - { - move_forward(); - name.Length++; - } - - if ( left && current == '(' ) - { - move_forward(); - name.Length++; - } - - Tokens.append( name ); - - u64 key = crc32( name.Text, name.Length ); - defines.set( key, name ); - } - - Token preprocess_content = { scanner, 0, TokType::Preprocess_Content, line, column, TF_Preprocess }; - - if ( token.Type == TokType::Preprocess_Include ) - { - preprocess_content.Type = TokType::String; - - if ( current != '"' && current != '<' ) - { - String directive_str = String::fmt_buf( GlobalAllocator, "%.*s", min( 80, left + preprocess_content.Length ), token.Text ); - - log_failure( - "gen::Parser::lex: Expected '\"' or '<' after #include, not '%c' (%d, %d)\n%s", - current, - preprocess_content.Line, - preprocess_content.Column, - directive_str.Data - ); - return Lex_ReturnNull; - } - move_forward(); - preprocess_content.Length++; - - while ( left && current != '"' && current != '>' ) - { - move_forward(); - preprocess_content.Length++; - } - - move_forward(); - preprocess_content.Length++; - - if ( current == '\r' && scanner[1] == '\n' ) - { - move_forward(); - move_forward(); - } - else if ( current == '\n' ) - { - move_forward(); - } - - Tokens.append( preprocess_content ); - return Lex_Continue; // Skip found token, its all handled here. - } - - s32 within_string = false; - s32 within_char = false; - - // SkipWhitespace(); - while ( left ) - { - if ( current == '"' && ! within_char ) - within_string ^= true; - - if ( current == '\'' && ! within_string ) - within_char ^= true; - - if ( current == '\\' && ! within_string && ! within_char ) - { - move_forward(); - preprocess_content.Length++; - - if ( current == '\r' ) - { - move_forward(); - preprocess_content.Length++; - } - - if ( current == '\n' ) - { - move_forward(); - preprocess_content.Length++; - continue; - } - else - { - String directive_str = String::make_length( GlobalAllocator, token.Text, token.Length ); - String content_str = String::fmt_buf( GlobalAllocator, "%.*s", min( 400, left + preprocess_content.Length ), preprocess_content.Text ); - - log_failure( - "gen::Parser::lex: Invalid escape sequence '\\%c' (%d, %d)" - " in preprocessor directive '%s' (%d, %d)\n%s", - current, - line, - column, - directive_str, - preprocess_content.Line, - preprocess_content.Column, - content_str - ); - break; - } - } - - if ( current == '\r' ) - { - move_forward(); - } - - if ( current == '\n' ) - { - move_forward(); - break; - } - - move_forward(); - preprocess_content.Length++; - } - - Tokens.append( preprocess_content ); - return Lex_Continue; // Skip found token, its all handled here. - } - - FORCEINLINE void lex_found_token( StrC& content, s32& left, char const*& scanner, s32& line, s32& column, HashTable& defines, Token& token ) - { - if ( token.Type != TokType::Invalid ) - { - Tokens.append( token ); - return; - } - - TokType type = ETokType::to_type( token ); - - if ( type <= TokType::Access_Public && type >= TokType::Access_Private ) - { - token.Flags |= TF_AccessSpecifier; - } - - if ( type > TokType::__Attributes_Start ) - { - token.Flags |= TF_Attribute; - } - - if ( type == ETokType::Decl_Extern_Linkage ) - { - SkipWhitespace(); - - if ( current != '"' ) - { - type = ETokType::Spec_Extern; - token.Flags |= TF_Specifier; - } - - token.Type = type; - Tokens.append( token ); - return; - } - - if ( ( type <= TokType::Star && type >= TokType::Spec_Alignas ) || type == TokType::Ampersand || type == TokType::Ampersand_DBL ) - { - token.Type = type; - token.Flags |= TF_Specifier; - Tokens.append( token ); - return; - } - - - if ( type != TokType::Invalid ) - { - token.Type = type; - Tokens.append( token ); - return; - } - - u64 key = 0; - if ( current == '(' ) - key = crc32( token.Text, token.Length + 1 ); - else - key = crc32( token.Text, token.Length ); - - StrC* define = defines.get( key ); - if ( define ) - { - token.Type = TokType::Preprocess_Macro; - - // Want to ignore any arguments the define may have as they can be execution expressions. - if ( left && current == '(' ) - { - move_forward(); - token.Length++; - - s32 level = 0; - while ( left && ( current != ')' || level > 0 ) ) - { - if ( current == '(' ) - level++; - - else if ( current == ')' && level > 0 ) - level--; - - move_forward(); - token.Length++; - } - - move_forward(); - token.Length++; - } - - if ( current == '\r' && scanner[1] == '\n' ) - { - move_forward(); - } - else if ( current == '\n' ) - { - move_forward(); - } - } - else - { - token.Type = TokType::Identifier; - } - - Tokens.append( token ); - } - - neverinline - // TokArray lex( Array tokens, StrC content ) - TokArray - lex( StrC content ) - { - s32 left = content.Len; - char const* scanner = content.Ptr; - - char const* word = scanner; - s32 word_length = 0; - - s32 line = 1; - s32 column = 1; - - SkipWhitespace(); - if ( left <= 0 ) - { - log_failure( "gen::lex: no tokens found (only whitespace provided)" ); - return { { nullptr }, 0 }; - } - - for ( StringCached entry : PreprocessorDefines ) - { - s32 length = 0; - char const* scanner = entry.Data; - while ( entry.length() > length && (char_is_alphanumeric( *scanner ) || *scanner == '_') ) - { - scanner++; - length++; - } - if ( scanner[0] == '(' ) - { - length++; - } - - u64 key = crc32( entry.Data, length ); - defines.set( key, entry ); - } - - Tokens.clear(); - - while ( left ) - { -#if 0 - if (Tokens.num()) - { - log_fmt("\nLastTok: %S", Tokens.back().to_string()); - } -#endif - - Token token = { scanner, 0, TokType::Invalid, line, column, TF_Null }; - - bool is_define = false; - - if ( column == 1 ) - { - if ( current == '\r' ) - { - move_forward(); - token.Length = 1; - } - - if ( current == '\n' ) - { - move_forward(); - - token.Type = TokType::NewLine; - token.Length++; - - Tokens.append( token ); - continue; - } - } - - token.Length = 0; - - SkipWhitespace(); - if ( left <= 0 ) - break; - - switch ( current ) - { - case '#' : - { - s32 result = lex_preprocessor_directive( content, left, scanner, line, column, defines, token ); - switch ( result ) - { - case Lex_Continue : - continue; - - case Lex_ReturnNull : - return { { nullptr }, 0 }; - } - } - case '.' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Access_MemberSymbol; - token.Flags = TF_AccessOperator; - - if ( left ) - { - move_forward(); - } - - if ( current == '.' ) - { - move_forward(); - if ( current == '.' ) - { - token.Length = 3; - token.Type = TokType::Varadic_Argument; - token.Flags = TF_Null; - move_forward(); - } - else - { - String context_str = String::fmt_buf( GlobalAllocator, "%s", scanner, min( 100, left ) ); - - log_failure( "gen::lex: invalid varadic argument, expected '...' got '..%c' (%d, %d)\n%s", current, line, column, context_str ); - } - } - - goto FoundToken; - } - case '&' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Ampersand; - token.Flags |= TF_Operator; - token.Flags |= TF_Specifier; - - if ( left ) - move_forward(); - - if ( current == '&' ) // && - { - token.Length = 2; - token.Type = TokType::Ampersand_DBL; - - if ( left ) - move_forward(); - } - - goto FoundToken; - } - case ':' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Assign_Classifer; - // Can be either a classifier (ParentType, Bitfield width), or ternary else - // token.Type = TokType::Colon; - - if ( left ) - move_forward(); - - if ( current == ':' ) - { - move_forward(); - token.Type = TokType::Access_StaticSymbol; - token.Length++; - } - goto FoundToken; - } - case '{' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::BraceCurly_Open; - - if ( left ) - move_forward(); - goto FoundToken; - } - case '}' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::BraceCurly_Close; - token.Flags = TF_EndDefinition; - - if ( left ) - move_forward(); - - end_line(); - goto FoundToken; - } - case '[' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::BraceSquare_Open; - if ( left ) - { - move_forward(); - - if ( current == ']' ) - { - token.Length = 2; - token.Type = TokType::Operator; - move_forward(); - } - } - goto FoundToken; - } - case ']' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::BraceSquare_Close; - - if ( left ) - move_forward(); - goto FoundToken; - } - case '(' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Capture_Start; - - if ( left ) - move_forward(); - goto FoundToken; - } - case ')' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Capture_End; - - if ( left ) - move_forward(); - goto FoundToken; - } - case '\'' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Char; - token.Flags = TF_Literal; - - move_forward(); - - if ( left && current == '\\' ) - { - move_forward(); - token.Length++; - - if ( current == '\'' ) - { - move_forward(); - token.Length++; - } - } - - while ( left && current != '\'' ) - { - move_forward(); - token.Length++; - } - - if ( left ) - { - move_forward(); - token.Length++; - } - goto FoundToken; - } - case ',' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Comma; - token.Flags = TF_Operator; - - if ( left ) - move_forward(); - goto FoundToken; - } - case '*' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Star; - token.Flags |= TF_Specifier; - token.Flags |= TF_Operator; - - if ( left ) - move_forward(); - - if ( current == '=' ) - { - token.Length++; - token.Flags |= TF_Assign; - // token.Type = TokType::Assign_Multiply; - - if ( left ) - move_forward(); - } - - goto FoundToken; - } - case ';' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Statement_End; - token.Flags = TF_EndDefinition; - - if ( left ) - move_forward(); - - end_line(); - goto FoundToken; - } - case '"' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::String; - token.Flags |= TF_Literal; - - move_forward(); - while ( left ) - { - if ( current == '"' ) - { - move_forward(); - break; - } - - if ( current == '\\' ) - { - move_forward(); - token.Length++; - - if ( left ) - { - move_forward(); - token.Length++; - } - continue; - } - - move_forward(); - token.Length++; - } - goto FoundToken; - } - case '?' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Operator; - // token.Type = TokType::Ternary; - token.Flags = TF_Operator; - - if ( left ) - move_forward(); - - goto FoundToken; - } - case '=' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Operator; - // token.Type = TokType::Assign; - token.Flags = TF_Operator; - token.Flags |= TF_Assign; - - if ( left ) - move_forward(); - - if ( current == '=' ) - { - token.Length++; - token.Flags = TF_Operator; - - if ( left ) - move_forward(); - } - - goto FoundToken; - } - case '+' : - { - // token.Type = TokType::Add - } - case '%' : - { - // token.Type = TokType::Modulo; - } - case '^' : - { - // token.Type = TokType::B_XOr; - } - case '~' : - { - // token.Type = TokType::Unary_Not; - } - case '!' : - { - // token.Type = TokType::L_Not; - } - case '<' : - { - // token.Type = TokType::Lesser; - } - case '>' : - { - // token.Type = TokType::Greater; - } - case '|' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Operator; - token.Flags = TF_Operator; - // token.Type = TokType::L_Or; - - if ( left ) - move_forward(); - - if ( current == '=' ) - { - token.Length++; - token.Flags |= TF_Assign; - // token.Flags |= TokFlags::Assignment; - // token.Type = TokType::Assign_L_Or; - - if ( left ) - move_forward(); - } - else - while ( left && current == *( scanner - 1 ) && token.Length < 3 ) - { - token.Length++; - - if ( left ) - move_forward(); - } - goto FoundToken; - } - - // Dash is unfortunatlly a bit more complicated... - case '-' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Operator; - // token.Type = TokType::Subtract; - token.Flags = TF_Operator; - if ( left ) - { - move_forward(); - - if ( current == '>' ) - { - token.Length++; - // token.Type = TokType::Access_PointerToMemberSymbol; - token.Flags |= TF_AccessOperator; - move_forward(); - - if ( current == '*' ) - { - // token.Type = TokType::Access_PointerToMemberOfPointerSymbol; - token.Length++; - move_forward(); - } - } - else if ( current == '=' ) - { - token.Length++; - // token.Type = TokType::Assign_Subtract; - token.Flags |= TF_Assign; - - if ( left ) - move_forward(); - } - else - while ( left && current == *( scanner - 1 ) && token.Length < 3 ) - { - token.Length++; - - if ( left ) - move_forward(); - } - } - goto FoundToken; - } - case '/' : - { - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Operator; - // token.Type = TokType::Divide; - token.Flags = TF_Operator; - move_forward(); - - if ( left ) - { - if ( current == '=' ) - { - // token.Type = TokeType::Assign_Divide; - move_forward(); - token.Length++; - token.Flags = TF_Assign; - } - else if ( current == '/' ) - { - token.Type = TokType::Comment; - token.Length = 2; - token.Flags = TF_Null; - move_forward(); - - while ( left && current != '\n' && current != '\r' ) - { - move_forward(); - token.Length++; - } - - if ( current == '\r' ) - { - move_forward(); - token.Length++; - } - if ( current == '\n' ) - { - move_forward(); - token.Length++; - } - Tokens.append( token ); - continue; - } - else if ( current == '*' ) - { - token.Type = TokType::Comment; - token.Length = 2; - token.Flags = TF_Null; - move_forward(); - - bool star = current == '*'; - bool slash = scanner[1] == '/'; - bool at_end = star && slash; - while ( left && ! at_end ) - { - move_forward(); - token.Length++; - - star = current == '*'; - slash = scanner[1] == '/'; - at_end = star && slash; - } - token.Length += 2; - move_forward(); - move_forward(); - - if ( current == '\r' ) - { - move_forward(); - token.Length++; - } - if ( current == '\n' ) - { - move_forward(); - token.Length++; - } - Tokens.append( token ); - // end_line(); - continue; - } - } - goto FoundToken; - } - } - - if ( char_is_alpha( current ) || current == '_' ) - { - token.Text = scanner; - token.Length = 1; - move_forward(); - - while ( left && ( char_is_alphanumeric( current ) || current == '_' ) ) - { - move_forward(); - token.Length++; - } - - goto FoundToken; - } - else if ( char_is_digit( current ) ) - { - // This is a very brute force lex, no checks are done for validity of literal. - - token.Text = scanner; - token.Length = 1; - token.Type = TokType::Number; - token.Flags = TF_Literal; - move_forward(); - - if ( left && ( current == 'x' || current == 'X' || current == 'b' || current == 'B' || current == 'o' || current == 'O' ) ) - { - move_forward(); - token.Length++; - - while ( left && char_is_hex_digit( current ) ) - { - move_forward(); - token.Length++; - } - - goto FoundToken; - } - - while ( left && char_is_digit( current ) ) - { - move_forward(); - token.Length++; - } - - if ( left && current == '.' ) - { - move_forward(); - token.Length++; - - while ( left && char_is_digit( current ) ) - { - move_forward(); - token.Length++; - } - } - - goto FoundToken; - } - else - { - s32 start = max( 0, Tokens.num() - 100 ); - log_fmt( "\n%d\n", start ); - for ( s32 idx = start; idx < Tokens.num(); idx++ ) - { - log_fmt( "Token %d Type: %s : %.*s\n", idx, ETokType::to_str( Tokens[idx].Type ).Ptr, Tokens[idx].Length, Tokens[idx].Text ); - } - - String context_str = String::fmt_buf( GlobalAllocator, "%.*s", min( 100, left ), scanner ); - log_failure( "Failed to lex token '%c' (%d, %d)\n%s", current, line, column, context_str ); - - // Skip to next whitespace since we can't know if anything else is valid until then. - while ( left && ! char_is_space( current ) ) - { - move_forward(); - } - } - - FoundToken: - lex_found_token( content, left, scanner, line, column, defines, token ); - } - - if ( Tokens.num() == 0 ) - { - log_failure( "Failed to lex any tokens" ); - return { { nullptr }, 0 }; - } - - defines.clear(); - // defines_map_arena.free(); - return { Tokens, 0 }; - } - -#undef current -#undef move_forward -#undef SkipWhitespace - - // namespace parser -} // namespace parser - -namespace parser -{ - - // TODO(Ed) : Rename ETokType::Capture_Start, ETokType::Capture_End to Open_Parenthesis adn Close_Parenthesis - - constexpr bool dont_skip_formatting = false; - - struct StackNode - { - StackNode* Prev; - - Token Start; - Token Name; // The name of the AST node (if parsed) - StrC ProcName; // The name of the procedure - }; - - struct ParseContext - { - TokArray Tokens; - StackNode* Scope; - - void push( StackNode* node ) - { - node->Prev = Scope; - Scope = node; - -#if 0 && Build_Debug - log_fmt("\tEntering Context: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); -#endif - } - - void pop() - { -#if 0 && Build_Debug - log_fmt("\tPopping Context: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); -#endif - Scope = Scope->Prev; - } - - String to_string() - { - String result = String::make_reserve( GlobalAllocator, kilobytes( 4 ) ); - - Token scope_start = Scope->Start; - Token last_valid = Tokens.Idx >= Tokens.Arr.num() ? Tokens.Arr[Tokens.Arr.num() - 1] : Tokens.current(); - - sptr length = scope_start.Length; - char const* current = scope_start.Text + length; - while ( current <= Tokens.Arr.back().Text && *current != '\n' && length < 74 ) - { - current++; - length++; - } - - String line = String::make( GlobalAllocator, { length, scope_start.Text } ); - result.append_fmt( "\tScope : %s\n", line ); - line.free(); - - sptr dist = (sptr)last_valid.Text - (sptr)scope_start.Text + 2; - sptr length_from_err = dist; - String line_from_err = String::make( GlobalAllocator, { length_from_err, last_valid.Text } ); - - if ( length_from_err < 100 ) - result.append_fmt( "\t(%d, %d):%*c\n", last_valid.Line, last_valid.Column, length_from_err, '^' ); - else - result.append_fmt( "\t(%d, %d)\n", last_valid.Line, last_valid.Column ); - - StackNode* curr_scope = Scope; - s32 level = 0; - do - { - if ( curr_scope->Name ) - { - result.append_fmt( "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Length, curr_scope->Name.Text ); - } - else - { - result.append_fmt( "\t%d: %s\n", level, curr_scope->ProcName.Ptr ); - } - - curr_scope = curr_scope->Prev; - level++; - } while ( curr_scope ); - return result; - } - }; - - global ParseContext Context; - - bool TokArray::__eat( TokType type ) - { - if ( Arr.num() - Idx <= 0 ) - { - log_failure( "No tokens left.\n%s", Context.to_string() ); - return false; - } - - if ( ( Arr[Idx].Type == TokType::NewLine && type != TokType::NewLine ) || ( Arr[Idx].Type == TokType::Comment && type != TokType::Comment ) ) - { - Idx++; - } - - if ( Arr[Idx].Type != type ) - { - log_failure( - "Parse Error, TokArray::eat, Expected: ' %s ' not ' %.*s ' (%d, %d)`\n%s", - ETokType::to_str( type ).Ptr, - Arr[Idx].Length, - Arr[Idx].Text, - current().Line, - current().Column, - Context.to_string() - ); - - return false; - } - -#if 0 && Build_Debug - log_fmt("Ate: %S\n", Arr[Idx].to_string() ); -#endif - - Idx++; - return true; - } - - internal void init() - { - Tokens = Array::init_reserve( LexArena, ( LexAllocator_Size - sizeof( Array::Header ) ) / sizeof( Token ) ); - - defines_map_arena = Arena_256KB::init(); - defines = HashTable::init_reserve( defines_map_arena, 256 ); - } - - internal void deinit() - { - parser::Tokens = { nullptr }; - } - -#pragma region Helper Macros - -#define check_parse_args( def ) \ - if ( def.Len <= 0 ) \ - { \ - log_failure( "gen::" stringize( __func__ ) ": length must greater than 0" ); \ - parser::Context.pop(); \ - return CodeInvalid; \ - } \ - if ( def.Ptr == nullptr ) \ - { \ - log_failure( "gen::" stringize( __func__ ) ": def was null" ); \ - parser::Context.pop(); \ - return CodeInvalid; \ - } - -#define currtok_noskip Context.Tokens.current( dont_skip_formatting ) -#define currtok Context.Tokens.current() -#define prevtok Context.Tokens.previous() -#define nexttok Context.Tokens.next() -#define eat( Type_ ) Context.Tokens.__eat( Type_ ) -#define left ( Context.Tokens.Arr.num() - Context.Tokens.Idx ) - -#define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ ) -#define check( Type_ ) ( left && currtok.Type == Type_ ) - -#define push_scope() \ - StackNode scope { nullptr, currtok_noskip, NullToken, txt( __func__ ) }; \ - Context.push( &scope ) - -#pragma endregion Helper Macros - - // Procedure Forwards ( Entire parser internal parser interface ) - - internal Code parse_array_decl(); - internal CodeAttributes parse_attributes(); - internal CodeComment parse_comment(); - internal Code parse_complicated_definition( TokType which ); - internal CodeBody parse_class_struct_body( TokType which, Token name = NullToken ); - internal Code parse_class_struct( TokType which, bool inplace_def ); - internal CodeDefine parse_define(); - internal Code parse_expression(); - internal Code parse_forward_or_definition( TokType which, bool is_inplace ); - internal CodeFn parse_function_after_name( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeType ret_type, Token name ); - internal Code parse_function_body(); - internal Code parse_global_nspace(); - internal Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ); - internal Token parse_identifier( bool* possible_member_function = nullptr ); - internal CodeInclude parse_include(); - internal CodeOperator parse_operator_after_ret_type( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeType ret_type ); - internal Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ); - internal CodePragma parse_pragma(); - internal CodeParam parse_params( bool use_template_capture = false ); - internal CodePreprocessCond parse_preprocess_cond(); - internal Code parse_simple_preprocess( TokType which ); - internal Code parse_static_assert(); - internal void parse_template_args( Token& token ); - internal CodeVar parse_variable_after_name( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeType type, StrC name ); - internal CodeVar parse_variable_declaration_list(); - - internal CodeClass parse_class( bool inplace_def = false ); - internal CodeConstructor parse_constructor( CodeSpecifiers specifiers ); - internal CodeDestructor parse_destructor( CodeSpecifiers specifiers = NoCode ); - internal CodeEnum parse_enum( bool inplace_def = false ); - internal CodeBody parse_export_body(); - internal CodeBody parse_extern_link_body(); - internal CodeExtern parse_extern_link(); - internal CodeFriend parse_friend(); - internal CodeFn parse_function(); - internal CodeNS parse_namespace(); - internal CodeOpCast parse_operator_cast( CodeSpecifiers specifiers = NoCode ); - internal CodeStruct parse_struct( bool inplace_def = false ); - internal CodeVar parse_variable(); - internal CodeTemplate parse_template(); - internal CodeType parse_type( bool from_template = false, bool* is_function = nullptr ); - internal CodeTypedef parse_typedef(); - internal CodeUnion parse_union( bool inplace_def = false ); - internal CodeUsing parse_using(); - - constexpr bool inplace_def = true; - - // Internal parsing functions - - constexpr bool strip_formatting_dont_preserve_newlines = false; - - /* - This function was an attempt at stripping formatting from any c++ code. - It has edge case failures that prevent it from being used in function bodies. - */ - internal String strip_formatting( StrC raw_text, bool preserve_newlines = true ) - { - String content = String::make_reserve( GlobalAllocator, raw_text.Len ); - - if ( raw_text.Len == 0 ) - return content; - -#define cut_length ( scanner - raw_text.Ptr - last_cut ) -#define cut_ptr ( raw_text.Ptr + last_cut ) -#define pos ( sptr( scanner ) - sptr( raw_text.Ptr ) ) -#define move_fwd() \ - do \ - { \ - scanner++; \ - tokleft--; \ - } while ( 0 ) - - s32 tokleft = raw_text.Len; - sptr last_cut = 0; - char const* scanner = raw_text.Ptr; - - if ( scanner[0] == ' ' ) - { - move_fwd(); - last_cut = 1; - } - - bool within_string = false; - bool within_char = false; - bool must_keep_newline = false; - while ( tokleft ) - { - // Skip over the content of string literals - if ( scanner[0] == '"' ) - { - move_fwd(); - - while ( tokleft && ( scanner[0] != '"' || *( scanner - 1 ) == '\\' ) ) - { - if ( scanner[0] == '\\' && tokleft > 1 ) - { - scanner += 2; - tokleft -= 2; - } - else - { - move_fwd(); - } - } - - // Skip the closing " - if ( tokleft ) - move_fwd(); - - content.append( cut_ptr, cut_length ); - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - // Skip over the content of character literals - if ( scanner[0] == '\'' ) - { - move_fwd(); - - while ( tokleft && ( scanner[0] != '\'' || ( *( scanner - 1 ) == '\\' ) ) ) - { - move_fwd(); - } - - // Skip the closing ' - if ( tokleft ) - move_fwd(); - - content.append( cut_ptr, cut_length ); - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - // Block comments - if ( tokleft > 1 && scanner[0] == '/' && scanner[1] == '*' ) - { - while ( tokleft > 1 && ! ( scanner[0] == '*' && scanner[1] == '/' ) ) - move_fwd(); - - scanner += 2; - tokleft -= 2; - - content.append( cut_ptr, cut_length ); - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - // Line comments - if ( tokleft > 1 && scanner[0] == '/' && scanner[1] == '/' ) - { - must_keep_newline = true; - - scanner += 2; - tokleft -= 2; - - while ( tokleft && scanner[0] != '\n' ) - move_fwd(); - - if ( tokleft ) - move_fwd(); - - content.append( cut_ptr, cut_length ); - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - // Tabs - if ( scanner[0] == '\t' ) - { - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - if ( content.back() != ' ' ) - content.append( ' ' ); - - move_fwd(); - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - if ( tokleft > 1 && scanner[0] == '\r' && scanner[1] == '\n' ) - { - if ( must_keep_newline || preserve_newlines ) - { - must_keep_newline = false; - - scanner += 2; - tokleft -= 2; - - content.append( cut_ptr, cut_length ); - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - // Replace with a space - if ( content.back() != ' ' ) - content.append( ' ' ); - - scanner += 2; - tokleft -= 2; - - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - if ( scanner[0] == '\n' ) - { - if ( must_keep_newline || preserve_newlines ) - { - must_keep_newline = false; - - move_fwd(); - - content.append( cut_ptr, cut_length ); - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - if ( pos > last_cut ) - content.append( cut_ptr, cut_length ); - - // Replace with a space - if ( content.back() != ' ' ) - content.append( ' ' ); - - move_fwd(); - - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - // Escaped newlines - if ( scanner[0] == '\\' ) - { - content.append( cut_ptr, cut_length ); - - s32 amount_to_skip = 1; - if ( tokleft > 1 && scanner[1] == '\n' ) - { - amount_to_skip = 2; - } - else if ( tokleft > 2 && scanner[1] == '\r' && scanner[2] == '\n' ) - { - amount_to_skip = 3; - } - - if ( amount_to_skip > 1 && pos == last_cut ) - { - scanner += amount_to_skip; - tokleft -= amount_to_skip; - } - else - move_fwd(); - - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - continue; - } - - // Consectuive spaces - if ( tokleft > 1 && char_is_space( scanner[0] ) && char_is_space( scanner[1] ) ) - { - content.append( cut_ptr, cut_length ); - do - { - move_fwd(); - } while ( tokleft && char_is_space( scanner[0] ) ); - - last_cut = sptr( scanner ) - sptr( raw_text.Ptr ); - - // Preserve only 1 space of formattting - if ( content.back() != ' ' ) - content.append( ' ' ); - - continue; - } - - move_fwd(); - } - - if ( last_cut < raw_text.Len ) - { - content.append( cut_ptr, raw_text.Len - last_cut ); - } - -#undef cut_ptr -#undef cut_length -#undef pos -#undef move_fwd - - return content; - } - - internal Code parse_array_decl() - { - push_scope(); - - if ( check( TokType::Operator ) && currtok.Text[0] == '[' && currtok.Text[1] == ']' ) - { - Code array_expr = untyped_str( get_cached_string(txt(" ")) ); - eat( TokType::Operator ); - // [] - - Context.pop(); - return array_expr; - } - - if ( check( TokType::BraceSquare_Open ) ) - { - eat( TokType::BraceSquare_Open ); - // [ - - if ( left == 0 ) - { - log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - if ( currtok.Type == TokType::BraceSquare_Close ) - { - log_failure( "Error, empty array expression in definition\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - Token untyped_tok = currtok; - - while ( left && currtok.Type != TokType::BraceSquare_Close ) - { - eat( currtok.Type ); - } - - untyped_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)untyped_tok.Text; - - Code array_expr = untyped_str( untyped_tok ); - // [ - - if ( left == 0 ) - { - log_failure( "Error, unexpected end of array declaration, expected ]\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - if ( currtok.Type != TokType::BraceSquare_Close ) - { - log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", ETokType::to_str( currtok.Type ), Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - eat( TokType::BraceSquare_Close ); - // [ ] - - // Its a multi-dimensional array - if ( check( TokType::BraceSquare_Open ) ) - { - Code adjacent_arr_expr = parse_array_decl(); - // [ ][ ]... - - array_expr->Next = adjacent_arr_expr.ast; - } - - Context.pop(); - return array_expr; - } - - Context.pop(); - return { nullptr }; - } - - internal inline CodeAttributes parse_attributes() - { - push_scope(); - - Token start = currtok; - s32 len = 0; - - // There can be more than one attribute. If there is flatten them to a single string. - // TODO(Ed): Support keeping an linked list of attributes similar to parameters - while ( left && currtok.is_attribute() ) - { - if ( check( TokType::Attribute_Open ) ) - { - eat( TokType::Attribute_Open ); - // [[ - - while ( left && currtok.Type != TokType::Attribute_Close ) - { - eat( currtok.Type ); - } - // [[ - - eat( TokType::Attribute_Close ); - // [[ ]] - - len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; - } - else if ( check( TokType::Decl_GNU_Attribute ) ) - { - eat( TokType::Decl_GNU_Attribute ); - eat( TokType::Capture_Start ); - eat( TokType::Capture_Start ); - // __attribute__(( - - while ( left && currtok.Type != TokType::Capture_End ) - { - eat( currtok.Type ); - } - // __attribute__(( - - eat( TokType::Capture_End ); - eat( TokType::Capture_End ); - // __attribute__(( )) - - len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; - } - else if ( check( TokType::Decl_MSVC_Attribute ) ) - { - eat( TokType::Decl_MSVC_Attribute ); - eat( TokType::Capture_Start ); - // __declspec( - - while ( left && currtok.Type != TokType::Capture_End ) - { - eat( currtok.Type ); - } - // __declspec( - - eat( TokType::Capture_End ); - // __declspec( ) - - len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; - } - else if ( currtok.is_attribute() ) - { - eat( currtok.Type ); - // - - // If its a macro based attribute, this could be a functional macro such as Unreal's UE_DEPRECATED(...) - if ( check( TokType::Capture_Start ) ) - { - eat( TokType::Capture_Start ); - - s32 level = 0; - while ( left && currtok.Type != TokType::Capture_End && level == 0 ) - { - if ( currtok.Type == TokType::Capture_Start ) - ++level; - if ( currtok.Type == TokType::Capture_End ) - --level; - eat( currtok.Type ); - } - eat( TokType::Capture_End ); - } - - len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; - // ( ... ) - } - } - - if ( len > 0 ) - { - StrC attribute_txt = { len, start.Text }; - Context.pop(); - - String name_stripped = strip_formatting( attribute_txt, strip_formatting_dont_preserve_newlines ); - - Code result = make_code(); - result->Type = ECode::PlatformAttributes; - result->Name = get_cached_string( name_stripped ); - result->Content = result->Name; - // result->Token = - - return (CodeAttributes)result; - } - - Context.pop(); - return { nullptr }; - } - - internal Code parse_class_struct( TokType which, bool inplace_def = false ) - { - if ( which != TokType::Decl_Class && which != TokType::Decl_Struct ) - { - log_failure( "Error, expected class or struct, not %s\n%s", ETokType::to_str( which ), Context.to_string() ); - return CodeInvalid; - } - - Token name { nullptr, 0, TokType::Invalid }; - - AccessSpec access = AccessSpec::Default; - CodeType parent = { nullptr }; - CodeBody body = { nullptr }; - CodeAttributes attributes = { nullptr }; - ModuleFlag mflags = ModuleFlag::None; - - CodeClass result = CodeInvalid; - - if ( check( TokType::Module_Export ) ) - { - mflags = ModuleFlag::Export; - eat( TokType::Module_Export ); - } - // - - eat( which ); - // - - attributes = parse_attributes(); - // - - if ( check( TokType::Identifier ) ) - { - name = parse_identifier(); - Context.Scope->Name = name; - } - // - - local_persist char interface_arr_mem[kilobytes( 4 )] { 0 }; - Array interfaces = Array::init_reserve( Arena::init_from_memory( interface_arr_mem, kilobytes( 4 ) ), 4 ); - - // TODO(Ed) : Make an AST_DerivedType, we'll store any arbitary derived type into there as a linear linked list of them. - if ( check( TokType::Assign_Classifer ) ) - { - eat( TokType::Assign_Classifer ); - // : - - if ( currtok.is_access_specifier() ) - { - access = currtok.to_access_specifier(); - // : - eat( currtok.Type ); - } - - Token parent_tok = parse_identifier(); - parent = def_type( parent_tok ); - // : - - while ( check( TokType::Comma ) ) - { - eat( TokType::Comma ); - // : , - - if ( currtok.is_access_specifier() ) - { - eat( currtok.Type ); - } - Token interface_tok = parse_identifier(); - - interfaces.append( def_type( interface_tok ) ); - // : , ... - } - } - - if ( check( TokType::BraceCurly_Open ) ) - { - body = parse_class_struct_body( which, name ); - } - // : , ... { } - - CodeComment inline_cmt = NoCode; - if ( ! inplace_def ) - { - Token stmt_end = currtok; - eat( TokType::Statement_End ); - // : , ... { }; - - if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); - // : , ... { }; - } - - if ( which == TokType::Decl_Class ) - result = def_class( name, body, parent, access, attributes, mflags ); - - else - result = def_struct( name, body, (CodeType)parent, access, attributes, mflags ); - - if ( inline_cmt ) - result->InlineCmt = inline_cmt; - - interfaces.free(); - return result; - } - - internal neverinline CodeBody parse_class_struct_body( TokType which, Token name ) - { - using namespace ECode; - push_scope(); - - eat( TokType::BraceCurly_Open ); - // { - - CodeBody result = (CodeBody)make_code(); - - if ( which == TokType::Decl_Class ) - result->Type = Class_Body; - - else - result->Type = Struct_Body; - - while ( left && currtok_noskip.Type != TokType::BraceCurly_Close ) - { - Code member = Code::Invalid; - CodeAttributes attributes = { nullptr }; - CodeSpecifiers specifiers = { nullptr }; - - bool expects_function = false; - - // Context.Scope->Start = currtok_noskip; - - if ( currtok_noskip.Type == TokType::Preprocess_Hash ) - eat( TokType::Preprocess_Hash ); - - switch ( currtok_noskip.Type ) - { - case TokType::Statement_End : - { - // TODO(Ed): Convert this to a general warning procedure - log_fmt( "Dangling end statement found %S\n", currtok_noskip.to_string() ); - eat( TokType::Statement_End ); - continue; - } - case TokType::NewLine : - member = fmt_newline; - eat( TokType::NewLine ); - break; - - case TokType::Comment : - member = parse_comment(); - break; - - case TokType::Access_Public : - member = access_public; - eat( TokType::Access_Public ); - eat( TokType::Assign_Classifer ); - // public: - break; - - case TokType::Access_Protected : - member = access_protected; - eat( TokType::Access_Protected ); - eat( TokType::Assign_Classifer ); - // protected: - break; - - case TokType::Access_Private : - member = access_private; - eat( TokType::Access_Private ); - eat( TokType::Assign_Classifer ); - // private: - break; - - case TokType::Decl_Class : - member = parse_complicated_definition( TokType::Decl_Class ); - // class - break; - - case TokType::Decl_Enum : - member = parse_complicated_definition( TokType::Decl_Enum ); - // enum - break; - - case TokType::Decl_Friend : - member = parse_friend(); - // friend - break; - - case TokType::Decl_Operator : - member = parse_operator_cast(); - // operator () - break; - - case TokType::Decl_Struct : - member = parse_complicated_definition( TokType::Decl_Struct ); - // struct - break; - - case TokType::Decl_Template : - member = parse_template(); - // template< ... > - break; - - case TokType::Decl_Typedef : - member = parse_typedef(); - // typedef - break; - - case TokType::Decl_Union : - member = parse_complicated_definition( TokType::Decl_Union ); - // union - break; - - case TokType::Decl_Using : - member = parse_using(); - // using - break; - - case TokType::Operator : - if ( currtok.Text[0] != '~' ) - { - log_failure( "Operator token found in global body but not destructor unary negation\n%s", Context.to_string() ); - return CodeInvalid; - } - - member = parse_destructor(); - // ~() - break; - - case TokType::Preprocess_Define : - member = parse_define(); - // #define - break; - - case TokType::Preprocess_Include : - member = parse_include(); - // #include - break; - - case TokType::Preprocess_If : - case TokType::Preprocess_IfDef : - case TokType::Preprocess_IfNotDef : - case TokType::Preprocess_ElIf : - member = parse_preprocess_cond(); - // # - break; - - case TokType::Preprocess_Else : - member = preprocess_else; - eat( TokType::Preprocess_Else ); - // #else - break; - - case TokType::Preprocess_EndIf : - member = preprocess_endif; - eat( TokType::Preprocess_EndIf ); - // #endif - break; - - case TokType::Preprocess_Macro : - member = parse_simple_preprocess( TokType::Preprocess_Macro ); - // - break; - - case TokType::Preprocess_Pragma : - member = parse_pragma(); - // #pragma - break; - - case TokType::Preprocess_Unsupported : - member = parse_simple_preprocess( TokType::Preprocess_Unsupported ); - // # - break; - - case TokType::StaticAssert : - member = parse_static_assert(); - // static_assert - break; - - case TokType::Attribute_Open : - case TokType::Decl_GNU_Attribute : - case TokType::Decl_MSVC_Attribute : -#define Entry( attribute, str ) case TokType::attribute : - GEN_DEFINE_ATTRIBUTE_TOKENS -#undef Entry - { - attributes = parse_attributes(); - // - } - //! Fallthrough intended - case TokType::Spec_Consteval : - case TokType::Spec_Constexpr : - case TokType::Spec_Constinit : - case TokType::Spec_Explicit : - case TokType::Spec_gb_inline : - case TokType::Spec_Inline : - case TokType::Spec_Mutable : - case TokType::Spec_NeverInline : - case TokType::Spec_Static : - case TokType::Spec_Volatile : - case TokType::Spec_Virtual : - { - SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; - s32 NumSpecifiers = 0; - - while ( left && currtok.is_specifier() ) - { - SpecifierT spec = ESpecifier::to_type( currtok ); - - b32 ignore_spec = false; - - switch ( spec ) - { - case ESpecifier::Constexpr : - case ESpecifier::Constinit : - case ESpecifier::Explicit : - case ESpecifier::Inline : - case ESpecifier::gb_inline : - case ESpecifier::Mutable : - case ESpecifier::NeverInline : - case ESpecifier::Static : - case ESpecifier::Volatile : - case ESpecifier::Virtual : - break; - - case ESpecifier::Consteval : - expects_function = true; - break; - - case ESpecifier::Const : - ignore_spec = true; - break; - - default : - log_failure( "Invalid specifier %s for variable\n%s", ESpecifier::to_str( spec ), Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - // Every specifier after would be considered part of the type type signature - if ( ignore_spec ) - break; - - specs_found[NumSpecifiers] = spec; - NumSpecifiers++; - eat( currtok.Type ); - } - - if ( NumSpecifiers ) - { - specifiers = def_specifiers( NumSpecifiers, specs_found ); - } - // - - if ( currtok.is_attribute() ) - { - // Unfortuantely Unreal has code where there is attirbutes before specifiers - CodeAttributes more_attributes = parse_attributes(); - - if ( attributes ) - { - String fused = String::make_reserve( GlobalAllocator, attributes->Content.length() + more_attributes->Content.length() ); - fused.append_fmt( "%S %S", attributes->Content, more_attributes->Content ); - - attributes->Name = get_cached_string( fused ); - attributes->Content = attributes->Name; - // - } - - attributes = more_attributes; - } - - if ( currtok.Type == TokType::Operator && currtok.Text[0] == '~' ) - { - member = parse_destructor( specifiers ); - // ~() - break; - } - - if ( currtok.Type == TokType::Decl_Operator ) - { - member = parse_operator_cast( specifiers ); - // operator () - break; - } - } - //! Fallthrough intentional - case TokType::Identifier : - case TokType::Spec_Const : - case TokType::Type_Unsigned : - case TokType::Type_Signed : - case TokType::Type_Short : - case TokType::Type_Long : - case TokType::Type_bool : - case TokType::Type_char : - case TokType::Type_int : - case TokType::Type_double : - { - if ( nexttok.Type == TokType::Capture_Start && name.Length && currtok.Type == TokType::Identifier ) - { - if ( str_compare( name.Text, currtok.Text, name.Length ) == 0 ) - { - member = parse_constructor( specifiers ); - // () - break; - } - } - - member = parse_operator_function_or_variable( expects_function, attributes, specifiers ); - // operator ... - // or - // ... - } - break; - - default : - Token untyped_tok = currtok; - - while ( left && currtok.Type != TokType::BraceCurly_Close ) - { - untyped_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)untyped_tok.Text; - eat( currtok.Type ); - } - - member = untyped_str( untyped_tok ); - // Something unknown - break; - } - - if ( member == Code::Invalid ) - { - log_failure( "Failed to parse member\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - result.append( member ); - } - - eat( TokType::BraceCurly_Close ); - // { } - Context.pop(); - return result; - } - - internal CodeComment parse_comment() - { - StackNode scope { nullptr, currtok_noskip, NullToken, txt( __func__ ) }; - Context.push( &scope ); - - CodeComment result = (CodeComment)make_code(); - result->Type = ECode::Comment; - result->Content = get_cached_string( currtok_noskip ); - result->Name = result->Content; - // result->Token = currtok_noskip; - eat( TokType::Comment ); - - Context.pop(); - return result; - } - - internal Code parse_complicated_definition( TokType which ) - { - push_scope(); - - bool is_inplace = false; - - TokArray tokens = Context.Tokens; - - s32 idx = tokens.Idx; - s32 level = 0; - for ( ; idx < tokens.Arr.num(); idx++ ) - { - if ( tokens[idx].Type == TokType::BraceCurly_Open ) - level++; - - if ( tokens[idx].Type == TokType::BraceCurly_Close ) - level--; - - if ( level == 0 && tokens[idx].Type == TokType::Statement_End ) - break; - } - - if ( ( idx - 2 ) == tokens.Idx ) - { - // Its a forward declaration only - Code result = parse_forward_or_definition( which, is_inplace ); - // ; - Context.pop(); - return result; - } - - Token tok = tokens[idx - 1]; - if ( tok.is_specifier() && is_trailing( ESpecifier::to_type( tok ) ) ) - { - // (...) ...; - - s32 spec_idx = idx - 1; - Token spec = tokens[spec_idx]; - while ( spec.is_specifier() && is_trailing( ESpecifier::to_type( spec ) ) ) - { - --spec_idx; - spec = tokens[spec_idx]; - } - - if ( tokens[spec_idx].Type == TokType::Capture_End ) - { - // Forward declaration with trailing specifiers for a procedure - tok = tokens[spec_idx]; - - Code result = parse_operator_function_or_variable( false, { nullptr }, { nullptr } ); - // , or Name> ... - Context.pop(); - return result; - } - - log_failure( "Unsupported or bad member definition after %s declaration\n%s", to_str( which ), Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - if ( tok.Type == TokType::Identifier ) - { - tok = tokens[idx - 2]; - bool is_indirection = tok.Type == TokType::Ampersand || tok.Type == TokType::Star; - bool ok_to_parse = false; - - if ( tok.Type == TokType::BraceCurly_Close ) - { - // Its an inplace definition - // { ... } ; - ok_to_parse = true; - is_inplace = true; - } - else if ( tok.Type == TokType::Identifier && tokens[idx - 3].Type == which ) - { - // Its a variable with type ID using namespace. - // ; - ok_to_parse = true; - } - else if ( tok.Type == TokType::Assign_Classifer - && ( ( tokens[idx - 5].Type == which && tokens[idx - 4].Type == TokType::Decl_Class ) || ( tokens[idx - 4].Type == which ) ) ) - { - // Its a forward declaration of an enum - // : ; - // : ; - ok_to_parse = true; - Code result = parse_enum(); - Context.pop(); - return result; - } - else if ( is_indirection ) - { - // Its a indirection type with type ID using struct namespace. - // * ; - ok_to_parse = true; - } - - if ( ! ok_to_parse ) - { - log_failure( "Unsupported or bad member definition after %s declaration\n%s", to_str( which ), Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - Code result = parse_operator_function_or_variable( false, { nullptr }, { nullptr } ); - // , or Name> ... - Context.pop(); - return result; - } - else if ( tok.Type >= TokType::Type_Unsigned && tok.Type <= TokType::Type_MS_W64 ) - { - tok = tokens[idx - 2]; - - if ( tok.Type != TokType::Assign_Classifer - || ( ( tokens[idx - 5].Type != which && tokens[idx - 4].Type != TokType::Decl_Class ) && ( tokens[idx - 4].Type != which ) ) ) - { - log_failure( "Unsupported or bad member definition after %s declaration\n%s", to_str( which ), Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - // Its a forward declaration of an enum class - // : ; - // : ; - Code result = parse_enum(); - Context.pop(); - return result; - } - else if ( tok.Type == TokType::BraceCurly_Close ) - { - // Its a definition - Code result = parse_forward_or_definition( which, is_inplace ); - // { ... }; - Context.pop(); - return result; - } - else if ( tok.Type == TokType::BraceSquare_Close ) - { - // Its an array definition - Code result = parse_operator_function_or_variable( false, { nullptr }, { nullptr } ); - // [ ... ]; - Context.pop(); - return result; - } - else - { - log_failure( "Unsupported or bad member definition after %s declaration\n%S", to_str( which ).Ptr, Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - } - - internal inline CodeDefine parse_define() - { - push_scope(); - eat( TokType::Preprocess_Define ); - // #define - - CodeDefine define = (CodeDefine)make_code(); - define->Type = ECode::Preprocess_Define; - - if ( ! check( TokType::Identifier ) ) - { - log_failure( "Error, expected identifier after #define\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - Context.Scope->Name = currtok; - define->Name = get_cached_string( currtok ); - eat( TokType::Identifier ); - // #define - - if ( ! check( TokType::Preprocess_Content ) ) - { - log_failure( "Error, expected content after #define %s\n%s", define->Name, Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - if ( currtok.Length == 0 ) - { - define->Content = get_cached_string( currtok ); - eat( TokType::Preprocess_Content ); - // #define - - Context.pop(); - return define; - } - - define->Content = get_cached_string( strip_formatting( currtok, strip_formatting_dont_preserve_newlines ) ); - eat( TokType::Preprocess_Content ); - // #define - - Context.pop(); - return define; - } - - internal inline Code parse_assignment_expression() - { - Code expr = { nullptr }; - - eat( TokType::Operator ); - // = - - Token expr_tok = currtok; - - if ( currtok.Type == TokType::Statement_End && currtok.Type != TokType::Comma ) - { - log_failure( "Expected expression after assignment operator\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - s32 level = 0; - while ( left && currtok.Type != TokType::Statement_End && (currtok.Type != TokType::Comma || level > 0 ) ) - { - if (currtok.Type == TokType::BraceCurly_Open ) - level++; - if (currtok.Type == TokType::BraceCurly_Close ) - level--; - if ( currtok.Type == TokType::Capture_Start ) - level++; - else if ( currtok.Type == TokType::Capture_End ) - level--; - - eat( currtok.Type ); - } - - expr_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)expr_tok.Text - 1; - expr = untyped_str( expr_tok ); - // = - return expr; - } - - internal inline Code parse_forward_or_definition( TokType which, bool is_inplace ) - { - Code result = CodeInvalid; - - switch ( which ) - { - case TokType::Decl_Class : - result = parse_class( is_inplace ); - return result; - - case TokType::Decl_Enum : - result = parse_enum( is_inplace ); - return result; - - case TokType::Decl_Struct : - result = parse_struct( is_inplace ); - return result; - - case TokType::Decl_Union : - result = parse_union( is_inplace ); - return result; - - default : - log_failure( - "Error, wrong token type given to parse_complicated_definition " - "(only supports class, enum, struct, union) \n%s", - Context.to_string() - ); - - return CodeInvalid; - } - - return CodeInvalid; - } - - // Function parsing is handled in multiple places because its initial signature is shared with variable parsing - internal inline CodeFn parse_function_after_name( ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeType ret_type, Token name ) - { - push_scope(); - CodeParam params = parse_params(); - // ( ) - - // TODO(Ed), Review old comment : These have to be kept separate from the return type's specifiers. - while ( left && currtok.is_specifier() ) - { - if ( specifiers.ast == nullptr ) - { - specifiers = def_specifier( ESpecifier::to_type( currtok ) ); - eat( currtok.Type ); - continue; - } - - specifiers.append( ESpecifier::to_type( currtok ) ); - eat( currtok.Type ); - } - // ( ) - - CodeBody body = NoCode; - CodeComment inline_cmt = NoCode; - if ( check( TokType::BraceCurly_Open ) ) - { - body = parse_function_body(); - if ( body == Code::Invalid ) - { - Context.pop(); - return CodeInvalid; - } - // ( ) { } - } - else if ( check( TokType::Operator ) && currtok.Text[0] == '=' ) - { - eat( TokType::Operator ); - specifiers.append( ESpecifier::Pure ); - - eat( TokType::Number ); - Token stmt_end = currtok; - eat( TokType::Statement_End ); - // ( ) = 0; - - if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); - // ( ) ; - } - else - { - Token stmt_end = currtok; - eat( TokType::Statement_End ); - // ( ) ; - - if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line ) - inline_cmt = parse_comment(); - // ( ) ; - } - - using namespace ECode; - - String name_stripped = String::make( GlobalAllocator, name ); - name_stripped.strip_space(); - - CodeFn result = (CodeFn)make_code(); - result->Name = get_cached_string( name_stripped ); - result->ModuleFlags = mflags; - - if ( body ) - { - switch ( body->Type ) - { - case Function_Body : - case Untyped : - break; - - default : - { - log_failure( "Body must be either of Function_Body or Untyped type, %s\n%s", body.debug_str(), Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - } - - result->Type = Function; - result->Body = body; - } - else - { - result->Type = Function_Fwd; - } - - if ( attributes ) - result->Attributes = attributes; - - if ( specifiers ) - result->Specs = specifiers; - - result->ReturnType = ret_type; - - if ( params ) - result->Params = params; - - if ( inline_cmt ) - result->InlineCmt = inline_cmt; - - Context.pop(); - return result; - } - - internal Code parse_function_body() - { - using namespace ECode; - push_scope(); - - eat( TokType::BraceCurly_Open ); - - CodeBody result = (CodeBody)make_code(); - result->Type = Function_Body; - - // TODO : Support actual parsing of function body - Token start = currtok_noskip; - - s32 level = 0; - while ( left && ( currtok_noskip.Type != TokType::BraceCurly_Close || level > 0 ) ) - { - if ( currtok_noskip.Type == TokType::BraceCurly_Open ) - level++; - - else if ( currtok_noskip.Type == TokType::BraceCurly_Close && level > 0 ) - level--; - - eat( currtok_noskip.Type ); - } - - Token previous = prevtok; - - s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; - - if ( len > 0 ) - { - result.append( def_execution( { len, start.Text } ) ); - } - - eat( TokType::BraceCurly_Close ); - - Context.pop(); - return result; - } - - internal neverinline CodeBody parse_global_nspace( CodeT which ) - { - using namespace ECode; - - push_scope(); - - if ( which != Namespace_Body && which != Global_Body && which != Export_Body && which != Extern_Linkage_Body ) - return CodeInvalid; - - if ( which != Global_Body ) - eat( TokType::BraceCurly_Open ); - // { - - CodeBody result = (CodeBody)make_code(); - result->Type = which; - - while ( left && currtok_noskip.Type != TokType::BraceCurly_Close ) - { - Code member = Code::Invalid; - CodeAttributes attributes = { nullptr }; - CodeSpecifiers specifiers = { nullptr }; - - bool expects_function = false; - - // Context.Scope->Start = currtok_noskip; - - if ( currtok_noskip.Type == TokType::Preprocess_Hash ) - eat( TokType::Preprocess_Hash ); - - switch ( currtok_noskip.Type ) - { - case TokType::Statement_End : - { - // TODO(Ed): Convert this to a general warning procedure - log_fmt( "Dangling end statement found %S\n", currtok_noskip.to_string() ); - eat( TokType::Statement_End ); - continue; - } - case TokType::NewLine : - // Empty lines are auto skipped by Tokens.current() - member = fmt_newline; - eat( TokType::NewLine ); - break; - - case TokType::Comment : - member = parse_comment(); - break; - - case TokType::Decl_Class : - member = parse_complicated_definition( TokType::Decl_Class ); - // class - break; - - case TokType::Decl_Enum : - member = parse_complicated_definition( TokType::Decl_Enum ); - // enum - break; - - case TokType::Decl_Extern_Linkage : - if ( which == Extern_Linkage_Body ) - log_failure( "Nested extern linkage\n%s", Context.to_string() ); - - member = parse_extern_link(); - // extern "..." { ... } - break; - - case TokType::Decl_Namespace : - member = parse_namespace(); - // namespace { ... } - break; - - case TokType::Decl_Struct : - member = parse_complicated_definition( TokType::Decl_Struct ); - // struct ... - break; - - case TokType::Decl_Template : - member = parse_template(); - // template<...> ... - break; - - case TokType::Decl_Typedef : - member = parse_typedef(); - // typedef ... - break; - - case TokType::Decl_Union : - member = parse_complicated_definition( TokType::Decl_Union ); - // union ... - break; - - case TokType::Decl_Using : - member = parse_using(); - // using ... - break; - - case TokType::Preprocess_Define : - member = parse_define(); - // #define ... - break; - - case TokType::Preprocess_Include : - member = parse_include(); - // #include ... - break; - - case TokType::Preprocess_If : - case TokType::Preprocess_IfDef : - case TokType::Preprocess_IfNotDef : - case TokType::Preprocess_ElIf : - member = parse_preprocess_cond(); - // # ... - break; - - case TokType::Preprocess_Else : - member = preprocess_else; - eat( TokType::Preprocess_Else ); - // #else - break; - - case TokType::Preprocess_EndIf : - member = preprocess_endif; - eat( TokType::Preprocess_EndIf ); - // #endif - break; - - case TokType::Preprocess_Macro : - member = parse_simple_preprocess( TokType::Preprocess_Macro ); - // - break; - - case TokType::Preprocess_Pragma : - member = parse_pragma(); - // #pragma ... - break; - - case TokType::Preprocess_Unsupported : - member = parse_simple_preprocess( TokType::Preprocess_Unsupported ); - // # ... - break; - - case TokType::StaticAssert : - member = parse_static_assert(); - // static_assert( , ... ); - break; - - case TokType::Module_Export : - if ( which == Export_Body ) - log_failure( "Nested export declaration\n%s", Context.to_string() ); - - member = parse_export_body(); - // export { ... } - break; - - case TokType::Module_Import : - { - not_implemented( context ); - // import ... - } - //! Fallthrough intentional - case TokType::Attribute_Open : - case TokType::Decl_GNU_Attribute : - case TokType::Decl_MSVC_Attribute : -#define Entry( attribute, str ) case TokType::attribute : - GEN_DEFINE_ATTRIBUTE_TOKENS -#undef Entry - { - attributes = parse_attributes(); - // - } - //! Fallthrough intentional - case TokType::Spec_Consteval : - case TokType::Spec_Constexpr : - case TokType::Spec_Constinit : - case TokType::Spec_Extern : - case TokType::Spec_gb_inline : - case TokType::Spec_gb_global : - case TokType::Spec_Inline : - case TokType::Spec_gb_internal : - case TokType::Spec_NeverInline : - case TokType::Spec_Static : - case TokType::Spec_gb_thread_local : - { - SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; - s32 NumSpecifiers = 0; - - while ( left && currtok.is_specifier() ) - { - SpecifierT spec = ESpecifier::to_type( currtok ); - - bool ignore_spec = false; - - switch ( spec ) - { - case ESpecifier::Constexpr : - case ESpecifier::Constinit : - case ESpecifier::gb_inline : - case ESpecifier::gb_global : - case ESpecifier::External_Linkage : - case ESpecifier::gb_internal : - case ESpecifier::Inline : - case ESpecifier::Mutable : - case ESpecifier::NeverInline : - case ESpecifier::Static : - case ESpecifier::Volatile : - case ESpecifier::gb_thread_local : - break; - - case ESpecifier::Consteval : - expects_function = true; - break; - - case ESpecifier::Const : - ignore_spec = true; - break; - - default : - StrC spec_str = ESpecifier::to_str( spec ); - - log_failure( "Invalid specifier %.*s for variable\n%s", spec_str.Len, spec_str, Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - if ( ignore_spec ) - break; - - specs_found[NumSpecifiers] = spec; - NumSpecifiers++; - eat( currtok.Type ); - } - - if ( NumSpecifiers ) - { - specifiers = def_specifiers( NumSpecifiers, specs_found ); - } - // - } - //! Fallthrough intentional - case TokType::Identifier : - case TokType::Spec_Const : - case TokType::Type_Long : - case TokType::Type_Short : - case TokType::Type_Signed : - case TokType::Type_Unsigned : - case TokType::Type_bool : - case TokType::Type_char : - case TokType::Type_double : - case TokType::Type_int : - { - Code constructor_destructor = parse_global_nspace_constructor_destructor( specifiers ); - // Possible constructor implemented at global file scope. - if ( constructor_destructor ) - { - member = constructor_destructor; - break; - } - - bool found_operator_cast_outside_class_implmentation = false; - s32 idx = Context.Tokens.Idx; - - for ( ; idx < Context.Tokens.Arr.num(); idx++ ) - { - Token tok = Context.Tokens[idx]; - - if ( tok.Type == TokType::Identifier ) - { - idx++; - tok = Context.Tokens[idx]; - if ( tok.Type == TokType::Access_StaticSymbol ) - continue; - - break; - } - - if ( tok.Type == TokType::Decl_Operator ) - found_operator_cast_outside_class_implmentation = true; - - break; - } - - if ( found_operator_cast_outside_class_implmentation ) - { - member = parse_operator_cast(); - // ::operator () { ... } - break; - } - - member = parse_operator_function_or_variable( expects_function, attributes, specifiers ); - // ... - } - } - - if ( member == Code::Invalid ) - { - log_failure( "Failed to parse member\n%s", Context.to_string() ); - Context.pop(); - return CodeInvalid; - } - - // log_fmt("Global Body Member: %s", member->debug_str()); - result.append( member ); - } - - if ( which != Global_Body ) - eat( TokType::BraceCurly_Close ); - // { } - - Context.pop(); - return result; - } - - internal inline Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) - { - Code result = { nullptr }; - - /* - To check if a definition is for a constructor we can go straight to the opening parenthesis for its parameters - From There we work backwards to see if we come across two identifiers with the same name between an member access - :: operator, there can be template parameters on the left of the :: so we ignore those. - Whats important is that its back to back. - - This has multiple possible faults. What we parse using this method may not filter out if something has a "return type" - This is bad since technically you could have a namespace nested into another namespace with the same name. - If this awful pattern is done the only way to distiguish with this coarse parse is to know there is no return type defined. - - TODO(Ed): We could fix this by attempting to parse a type, but we would have to have a way to have it soft fail and rollback. - */ - TokArray tokens = Context.Tokens; - - s32 idx = tokens.Idx; - Token nav = tokens[idx]; - for ( ; idx < tokens.Arr.num(); idx++, nav = tokens[idx] ) - { - if ( nav.Text[0] == '<' ) - { - // Skip templated expressions as they mey have expressions with the () operators - s32 capture_level = 0; - s32 template_level = 0; - for ( ; idx < tokens.Arr.num(); idx++, nav = tokens[idx] ) - { - if ( nav.Text[0] == '<' ) - ++template_level; - - if ( nav.Text[0] == '>' ) - --template_level; - if ( nav.Type == TokType::Operator && nav.Text[1] == '>' ) - --template_level; - - if ( nav.Type == ETokType::Capture_Start ) - { - if ( template_level != 0 ) - ++capture_level; - else - break; - } - - if ( template_level != 0 && nav.Type == ETokType::Capture_End ) - --capture_level; - } - } - - if ( nav.Type == TokType::Capture_Start ) - break; - } - - --idx; - Token tok_right = tokens[idx]; - Token tok_left = NullToken; - - if ( tok_right.Type != TokType::Identifier ) - { - // We're not dealing with a constructor if there is no identifier right before the opening of a parameter's scope. - return result; - } - - --idx; - tok_left = tokens[idx]; - // ... - - bool possible_destructor = false; - if ( tok_left.Type == TokType::Operator && tok_left.Text[0] == '~' ) - { - possible_destructor = true; - --idx; - tok_left = tokens[idx]; - } - - if ( tok_left.Type != TokType::Access_StaticSymbol ) - return result; - - --idx; - tok_left = tokens[idx]; - // ... :: - - // We search toward the left until we find the next valid identifier - s32 capture_level = 0; - s32 template_level = 0; - while ( idx != tokens.Idx ) - { - if ( tok_left.Text[0] == '<' ) - ++template_level; - - if ( tok_left.Text[0] == '>' ) - --template_level; - if ( tok_left.Type == TokType::Operator && tok_left.Text[1] == '>' ) - --template_level; - - if ( template_level != 0 && tok_left.Type == ETokType::Capture_Start ) - ++capture_level; - - if ( template_level != 0 && tok_left.Type == ETokType::Capture_End ) - --capture_level; - - if ( capture_level == 0 && template_level == 0 && tok_left.Type == TokType::Identifier ) - break; - - --idx; - tok_left = tokens[idx]; - } - - bool is_same = str_compare( tok_right.Text, tok_left.Text, tok_right.Length ) == 0; - if ( tok_left.Type == TokType::Identifier && is_same ) - { - // We have found the pattern we desired - if ( possible_destructor ) - { - // :: ~ ( - result = parse_destructor( specifiers ); - } - else - { - // :: ( - result = parse_constructor( specifiers ); - } - } - - return result; - } - - // TODO(Ed): I want to eventually change the identifier to its own AST type. - // This would allow distinction of the qualifier for a symbol :: - // This would also allow - internal Token parse_identifier( bool* possible_member_function ) - { - push_scope(); - - Token name = currtok; - Context.Scope->Name = name; - eat( TokType::Identifier ); - // - - parse_template_args( name ); - //