From c5afede7b5b77526fb432a088633b6e5c290fb4c Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sat, 29 Jul 2023 05:52:06 -0400 Subject: [PATCH] Reorganization of files, refactors, doc updates (WIP) Removing the gen. namespace from the files for components, dependencies, and file_processors. They are only necessary if the include directory is transparent, and in my case those are not. Made a docs directory. I'm offloading information from the main readme to there along with additional informationn I end up elaborating on down the line. Enum tables were moved to their own directory (project/enums). Library will not compile for now. Major refactor occuring with parsing related components. --- Docs/Parsing.md | 49 ++ Docs/Readme.md | 537 +++++++++++++++++ Readme.md | 564 +----------------- gencpp.vcxproj | 55 +- project/Readme.md | 40 +- project/components/{gen.ast.cpp => ast.cpp} | 97 +-- ...st_case_macros.cpp => ast_case_macros.cpp} | 0 ...ata_structures.hpp => data_structures.hpp} | 0 .../components/{gen.ecode.hpp => ecode.hpp} | 0 .../{gen.eoperator.hpp => eoperator.hpp} | 2 +- .../{gen.especifier.hpp => especifier.hpp} | 1 - .../{gen.etoktype.cpp => etoktype.cpp} | 10 +- .../{gen.header_end.hpp => header_end.hpp} | 0 ...{gen.header_start.hpp => header_start.hpp} | 0 .../{gen.impl_start.cpp => impl_start.cpp} | 0 .../{gen.interface.cpp => interface.cpp} | 6 +- .../{gen.interface.hpp => interface.hpp} | 0 ...face.parsing.cpp => interface.parsing.cpp} | 363 +++++++---- ...face.upfront.cpp => interface.upfront.cpp} | 0 .../{gen.data.cpp => static_data.cpp} | 0 .../components/{gen.types.hpp => types.hpp} | 2 +- .../{gen.untyped.cpp => untyped.cpp} | 3 +- .../{gen.basic_types.hpp => basic_types.hpp} | 0 .../{gen.containers.hpp => containers.hpp} | 0 .../dependencies/{gen.debug.cpp => debug.cpp} | 0 .../dependencies/{gen.debug.hpp => debug.hpp} | 0 ...en.file_handling.cpp => file_handling.cpp} | 0 ...en.file_handling.hpp => file_handling.hpp} | 0 .../{gen.hashing.cpp => hashing.cpp} | 0 .../{gen.hashing.hpp => hashing.hpp} | 0 ...{gen.header_start.hpp => header_start.hpp} | 0 .../{gen.impl_start.cpp => impl_start.cpp} | 0 .../{gen.macros.hpp => macros.hpp} | 0 .../{gen.memory.cpp => memory.cpp} | 0 .../{gen.memory.hpp => memory.hpp} | 0 .../{gen.parsing.cpp => parsing.cpp} | 0 .../{gen.parsing.hpp => parsing.hpp} | 0 .../{gen.printing.cpp => printing.cpp} | 0 .../{gen.printing.hpp => printing.hpp} | 7 +- .../{gen.string.cpp => string.cpp} | 0 .../{gen.string.hpp => string.hpp} | 0 .../{gen.string_ops.cpp => string_ops.cpp} | 0 .../{gen.string_ops.hpp => string_ops.hpp} | 0 .../{gen.timing.cpp => timing.cpp} | 0 .../{gen.timing.hpp => timing.hpp} | 0 .../{components => enums}/AttributeTokens.csv | 0 project/{components => enums}/ECode.csv | 0 project/{components => enums}/EOperator.csv | 0 project/{components => enums}/ESpecifier.csv | 0 project/{components => enums}/ETokType.csv | 0 .../builder.cpp} | 0 .../builder.hpp} | 0 .../editor.hpp} | 0 .../scanner.hpp} | 0 project/gen.bootstrap.cpp | 92 +-- project/gen.cpp | 22 +- project/gen.dep.cpp | 20 +- project/gen.dep.hpp | 25 +- project/gen.hpp | 22 +- .../helpers/{gen.helper.hpp => helper.hpp} | 0 ...ores.inline.hpp => pop_ignores.inline.hpp} | 0 ...res.inline.hpp => push_ignores.inline.hpp} | 0 ...{gen.undef.macros.hpp => undef.macros.hpp} | 0 scripts/Readme.md | 36 +- scripts/get_sources.ps1 | 12 - .../{build.ci.ps1 => test.gen_build.ci.ps1} | 0 scripts/{build.ps1 => test.gen_build.ps1} | 0 scripts/{gen.ps1 => test.gen_run.ps1} | 0 68 files changed, 1077 insertions(+), 888 deletions(-) create mode 100644 Docs/Parsing.md create mode 100644 Docs/Readme.md rename project/components/{gen.ast.cpp => ast.cpp} (87%) rename project/components/{gen.ast_case_macros.cpp => ast_case_macros.cpp} (100%) rename project/components/{gen.data_structures.hpp => data_structures.hpp} (100%) rename project/components/{gen.ecode.hpp => ecode.hpp} (100%) rename project/components/{gen.eoperator.hpp => eoperator.hpp} (98%) rename project/components/{gen.especifier.hpp => especifier.hpp} (99%) rename project/components/{gen.etoktype.cpp => etoktype.cpp} (96%) rename project/components/{gen.header_end.hpp => header_end.hpp} (100%) rename project/components/{gen.header_start.hpp => header_start.hpp} (100%) rename project/components/{gen.impl_start.cpp => impl_start.cpp} (100%) rename project/components/{gen.interface.cpp => interface.cpp} (98%) rename project/components/{gen.interface.hpp => interface.hpp} (100%) rename project/components/{gen.interface.parsing.cpp => interface.parsing.cpp} (87%) rename project/components/{gen.interface.upfront.cpp => interface.upfront.cpp} (100%) rename project/components/{gen.data.cpp => static_data.cpp} (100%) rename project/components/{gen.types.hpp => types.hpp} (96%) rename project/components/{gen.untyped.cpp => untyped.cpp} (97%) rename project/dependencies/{gen.basic_types.hpp => basic_types.hpp} (100%) rename project/dependencies/{gen.containers.hpp => containers.hpp} (100%) rename project/dependencies/{gen.debug.cpp => debug.cpp} (100%) rename project/dependencies/{gen.debug.hpp => debug.hpp} (100%) rename project/dependencies/{gen.file_handling.cpp => file_handling.cpp} (100%) rename project/dependencies/{gen.file_handling.hpp => file_handling.hpp} (100%) rename project/dependencies/{gen.hashing.cpp => hashing.cpp} (100%) rename project/dependencies/{gen.hashing.hpp => hashing.hpp} (100%) rename project/dependencies/{gen.header_start.hpp => header_start.hpp} (100%) rename project/dependencies/{gen.impl_start.cpp => impl_start.cpp} (100%) rename project/dependencies/{gen.macros.hpp => macros.hpp} (100%) rename project/dependencies/{gen.memory.cpp => memory.cpp} (100%) rename project/dependencies/{gen.memory.hpp => memory.hpp} (100%) rename project/dependencies/{gen.parsing.cpp => parsing.cpp} (100%) rename project/dependencies/{gen.parsing.hpp => parsing.hpp} (100%) rename project/dependencies/{gen.printing.cpp => printing.cpp} (100%) rename project/dependencies/{gen.printing.hpp => printing.hpp} (86%) rename project/dependencies/{gen.string.cpp => string.cpp} (100%) rename project/dependencies/{gen.string.hpp => string.hpp} (100%) rename project/dependencies/{gen.string_ops.cpp => string_ops.cpp} (100%) rename project/dependencies/{gen.string_ops.hpp => string_ops.hpp} (100%) rename project/dependencies/{gen.timing.cpp => timing.cpp} (100%) rename project/dependencies/{gen.timing.hpp => timing.hpp} (100%) rename project/{components => enums}/AttributeTokens.csv (100%) rename project/{components => enums}/ECode.csv (100%) rename project/{components => enums}/EOperator.csv (100%) rename project/{components => enums}/ESpecifier.csv (100%) rename project/{components => enums}/ETokType.csv (100%) rename project/{filesystem/gen.builder.cpp => file_processors/builder.cpp} (100%) rename project/{filesystem/gen.builder.hpp => file_processors/builder.hpp} (100%) rename project/{filesystem/gen.editor.hpp => file_processors/editor.hpp} (100%) rename project/{filesystem/gen.scanner.hpp => file_processors/scanner.hpp} (100%) rename project/helpers/{gen.helper.hpp => helper.hpp} (100%) rename project/helpers/{gen.pop_ignores.inline.hpp => pop_ignores.inline.hpp} (100%) rename project/helpers/{gen.push_ignores.inline.hpp => push_ignores.inline.hpp} (100%) rename project/helpers/{gen.undef.macros.hpp => undef.macros.hpp} (100%) delete mode 100644 scripts/get_sources.ps1 rename scripts/{build.ci.ps1 => test.gen_build.ci.ps1} (100%) rename scripts/{build.ps1 => test.gen_build.ps1} (100%) rename scripts/{gen.ps1 => test.gen_run.ps1} (100%) diff --git a/Docs/Parsing.md b/Docs/Parsing.md new file mode 100644 index 0000000..e3e23e0 --- /dev/null +++ b/Docs/Parsing.md @@ -0,0 +1,49 @@ +# Parsing + +The library features a naive parser tailored for only what the library needs to construct the supported syntax of C++ into its AST. +This parser does not, and should not do the compiler's job. By only supporting this minimal set of features, the parser is kept under 5000 loc. + + +Everything is done in one pass for both the preprocessor directives and the rest of the language. +The parser performs no macro expansion as the scope of gencpp feature-set is to only support the preprocessor for the goal of having rudimentary awareness of preprocessor ***conditionals***, ***defines***, and ***includes***. +*(Conditionals and defines are a TODO)* + +The keywords supported for the preprocessor are: + +* include +* define +* if +* ifdef +* elif +* endif +* undef + +Just like with actual preprocessor, each directive # line is considered one preproecessor unit, and will be treated as one Preprocessor AST. *These ASTs will be considered members or entries of braced scope they reside within*. +All keywords except *include* are suppported as members of a scope for a class/struct, global, or namespace body. + +Any preprocessor definition abuse that changes the syntax of the core language is unsupported and will fail to parse if not kept within an execution scope (function body, or expression assignment). + +The parsing implementation supports the following for the user: + +```cpp +CodeClass parse_class ( StrC class_def ); +CodeEnum parse_enum ( StrC enum_def ); +CodeBody parse_export_body ( StrC export_def ); +CodeExtern parse_extern_link ( StrC exten_link_def); +CodeFriend parse_friend ( StrC friend_def ); +CodeFn parse_function ( StrC fn_def ); +CodeBody parse_global_body ( StrC body_def ); +CodeNamespace parse_namespace ( StrC namespace_def ); +CodeOperator parse_operator ( StrC operator_def ); +CodeOpCast parse_operator_cast( StrC operator_def ); +CodeStruct parse_struct ( StrC struct_def ); +CodeTemplate parse_template ( StrC template_def ); +CodeType parse_type ( StrC type_def ); +CodeTypedef parse_typedef ( StrC typedef_def ); +CodeUnion parse_union ( StrC union_def ); +CodeUsing parse_using ( StrC using_def ); +CodeVar parse_variable ( StrC var_def ); +``` + +***Parsing will aggregate any tokens within a function body or expression statement to an untyped Code AST.*** + diff --git a/Docs/Readme.md b/Docs/Readme.md new file mode 100644 index 0000000..1787d75 --- /dev/null +++ b/Docs/Readme.md @@ -0,0 +1,537 @@ +## Documentation + +The project has no external dependencies beyond: + +* `errno.h` +* `stat.h` +* `stdarg.h` +* `stddef.h` +* `stdio.h` +* `copyfile.h` (Mac) +* `types.h` (Linux) +* `unistd.h` (Linux/Mac) +* `intrin.h` (Windows) +* `io.h` (Windows with gcc) +* `windows.h` (Windows) + +Dependencies for the project are wrapped within `GENCPP_ROLL_OWN_DEPENDENCIES` (Defining it will disable them). +The majority of the dependency's implementation was derived from the [c-zpl library](https://github.com/zpl-c/zpl). + +This library was written in a subset of C++ where the following are not used at all: + +* RAII (Constructors/Destructors), lifetimes are managed using named static or regular functions. +* Language provide dynamic dispatch, RTTI +* Object-Oriented Inheritance +* Exceptions + +Polymorphic & Member-functions are used as an ergonomic choice, along with a conserative use of operator overloads. +There are only 4 template definitions in the entire library. (`Array`, `Hashtable`, `swap`, and `AST/Code::cast`) + +Two generic templated containers are used throughout the library: + +* `template< class Type> struct Array` +* `template< class Type> struct HashTable` + +Both Code and AST definitions have a `template< class Type> Code/AST cast()`. Its just an alternative way to explicitly cast to each other. + +`template< class Type> swap( Type& a, Type& b)` is used over a macro. + +Otherwise the library is free of any templates. + +### *WHAT IS NOT PROVIDED* + +* Execution statement validation : Execution expressions are defined using the untyped AST. + * Lambdas (This naturally means its unsupported) +* RAII : This needs support for constructors/destructor parsing + * Haven't gotten around to yet (its in the github issues) + +Keywords kept from "Modern C++": + +* constexpr : Great to store compile-time constants. +* consteval : Technically fine, need to make sure to execute in moderation. +* constinit : Better than constexpr at doing its job, however, its only c++ 20. +* export : Useful if c++ modules ever come around to actually being usable. +* import : ^^ +* module : ^^ + +When it comes to expressions: + +**There is no support for validating expressions.** +Its difficult to parse without enough benefits (At the metaprogramming level). + +When it comes to templates: + +**Only trivial template support is provided.** +The intention is for only simple, non-recursive substitution. +The parameters of the template are treated like regular parameter AST entries. +This means that the typename entry for the parameter AST would be either: + +* `class` +* `typename` +* A fundamental type, function, or pointer type. + +Anything beyond this usage is not supported by parse_template for arguments (at least not intentionally). +Use at your own mental peril. + +*Concepts and Constraints are not supported, its usage is non-trivial substitution.* + +### The Data & Interface + +As mentioned in [Usage](#usage), the user is provided Code objects by calling the constructor's functions to generate them or find existing matches. + +The AST is managed by the library and provided the user via its interface. +However, the user may specifiy memory configuration. + +Data layout of AST struct: + +```cpp +union { + struct + { + AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable + AST* Specs; // Function, Operator, Type symbol, Variable + union { + AST* ParentType; // Class, Struct + AST* ReturnType; // Function, Operator + AST* UnderlyingType; // Enum, Typedef + AST* ValueType; // Parameter, Variable + }; + AST* Params; // Function, Operator, Template + union { + AST* ArrExpr; // Type Symbol + AST* Body; // Class, Enum, Function, Namespace, Struct, Union + AST* Declaration; // Friend, Template + AST* Value; // Parameter, Variable + }; + }; + StringCached Content; // Attributes, Comment, Execution, Include + SpecifierT ArrSpecs[AST::ArrSpecs_Cap]; // Specifiers +}; +union { + AST* Prev; + AST* Front; // Used by CodeBody + AST* Last; // Used by CodeParam +}; +union { + AST* Next; + AST* Back; // Used by CodeBody +}; +AST* Parent; +StringCached Name; +CodeT Type; +ModuleFlag ModuleFlags; +union { + OperatorT Op; + AccessSpec ParentAccess; + s32 NumEntries; +}; +``` + +*`CodeT` is a typedef for `ECode::Type` which has an underlying type of `u32`* +*`OperatorT` is a typedef for `EOperator::Type` which has an underlying type of `u32`* +*`StringCahced` is a typedef for `String const`, to denote it is an interned string* +*`String` is the dynamically allocated string type for the library* + +AST widths are setup to be AST_POD_Size. +The width dictates how much the static array can hold before it must give way to using an allocated array: + +```cpp +constexpr static +uw ArrSpecs_Cap = +( + AST_POD_Size + - sizeof(AST*) * 3 + - sizeof(StringCached) + - sizeof(CodeT) + - sizeof(ModuleFlag) + - sizeof(s32) +) +/ sizeof(SpecifierT) -1; // -1 for 4 extra bytes (Odd num of AST*) +``` + +*Ex: If the AST_POD_Size is 128 the capacity of the static array is 20.* + +Data Notes: + +* The allocator definitions used are exposed to the user incase they want to dictate memory usage + * You'll find the memory handling in `init`, `gen_string_allocator`, `get_cached_string`, `make_code`. +* ASTs are wrapped for the user in a Code struct which is a wrapper for a AST* type. +* Both AST and Code have member symbols but their data layout is enforced to be POD types. +* This library treats memory failures as fatal. +* Cached Strings are stored in their own set of arenas. AST constructors use cached strings for names, and content. + * `StringArenas`, `StringCache`, `Allocator_StringArena`, and `Allocator_StringTable` are the associated containers or allocators. +* Strings used for serialization and file buffers are not contained by those used for cached strings. + * They are currently using `GlobalAllocator`, which are tracked array of arenas that grows as needed (adds buckets when one runs out). + * Memory within the buckets is not reused, so its inherently wasteful. + * I will be augmenting the single arena with a simple slag allocator. +* Linked lists used children nodes on bodies, and parameters. +* Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup. +* Allocations can be tuned by defining the folloiwng macros: + * `GEN_BUILDER_STR_BUFFER_RESERVE` + * `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator + * `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator + * `GEN_LEX_ALLOCATOR_SIZE` + * `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line. + * `GEN_MAX_NAME_LENGTH` : Max length of any identifier. + * `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code. + * `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching. + * `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable. + +The following CodeTypes are used which the user may optionally use strong typing with if they enable: `GEN_ENFORCE_STRONG_CODE_TYPES` + +* CodeBody : Has support for `for-range` iterating across Code objects. +* CodeAttributes +* CodeComment +* CodeClass +* CodeConstructor +* CodeDefine +* CodeDestructor +* CodePreprocessCond +* CodeEnum +* CodeExec +* CodeExtern +* CodeInclude +* CodeFriend +* CodeFn +* CodeModule +* CodeNamespace +* CodeOperator +* CodeOpCast +* CodeParam : Has support for `for-range` iterating across parameters. +* CodeSpecifiers : Has support for `for-range` iterating across specifiers. +* CodeStruct +* CodeTemplate +* CodeType +* CodeTypedef +* CodeUnion +* CodeUsing +* CodeVar + +Each Code boy has an associated "filtered AST" with the naming convention: `AST_` +Unrelated fields of the AST for that node type are omitted and only necessary padding members are defined otherwise. +Retrieving a raw version of the ast can be done using the `raw()` function defined in each AST. + +## There are three sets of interfaces for Code AST generation the library provides + +* Upfront +* Parsing +* Untyped + +### Upfront Construction + +All component ASTs must be previously constructed, and provided on creation of the code AST. +The construction will fail and return Code::Invalid otherwise. + +Interface :`` + +* def_attributes + * *This is pre-appended right before the function symbol, or placed after the class or struct keyword for any flavor of attributes used.* + * *Its up to the user to use the desired attribute formatting: `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU).* +* def_comment +* def_class +* def_constructor +* def_destructor +* def_enum +* def_execution + * *This is equivalent to untyped_str, except that its intended for use only in execution scopes.* +* def_extern_link +* def_friend +* def_function +* def_include +* def_module +* def_namespace +* def_operator +* def_operator_cast +* def_param +* def_params +* def_specifier +* def_specifiers +* def_struct +* def_template +* def_type +* def_typedef +* def_union +* def_using +* def_using_namespace +* def_variable + +Bodies: + +* def_body +* def_class_body +* def_enum_body +* def_export_body +* def_extern_link_body +* def_function_body + * *Use this for operator bodies as well* +* def_global_body +* def_namespace_body +* def_struct_body +* def_union_body + +Usage: + +```cpp + = def_( ... ); + +Code +{ + ... + = def_( ... ); +} + +``` + +When using the body functions, its recommended to use the args macro to auto determine the number of arguments for the varadic: +```cpp +def_global_body( args( ht_entry, array_ht_entry, hashtable )); + +// instead of: +def_global_body( 3, ht_entry, array_ht_entry, hashtable ); +``` + +If a more incremental approach is desired for the body ASTs, `Code def_body( CodeT type )` can be used to create an empty body. +When the members have been populated use: `AST::validate_body` to verify that the members are valid entires for that type. + +### Parse construction + +A string provided to the API is parsed for the intended language construct. + +Interface : + +* parse_class +* parse_constructor +* parse_destructor +* parse_enum +* parse_export_body +* parse_extern_link +* parse_friend + * Purposefully are only support forward declares with this constructor. +* parse_function +* parse_global_body +* parse_namespace +* parse_operator +* parse_operator_cast +* parse_struct +* parse_template +* parse_type +* parse_typedef +* parse_union +* parse_using +* parse_variable + +The lexing and parsing takes shortcuts from whats expected in the standard. + +* Numeric literals are not check for validity. +* The parse API treats any execution scope definitions with no validation and are turned into untyped Code ASTs. + * *This includes the assignment of variables.* +* Attributes ( `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU) ) + * Assumed to *come before specifiers* (`const`, `constexpr`, `extern`, `static`, etc) for a function + * Or in the usual spot for class, structs, (*right after the declaration keyword*) + * typedefs have attributes with the type (`parse_type`) +* As a general rule; if its not available from the upfront constructors, its not available in the parsing constructors. + * *Upfront constructors are not necessarily used in the parsing constructors, this is just a good metric to know what can be parsed.* +* Parsing attributes can be extended to support user defined macros by defining `GEN_DEFINE_ATTRIBUTE_TOKENS` (see `gen.hpp` for the formatting) + +Usage: + +```cpp +Code = parse_( string with code ); + +Code = def_( ..., parse_( + +)); + +Code = make_( ... ) +{ + ->add( parse_( + + )); +} +``` + +### Untyped constructions + +Code ASTs are constructed using unvalidated strings. + +Interface : + +* token_fmt_va +* token_fmt +* untyped_str +* untyped_fmt +* untyped_token_fmt + +During serialization any untyped Code AST has its string value directly injected inline of whatever context the content existed as an entry within. +Even though these are not validated from somewhat correct c/c++ syntax or components, it doesn't mean that Untyped code can be added as any component of a Code AST: + +* Untyped code cannot have children, thus there cannot be recursive injection this way. +* Untyped code can only be a child of a parent of body AST, or for values of an assignment (ex: variable assignment). + +These restrictions help prevent abuse of untyped code to some extent. + +Usage Conventions: + +```cpp +Code = def_variable( , , untyped_( + +)); + +Code = untyped_str( code( + +)); +``` + +Optionally, `code_str`, and `code_fmt` macros can be used so that the code macro doesn't have to be used: +```cpp +Code = code_str( ) +``` + +Template metaprogramming in the traditional sense becomes possible with the use of `token_fmt` and parse constructors: + +```cpp +StrC value = txt_StrC("Something"); + +char const* template_str = txt( + Code with to replace with token_values + ... +); +char const* gen_code_str = token_fmt( "key", value, template_str ); +Code = parse_( gen_code_str ); +``` + +## Predefined Codes + +The following are provided predefined by the library as they are commonly used: + +* `access_public` +* `access_protected` +* `access_private` +* `module_global_fragment` +* `module_private_fragment` +* `param_varaidc` (Used for varadic definitions) +* `preprocess_else` +* `preprocess_endif` +* `pragma_once` +* `spec_const` +* `spec_consteval` +* `spec_constexpr` +* `spec_constinit` +* `spec_extern_linkage` (extern) +* `spec_final` +* `spec_global` (global macro) +* `spec_inline` +* `spec_internal_linkage` (internal macro) +* `spec_local_persist` (local_persist macro) +* `spec_mutable` +* `spec_override` +* `spec_ptr` +* `spec_ref` +* `spec_register` +* `spec_rvalue` +* `spec_static_member` (static) +* `spec_thread_local` +* `spec_virtual` +* `spec_volatile` +* `spec_type_signed` +* `spec_type_unsigned` +* `spec_type_short` +* `spec_type_long` +* `t_empty` (Used for varaidc macros) +* `t_auto` +* `t_void` +* `t_int` +* `t_bool` +* `t_char` +* `t_wchar_t` +* `t_class` +* `t_typename` + +Optionally the following may be defined if `GEN_DEFINE_LIBRARY_CODE_CONSTANTS` is defined + +* `t_b32` +* `t_s8` +* `t_s16` +* `t_s32` +* `t_s64` +* `t_u8` +* `t_u16` +* `t_u32` +* `t_u64` +* `t_sw` +* `t_uw` +* `t_f32` +* `t_f64` + +## Extent of operator overload validation + +The AST and constructors will be able to validate that the arguments provided for the operator type match the expected form: + +* If return type must match a parameter +* If number of parameters is correct +* If added as a member symbol to a class or struct, that operator matches the requirements for the class (types match up) + +The user is responsible for making sure the code types provided are correct +and have the desired specifiers assigned to them beforehand. + +## Code generation and modification + +There are three provided file interfaces: + +* Builder +* Editor +* Scanner + +Editor and Scanner are disabled by default, use `GEN_FEATURE_EDITOR` and `GEN_FEATURE_SCANNER` to enable them. + +### Builder is a similar object to the jai language's string_builder + +* The purpose of it is to generate a file. +* A file is specified and opened for writing using the open( file_path) ) function. +* The code is provided via print( code ) function will be serialized to its buffer. +* When all serialization is finished, use the write() command to write the buffer to the file. + +### Editor is for editing a series of files based on a set of requests provided to it + +**Note: Not implemented yet** + +* The purpose is to overrite a specific file, it places its contents in a buffer to scan. +* Requests are populated using the following interface: + * add : Add code. + * remove : Remove code. + * replace: Replace code. + +All three have the same parameters with exception to remove which only has SymbolInfo and Policy: + +* SymbolInfo: + * File : The file the symbol resides in. Leave null to indicate to search all files. Leave null to indicated all-file search. + * Marker : #define symbol that indicates a location or following signature is valid to manipulate. Leave null to indicate the signature should only be used. + * Signature : Use a Code symbol to find a valid location to manipulate, can be further filtered with the marker. Leave null to indicate the marker should only be used. + +* Policy : Additional policy info for completing the request (empty for now) +* Code : Code to inject if adding, or replace existing code with. + +Additionally if `GEN_FEATURE_EDITOR_REFACTOR` is defined, refactor( file_path, specification_path ) wil be made available. +Refactor is based of the refactor library and uses its interface. +It will on call add a request to the queue to run the refactor script on the file. + +### Scanner allows the user to generate Code ASTs by reading files + +**Note: Not implemented yet** + +* The purpose is to grab definitions to generate metadata or generate new code from these definitions. +* Requests are populated using the add( SymbolInfo, Policy ) function. The symbol info is the same as the one used for the editor. So is the case with Policy. + +The file will only be read from, no writing supported. + +One great use case is for example: generating the single-header library for gencpp! + +### Additional Info (Editor and Scanner) + +When all requests have been populated, call process_requests(). +It will provide an output of receipt data of the results when it completes. + +Files may be added to the Editor and Scanner additionally with add_files( num, files ). +This is intended for when you have requests that are for multiple files. + +Request queue in both Editor and Scanner are cleared once process_requests completes. diff --git a/Readme.md b/Readme.md index 9d91958..f1aab97 100644 --- a/Readme.md +++ b/Readme.md @@ -12,43 +12,12 @@ its not meant to be a black box metaprogramming utility, its meant for the user * [Notes](#notes) * [Usage](#usage) -* [Building](#notes) -* [Outline](#outline) -* [What is not provided](#what-is-not-provided) -* [The three constructors ](#there-are-three-sets-of-interfaces-for-code-ast-generation-the-library-provides) -* [Predefined Codes](#predefined-codes) -* [Code generation and modification](#code-generation-and-modification) +* [Building](#building) ## Notes -The project has reached an *alpha* state, all the current functionality works for the test cases but it will most likely break in many other cases. - -The project has no external dependencies beyond: - -* `errno.h` -* `stat.h` -* `stdarg.h` -* `stddef.h` -* `stdio.h` -* `copyfile.h` (Mac) -* `types.h` (Linux) -* `unistd.h` (Linux/Mac) -* `intrin.h` (Windows) -* `io.h` (Windows with gcc) -* `windows.h` (Windows) - -Dependencies for the project are wrapped within `GENCPP_ROLL_OWN_DEPENDENCIES` (Defining it will disable them). -The majority of the dependency's implementation was derived from the [c-zpl library](https://github.com/zpl-c/zpl). - -This library was written a subset of C++ where the following are not used at all: - -* RAII (Constructors/Destructors), lifetimes are managed using named static or regular functions. -* Language provide dynamic dispatch, RTTI -* Object-Oriented Inheritance -* Exceptions - -Polymorphic & Member-functions are used as an ergonomic choice, along with a conserative use of operator overloads. -There are only 4 template definitions in the entire library. (`Array`, `Hashtable`, `swap`, and `AST/Code::cast`) +The project has reached an *alpha* state, all the current functionality works for the test cases but it will most likely break in many other cases. +The [issues](https://github.com/Ed94/gencpp/issues) marked with v1.0 Feature indicate whats left before the library is considered feature complete. A `natvis` and `natstepfilter` are provided in the scripts directory. @@ -87,7 +56,7 @@ u32 gen_main() ``` The design uses a constructive builder API for the code to generate. -The user is given `Code` objects that are used to build up the AST. +The user is provided `Code` objects that are used to build up the AST. Example using each construction interface: @@ -154,529 +123,8 @@ struct ArrayHeader ``` **Note: The formatting shown here is not how it will look. For your desired formatting its recommended to run a pass through the files with an auto-formatter.** +*(The library currently uses clang-format for formatting, beaware its pretty slow...)* ## Building -An example of building is provided within project, singleheader, and test. - -(All generated files go to a corresponding `*/gen` directory) - -**Project** - -`gen.bootstrap.cpp` generates a segmented version of the library where code is divided into `gen.hpp`, `gen.cpp`, `gen_dep.hpp`, and `gen_dep.cpp`. -(Similar whats there already, but with the includes injected) - -**Singleheader** - -`gen.singleheader.cpp` generates a single-header version of the library following the convention shown in popular libraries such as: gb, stb, and zpl. - -**Test** - -There are two meson build files the one within test is the program's build specification. -The other one in the gen directory within test is the metaprogram's build specification. - -Both of them build the same source file: `test.cpp`. The only differences are that gen needs a different relative path to the include directories and defines the macro definition: `GEN_TIME`. - -This method is setup where all the metaprogram's code are the within the same files as the regular program's code. - -## Outline - -### *WHAT IS NOT PROVIDED* - -* Exceptions -* Execution statement validation : Execution expressions are defined using the untyped API. - * Lambdas (This naturally means its unsupported) -* RAII : This needs support for constructors/destructor parsing - * Haven't gotten around to yet, only useful (to me) for third-party scanning - -Keywords kept from "Modern C++": - -* constexpr : Great to store compile-time constants. -* consteval : Technically fine, need to make sure to execute in moderation. -* constinit : Better than constexpr at doing its job, however, its only c++ 20. -* export : Useful if c++ modules ever come around to actually being usable. -* import : ^^ -* module : ^^ - -When it comes to expressions: - -**There is no support for validating expressions.** -Its difficult to parse without enough benefits (At the metaprogramming level). - -When it comes to templates: - -Only trivial template support is provided. the intention is for only simple, non-recursive substitution. -The parameters of the template are treated like regular parameter AST entries. -This means that the typename entry for the parameter AST would be either: - -* `class` -* `typename` -* A fundamental type, function, or pointer type. - -Anything beyond this usage is not supported by parse_template for arguments (at least not intentionally). -Use at your own mental peril. - -*Concepts and Constraints are not supported, its usage is non-trivial substitution.* - -### The Data & Interface - -As mentioned in [Usage](#usage), the user is provided Code objects by calling the constructor's functions to generate them or find existing matches. - -The AST is managed by the library and provided the user via its interface. -However, the user may specifiy memory configuration. - -Data layout of AST struct: - -```cpp -union { - struct - { - AST* Attributes; // Class, Enum, Function, Struct, Typedef, Union, Using, Variable - AST* Specs; // Function, Operator, Type symbol, Variable - union { - AST* ParentType; // Class, Struct - AST* ReturnType; // Function, Operator - AST* UnderlyingType; // Enum, Typedef - AST* ValueType; // Parameter, Variable - }; - AST* Params; // Function, Operator, Template - union { - AST* ArrExpr; // Type Symbol - AST* Body; // Class, Enum, Function, Namespace, Struct, Union - AST* Declaration; // Friend, Template - AST* Value; // Parameter, Variable - }; - }; - StringCached Content; // Attributes, Comment, Execution, Include - SpecifierT ArrSpecs[AST::ArrSpecs_Cap]; // Specifiers -}; -union { - AST* Prev; - AST* Front; // Used by CodeBody - AST* Last; // Used by CodeParam -}; -union { - AST* Next; - AST* Back; // Used by CodeBody -}; -AST* Parent; -StringCached Name; -CodeT Type; -ModuleFlag ModuleFlags; -union { - OperatorT Op; - AccessSpec ParentAccess; - s32 NumEntries; -}; -``` - -*`CodeT` is a typedef for `ECode::Type` which has an underlying type of `u32`* -*`OperatorT` is a typedef for `EOperator::Type` which has an underlying type of `u32`* -*`StringCahced` is a typedef for `String const`, to denote it is an interned string* -*`String` is the dynamically allocated string type for the library* - -AST widths are setup to be AST_POD_Size. -The width dictates how much the static array can hold before it must give way to using an allocated array: - -```cpp -constexpr static -uw ArrSpecs_Cap = -( - AST_POD_Size - - sizeof(AST*) * 3 - - sizeof(StringCached) - - sizeof(CodeT) - - sizeof(ModuleFlag) - - sizeof(s32) -) -/ sizeof(SpecifierT) -1; // -1 for 4 extra bytes (Odd num of AST*) -``` - -*Ex: If the AST_POD_Size is 128 the capacity of the static array is 20.* - -Data Notes: - -* The allocator definitions used are exposed to the user incase they want to dictate memory usage - * You'll find the memory handling in `init`, `gen_string_allocator`, `get_cached_string`, `make_code`. -* ASTs are wrapped for the user in a Code struct which is a wrapper for a AST* type. -* Both AST and Code have member symbols but their data layout is enforced to be POD types. -* This library treats memory failures as fatal. -* Cached Strings are stored in their own set of arenas. AST constructors use cached strings for names, and content. - * `StringArenas`, `StringCache`, `Allocator_StringArena`, and `Allocator_StringTable` are the associated containers or allocators. -* Strings used for serialization and file buffers are not contained by those used for cached strings. - * They are currently using `GlobalAllocator`, which are tracked array of arenas that grows as needed (adds buckets when one runs out). - * Memory within the buckets is not reused, so its inherently wasteful. - * I will be augmenting the single arena with a simple slag allocator. -* Linked lists used children nodes on bodies, and parameters. -* Its intended to generate the AST in one go and serialize after. The constructors and serializer are designed to be a "one pass, front to back" setup. -* Allocations can be tuned by defining the folloiwng macros: - * `GEN_GLOBAL_BUCKET_SIZE` : Size of each bucket area for the global allocator - * `GEN_CODEPOOL_NUM_BLOCKS` : Number of blocks per code pool in the code allocator - * `GEN_SIZE_PER_STRING_ARENA` : Size per arena used with string caching. - * `GEN_MAX_COMMENT_LINE_LENGTH` : Longest length a comment can have per line. - * `GEN_MAX_NAME_LENGTH` : Max length of any identifier. - * `GEN_MAX_UNTYPED_STR_LENGTH` : Max content length for any untyped code. - * `GEN_TOKEN_FMT_TOKEN_MAP_MEM_SIZE` : token_fmt_va uses local_persit memory of this size for the hashtable. - * `GEN_LEX_ALLOCATOR_SIZE` - * `GEN_BUILDER_STR_BUFFER_RESERVE` - -Two generic templated containers are used throughout the library: - -* `template< class Type> struct Array` -* `template< class Type> struct HashTable` - -Both Code and AST definitions have a `template< class Type> Code/AST cast()`. Its just an alternative way to explicitly cast to each other. - -`template< class Type> swap( Type& a, Type& b)` is used over a macro. - -Otherwise the library is free of any templates. - -The following CodeTypes are used which the user may optionally use strong typing with if they enable: `GEN_ENFORCE_STRONG_CODE_TYPES` - -* CodeBody : Has support for `for-range` iterating across Code objects. -* CodeAttributes -* CodeComment -* CodeClass -* CodeEnum -* CodeExec -* CodeExtern -* CodeInclude -* CodeFriend -* CodeFn -* CodeModule -* CodeNamespace -* CodeOperator -* CodeOpCast -* CodeParam : Has support for `for-range` iterating across parameters. -* CodeSpecifiers : Has support for `for-range` iterating across specifiers. -* CodeStruct -* CodeTemplate -* CodeType -* CodeTypedef -* CodeUnion -* CodeUsing -* CodeUsingNamespace -* CodeVar - -Each Code boy has an associated "filtered AST" with the naming convention: `AST_` -Unrelated fields of the AST for that node type are omitted and only necessary padding members are defined otherwise. -Retrieving a raw version of the ast can be done using the `raw()` function defined in each AST. - -## There are three sets of interfaces for Code AST generation the library provides - -* Upfront -* Parsing -* Untyped - -### Upfront Construction - -All component ASTs must be previously constructed, and provided on creation of the code AST. -The construction will fail and return Code::Invalid otherwise. - -Interface :`` - -* def_attributes - * *This is pre-appended right before the function symbol, or placed after the class or struct keyword for any flavor of attributes used.* - * *Its up to the user to use the desired attribute formatting: `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU).* -* def_comment -* def_class -* def_enum -* def_execution - * *This is equivalent to untyped_str, except that its intended for use only in execution scopes.* -* def_extern_link -* def_friend -* def_function -* def_include -* def_module -* def_namespace -* def_operator -* def_operator_cast -* def_param -* def_params -* def_specifier -* def_specifiers -* def_struct -* def_template -* def_type -* def_typedef -* def_union -* def_using -* def_using_namespace -* def_variable - -Bodies: - -* def_body -* def_class_body -* def_enum_body -* def_export_body -* def_extern_link_body -* def_function_body - * *Use this for operator bodies as well* -* def_global_body -* def_namespace_body -* def_struct_body -* def_union_body - -Usage: - -```cpp - = def_( ... ); - -Code -{ - ... - = def_( ... ); -} - -``` - -When using the body functions, its recommended to use the args macro to auto determine the number of arguments for the varadic: -```cpp -def_global_body( args( ht_entry, array_ht_entry, hashtable )); - -// instead of: -def_global_body( 3, ht_entry, array_ht_entry, hashtable ); -``` - -If a more incremental approach is desired for the body ASTs, `Code def_body( CodeT type )` can be used to create an empty body. -When the members have been populated use: `AST::validate_body` to verify that the members are valid entires for that type. - -### Parse construction - -A string provided to the API is parsed for the intended language construct. - -Interface : - -* parse_class -* parse_enum -* parse_export_body -* parse_extern_link -* parse_friend - * Purposefully are only support forward declares with this constructor. -* parse_function -* parse_global_body -* parse_namespace -* parse_operator -* parse_operator_cast -* parse_struct -* parse_template -* parse_type -* parse_typedef -* parse_union -* parse_using -* parse_variable - -The lexing and parsing takes shortcuts from whats expected in the standard. - -* Numeric literals are not check for validity. -* The parse API treats any execution scope definitions with no validation and are turned into untyped Code ASTs. - * *This includes the assignment of variables.* -* Attributes ( `[[]]` (standard), `__declspec` (Microsoft), or `__attribute__` (GNU) ) - * Assumed to *come before specifiers* (`const`, `constexpr`, `extern`, `static`, etc) for a function - * Or in the usual spot for class, structs, (*right after the declaration keyword*) - * typedefs have attributes with the type (`parse_type`) -* As a general rule; if its not available from the upfront constructors, its not available in the parsing constructors. - * *Upfront constructors are not necessarily used in the parsing constructors, this is just a good metric to know what can be parsed.* -* Parsing attributes can be extended to support user defined macros by defining `GEN_Define_Attribute_Tokens` (see `gen.hpp` for the formatting) - -Usage: - -```cpp -Code = parse_( string with code ); - -Code = def_( ..., parse_( - -)); - -Code = make_( ... ) -{ - ->add( parse_( - - )); -} -``` - -### Untyped constructions - -Code ASTs are constructed using unvalidated strings. - -Interface : - -* token_fmt_va -* token_fmt -* untyped_str -* untyped_fmt -* untyped_token_fmt - -During serialization any untyped Code AST has its string value directly injected inline of whatever context the content existed as an entry within. -Even though these are not validated from somewhat correct c/c++ syntax or components, it doesn't mean that Untyped code can be added as any component of a Code AST: - -* Untyped code cannot have children, thus there cannot be recursive injection this way. -* Untyped code can only be a child of a parent of body AST, or for values of an assignment (ex: variable assignment). - -These restrictions help prevent abuse of untyped code to some extent. - -Usage Conventions: - -```cpp -Code = def_variable( , , untyped_( - -)); - -Code = untyped_str( code( - -)); -``` - -Optionally, `code_str`, and `code_fmt` macros can be used so that the code macro doesn't have to be used: -```cpp -Code = code_str( ) -``` - -Template metaprogramming in the traditional sense becomes possible with the use of `token_fmt` and parse constructors: - -```cpp -StrC value = txt_StrC("Something"); - -char const* template_str = txt( - Code with to replace with token_values - ... -); -char const* gen_code_str = token_fmt( "key", value, template_str ); -Code = parse_( gen_code_str ); -``` - -## Predefined Codes - -The following are provided predefined by the library as they are commonly used: - -* `access_public` -* `access_protected` -* `access_private` -* `module_global_fragment` -* `module_private_fragment` -* `param_varaidc` (Used for varadic definitions) -* `pragma_once` -* `spec_const` -* `spec_consteval` -* `spec_constexpr` -* `spec_constinit` -* `spec_extern_linkage` (extern) -* `spec_final` -* `spec_global` (global macro) -* `spec_inline` -* `spec_internal_linkage` (internal macro) -* `spec_local_persist` (local_persist macro) -* `spec_mutable` -* `spec_override` -* `spec_ptr` -* `spec_ref` -* `spec_register` -* `spec_rvalue` -* `spec_static_member` (static) -* `spec_thread_local` -* `spec_virtual` -* `spec_volatile` -* `spec_type_signed` -* `spec_type_unsigned` -* `spec_type_short` -* `spec_type_long` -* `t_empty` (Used for varaidc macros) -* `t_auto` -* `t_void` -* `t_int` -* `t_bool` -* `t_char` -* `t_wchar_t` -* `t_class` -* `t_typename` - -Optionally the following may be defined if `GEN_DEFINE_LIBRARY_CODE_CONSTANTS` is defined - -* `t_b32` -* `t_s8` -* `t_s16` -* `t_s32` -* `t_s64` -* `t_u8` -* `t_u16` -* `t_u32` -* `t_u64` -* `t_sw` -* `t_uw` -* `t_f32` -* `t_f64` - -## Extent of operator overload validation - -The AST and constructors will be able to validate that the arguments provided for the operator type match the expected form: - -* If return type must match a parameter -* If number of parameters is correct -* If added as a member symbol to a class or struct, that operator matches the requirements for the class (types match up) - -The user is responsible for making sure the code types provided are correct -and have the desired specifiers assigned to them beforehand. - -## Code generation and modification - -There are three provided file interfaces: - -* Builder -* Editor -* Scanner - -Editor and Scanner are disabled by default, use `GEN_FEATURE_EDITOR` and `GEN_FEATURE_SCANNER` to enable them. - -### Builder is a similar object to the jai language's string_builder - -* The purpose of it is to generate a file. -* A file is specified and opened for writing using the open( file_path) ) function. -* The code is provided via print( code ) function will be serialized to its buffer. -* When all serialization is finished, use the write() command to write the buffer to the file. - -### Editor is for editing a series of files based on a set of requests provided to it - -**Note: Not implemented yet** - -* The purpose is to overrite a specific file, it places its contents in a buffer to scan. -* Requests are populated using the following interface: - * add : Add code. - * remove : Remove code. - * replace: Replace code. - -All three have the same parameters with exception to remove which only has SymbolInfo and Policy: - -* SymbolInfo: - * File : The file the symbol resides in. Leave null to indicate to search all files. Leave null to indicated all-file search. - * Marker : #define symbol that indicates a location or following signature is valid to manipulate. Leave null to indicate the signature should only be used. - * Signature : Use a Code symbol to find a valid location to manipulate, can be further filtered with the marker. Leave null to indicate the marker should only be used. - -* Policy : Additional policy info for completing the request (empty for now) -* Code : Code to inject if adding, or replace existing code with. - -Additionally if `GEN_FEATURE_EDITOR_REFACTOR` is defined, refactor( file_path, specification_path ) wil be made available. -Refactor is based of the refactor library and uses its interface. -It will on call add a request to the queue to run the refactor script on the file. - -### Scanner allows the user to generate Code ASTs by reading files - -**Note: Not implemented yet** - -* The purpose is to grab definitions to generate metadata or generate new code from these definitions. -* Requests are populated using the add( SymbolInfo, Policy ) function. The symbol info is the same as the one used for the editor. So is the case with Policy. - -The file will only be read from, no writing supported. - -One great use case is for example: generating the single-header library for gencpp! - -### Additional Info (Editor and Scanner) - -When all requests have been populated, call process_requests(). -It will provide an output of receipt data of the results when it completes. - -Files may be added to the Editor and Scanner additionally with add_files( num, files ). -This is intended for when you have requests that are for multiple files. - -Request queue in both Editor and Scanner are cleared once process_requests completes. +See the [scripts directory](scripts/). diff --git a/gencpp.vcxproj b/gencpp.vcxproj index 8057c56..0887f2c 100644 --- a/gencpp.vcxproj +++ b/gencpp.vcxproj @@ -112,12 +112,18 @@ + + + + + - + + @@ -125,25 +131,17 @@ + - - - - - + - - - - - @@ -160,18 +158,28 @@ + + + + + - + + + + + + + + - - - + @@ -184,6 +192,23 @@ + + + + + + + + + + + + + + + + + diff --git a/project/Readme.md b/project/Readme.md index b8b1ae0..5698371 100644 --- a/project/Readme.md +++ b/project/Readme.md @@ -1,47 +1,43 @@ # Documentation -The core library is contained within `gen.hpp` and `gen.cpp`. -Things related to the editor and scanner are in their own respective files. (Ex: `gen.scanner.` ) - -Dependencies are within `gen.dep.` - The library is fragmented into a series of headers and sources files meant to be scanned in and then generated to a tailored format for the target `gen` files. +The principal (user) files are `gen.hpp` and `gen.cpp`. +They contain includes for its various components: `components/.` + +Dependencies are bundled into `gen.dep.`. +Just like the `gen.` they include their components: `dependencies/.` + +The fle processors are in their own respective files. (Ex: `file_processors/.` ) +They directly include `depedencies/file_handling.` as the core library does not include file processing by defualt. + +**TODO : Right now the library is not finished structurally, as such the first self-hosting iteration is still WIP** Both libraries use *pre-generated* (self-hosting I guess) version of the library to then generate the latest version of itself. -(sort of a verification that the generated version is equivalent) +(sort of a verification that the generated version is equivalent). The default `gen.bootstrap.cpp` located in the project folder is meant to be produce a standard segmented library, where the components of the library -have relatively dedicated header and source files. With dependencies included at the top of the file and each header starting with a pragma once. -This will overwrite the existing library implementation in the immediate directory. +have relatively dedicated header and source files. Dependencies included at the top of the file and each header starting with a pragma once. +The output will be in the `project/gen` directory (if the directory does not exist, it will create it). Use those to get a general idea of how to make your own tailored version. -If the naming convention is undesired, the `gencpp.refactor` script can be used with the [refactor]() - Feature Macros: +* `GEN_DEFINE_ATTRIBUTE_TOKENS` : Allows user to define their own attribute macros for use in parsing. + * This is auto-generated if using the bootstrap or single-header generation + * *Note: The user will use the `AttributeTokens.csv` when the library is fully self-hosting.* +* `GEN_DEFINE_LIBRARY_CORE_CONSTANTS` : Optional typename codes as they are non-standard to C/C++ and not necessary to library usage * `GEN_DONT_USE_NAMESPACE` : By default, the library is wrapped in a `gen` namespace, this will disable that expose it to the global scope. * `GEN_DONT_ENFORCE_GEN_TIME_GUARD` : By default, the library ( gen.hpp/ gen.cpp ) expects the macro `GEN_TIME` to be defined, this disables that. -* `GEN_ROLL_OWN_DEPENDENCIES` : Optional override so that user may define the dependencies themselves. -* `GEN_DEFINE_LIBRARY_CORE_CONSTANTS` : Optional typename codes as they are non-standard to C/C++ and not necessary to library usage * `GEN_ENFORCE_STRONG_CODE_TYPES` : Enforces casts to filtered code types. * `GEN_EXPOSE_BACKEND` : Will expose symbols meant for internal use only. -* `GEN_Define_Attribute_Tokens` : Allows user to define their own attribute macros for use in parsing. - -`GEN_USE_RECURSIVE_AST_DUPLICATION` is available but its not well tested and should not need to be used. -If constructing ASTs properly. There should be no modification of ASTs, and thus this would never become an issue. -(I will probably remove down the line...) +* `GEN_ROLL_OWN_DEPENDENCIES` : Optional override so that user may define the dependencies themselves. ## On multi-threading Currently unsupported. The following changes would have to be made: -* Setup static data access with fences if more than one thread will generate ASTs ( or keep a different set for each thread) -* Make sure local persistent data of functions are also thread local. -* The builder should be done on a per-thread basis. -* Due to the design of the editor and scanner, it will most likely be best to make each file a job to process request entries on. Receipts should have an an array to store per thread. They can be combined to the final receipts array when all files have been processed. - ## Extending the library This library is relatively very small, and can be extended without much hassle. diff --git a/project/components/gen.ast.cpp b/project/components/ast.cpp similarity index 87% rename from project/components/gen.ast.cpp rename to project/components/ast.cpp index e9054aa..846af4d 100644 --- a/project/components/gen.ast.cpp +++ b/project/components/ast.cpp @@ -17,20 +17,9 @@ AST* AST::duplicate() String AST::to_string() { -# define ProcessModuleFlags() \ - if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) \ - result.append( "export " ); \ - \ - if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Import )) \ - result.append( "import " ); \ - local_persist thread_local char SerializationLevel = 0; -#if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION) - u64 time_start = time_rel_ms(); -#endif - // TODO : Need to refactor so that intermeidate strings are freed conviently. String result = String::make( GlobalAllocator, "" ); @@ -82,7 +71,8 @@ String AST::to_string() case Class: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes || ParentType ) { @@ -129,7 +119,8 @@ String AST::to_string() case Class_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "class %s %s;", Attributes->to_string(), Name ); @@ -140,7 +131,8 @@ String AST::to_string() case Enum: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes || UnderlyingType ) { @@ -170,7 +162,8 @@ String AST::to_string() case Enum_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -181,7 +174,8 @@ String AST::to_string() case Enum_Class: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes || UnderlyingType ) { @@ -219,7 +213,8 @@ String AST::to_string() case Enum_Class_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append( "enum class " ); @@ -262,7 +257,8 @@ String AST::to_string() case Function: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -301,7 +297,8 @@ String AST::to_string() case Function_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -347,7 +344,8 @@ String AST::to_string() break; case Namespace: - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append_fmt( "namespace %s\n{\n%s}" , Name @@ -358,7 +356,8 @@ String AST::to_string() case Operator: case Operator_Member: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -395,7 +394,8 @@ String AST::to_string() case Operator_Fwd: case Operator_Member_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -513,7 +513,8 @@ String AST::to_string() case Struct: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Name == nullptr) { @@ -566,7 +567,8 @@ String AST::to_string() case Struct_Fwd: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "struct %s %s;", Attributes->to_string(), Name ); @@ -577,7 +579,8 @@ String AST::to_string() case Template: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append_fmt( "template< %s >\n%s", Params->to_string(), Declaration->to_string() ); } @@ -585,7 +588,8 @@ String AST::to_string() case Typedef: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append( "typedef "); @@ -624,7 +628,8 @@ String AST::to_string() case Union: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); result.append( "union " ); @@ -650,7 +655,8 @@ String AST::to_string() case Using: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes ) result.append_fmt( "%s ", Attributes->to_string() ); @@ -675,7 +681,8 @@ String AST::to_string() case Variable: { - ProcessModuleFlags(); + if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export )) + result.append( "export " ); if ( Attributes || Specs ) { @@ -726,11 +733,7 @@ String AST::to_string() break; } -#if defined(GEN_BENCHMARK) && defined(GEN_BENCHMARK_SERIALIZATION) - log_fmt("AST::to_string() time taken: %llu for: %s\n", time_rel_ms() - time_start, result ); -#endif return result; -#undef ProcessModuleFlags } bool AST::is_equal( AST* other ) @@ -760,19 +763,19 @@ bool AST::validate_body() { using namespace ECode; -#define CheckEntries( Unallowed_Types ) \ - do \ - { \ - for ( Code entry : cast() ) \ - { \ - switch ( entry->Type ) \ - { \ - Unallowed_Types \ - log_failure( "AST::validate_body: Invalid entry in body %s", entry.debug_str() ); \ - return false; \ - } \ - } \ - } \ +#define CheckEntries( Unallowed_Types ) \ + do \ + { \ + for ( Code entry : cast() ) \ + { \ + switch ( entry->Type ) \ + { \ + Unallowed_Types \ + log_failure( "AST::validate_body: Invalid entry in body %s", entry.debug_str() ); \ + return false; \ + } \ + } \ + } \ while (0); switch ( Type ) @@ -825,6 +828,8 @@ bool AST::validate_body() } return false; + +#undef CheckEntries } #pragma endregion AST diff --git a/project/components/gen.ast_case_macros.cpp b/project/components/ast_case_macros.cpp similarity index 100% rename from project/components/gen.ast_case_macros.cpp rename to project/components/ast_case_macros.cpp diff --git a/project/components/gen.data_structures.hpp b/project/components/data_structures.hpp similarity index 100% rename from project/components/gen.data_structures.hpp rename to project/components/data_structures.hpp diff --git a/project/components/gen.ecode.hpp b/project/components/ecode.hpp similarity index 100% rename from project/components/gen.ecode.hpp rename to project/components/ecode.hpp diff --git a/project/components/gen.eoperator.hpp b/project/components/eoperator.hpp similarity index 98% rename from project/components/gen.eoperator.hpp rename to project/components/eoperator.hpp index 2d9ca26..70c99dd 100644 --- a/project/components/gen.eoperator.hpp +++ b/project/components/eoperator.hpp @@ -72,4 +72,4 @@ namespace EOperator # undef Define_Operators } -using OperatorT = EOperator::Type; \ No newline at end of file +using OperatorT = EOperator::Type; diff --git a/project/components/gen.especifier.hpp b/project/components/especifier.hpp similarity index 99% rename from project/components/gen.especifier.hpp rename to project/components/especifier.hpp index fea0f59..9affa84 100644 --- a/project/components/gen.especifier.hpp +++ b/project/components/especifier.hpp @@ -101,5 +101,4 @@ namespace ESpecifier # undef Define_Specifiers } - using SpecifierT = ESpecifier::Type; diff --git a/project/components/gen.etoktype.cpp b/project/components/etoktype.cpp similarity index 96% rename from project/components/gen.etoktype.cpp rename to project/components/etoktype.cpp index c20a285..5516221 100644 --- a/project/components/gen.etoktype.cpp +++ b/project/components/etoktype.cpp @@ -9,8 +9,8 @@ namespace Parser Attributes_Start is only used to indicate the start of the user_defined attribute list. */ -#ifndef GEN_Define_Attribute_Tokens -# define GEN_Define_Attribute_Tokens \ +#ifndef GEN_DEFINE_ATTRIBUTE_TOKENS +# define GEN_DEFINE_ATTRIBUTE_TOKENS \ Entry( API_Export, "GEN_API_Export_Code" ) \ Entry( API_Import, "GEN_API_Import_Code" ) #endif @@ -97,7 +97,7 @@ namespace Parser { # define Entry( Name_, Str_ ) Name_, Define_TokType - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS # undef Entry NumTokens, }; @@ -110,7 +110,7 @@ namespace Parser { # define Entry( Name_, Str_ ) { sizeof(Str_), Str_ }, Define_TokType - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS # undef Entry }; @@ -137,7 +137,7 @@ namespace Parser { # define Entry( Name_, Str_ ) Str_, Define_TokType - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS # undef Entry }; diff --git a/project/components/gen.header_end.hpp b/project/components/header_end.hpp similarity index 100% rename from project/components/gen.header_end.hpp rename to project/components/header_end.hpp diff --git a/project/components/gen.header_start.hpp b/project/components/header_start.hpp similarity index 100% rename from project/components/gen.header_start.hpp rename to project/components/header_start.hpp diff --git a/project/components/gen.impl_start.cpp b/project/components/impl_start.cpp similarity index 100% rename from project/components/gen.impl_start.cpp rename to project/components/impl_start.cpp diff --git a/project/components/gen.interface.cpp b/project/components/interface.cpp similarity index 98% rename from project/components/gen.interface.cpp rename to project/components/interface.cpp index f4be7a8..d1e7d56 100644 --- a/project/components/gen.interface.cpp +++ b/project/components/interface.cpp @@ -105,7 +105,7 @@ void define_constants() #endif # undef def_constant_code_type - t_empty = (CodeType) make_code(); + t_empty = (CodeType) make_code(); t_empty->Type = ECode::Typename; t_empty->Name = get_cached_string( txt_StrC("") ); t_empty.set_global(); @@ -127,12 +127,12 @@ void define_constants() access_private->Name = get_cached_string( txt_StrC("private:") ); access_private.set_global(); - access_protected = make_code(); + access_protected = make_code(); access_protected->Type = ECode::Access_Protected; access_protected->Name = get_cached_string( txt_StrC("protected:") ); access_protected.set_global(); - access_public = make_code(); + access_public = make_code(); access_public->Type = ECode::Access_Public; access_public->Name = get_cached_string( txt_StrC("public:") ); access_public.set_global(); diff --git a/project/components/gen.interface.hpp b/project/components/interface.hpp similarity index 100% rename from project/components/gen.interface.hpp rename to project/components/interface.hpp diff --git a/project/components/gen.interface.parsing.cpp b/project/components/interface.parsing.cpp similarity index 87% rename from project/components/gen.interface.parsing.cpp rename to project/components/interface.parsing.cpp index f7f4445..85dea41 100644 --- a/project/components/gen.interface.parsing.cpp +++ b/project/components/interface.parsing.cpp @@ -6,19 +6,13 @@ namespace Parser { struct Token { - // TokType Type; - // s32 Start; - // s32 End; - // s32 Line; - // s32 Column; - // TokFlags Flags; - char const* Text; sptr Length; TokType Type; bool IsAssign; s32 Line; s32 Column; + // TokFlags Flags; operator bool() { @@ -59,6 +53,8 @@ namespace Parser } }; + constexpr Token NullToken { nullptr, 0, TokType::Invalid, false, 0, 0 }; + struct TokArray { Array Arr; @@ -101,20 +97,48 @@ namespace Parser result.append_fmt("\tContext:\n"); + char* current = Tokens.current().Text; + sptr length = Tokens.current().Length; + while ( current != Tokens.back().Text && current != '\n' ) + { + current++; + length--; + } + + String line = String::make( GlobalAllocator, { length, Tokens.current().Text } ); + result.append_fmt("\t(%d, %d): %s", Tokens.current().Line, Tokens.current().Column, line ); + line.free(); + StackNode* current = Scope; do { - String name = String::make( GlobalAllocator, current->Name ? (StrC)current->Name : txt_StrC("Unresolved") ); + if ( current->Name ) + { + result.append_fmt("\tProcedure: %s, AST Name: %s\n", current->ProcName, (StrC)current->Name ); + } + else + { + result.append_fmt("\tProcedure: %s\n", current->ProcName ); + } - result.append_fmt("\tProcedure: %s, AST Name: %s\n\t(%d, %d):", current->ProcName, name ); current = current->Prev; name.free(); } while ( current ); - return result; } + + void push( StackNode* node ) + { + node->Prev = Scope; + Scope = node; + } + + void pop() + { + Context.Scope = Context.Scope->Prev; + } }; global ParseContext Context; @@ -123,7 +147,7 @@ namespace Parser { if ( Arr.num() - Idx <= 0 ) { - log_failure( "No tokens left\n", Context.Scope->ProcName ); + log_failure( "No tokens left.\n%s", Context.to_string() ); return false; } @@ -131,7 +155,7 @@ namespace Parser { String token_str = String::make( GlobalAllocator, { Arr[Idx].Length, Arr[Idx].Text } ); - log_failure( "gen::%s: expected %s, got %s", Context.Scope->ProcName, ETokType::to_str(type), ETokType::to_str(Arr[Idx].Type) ); + log_failure( "Parse Error, TokArray::eat, Expected: %s, not %s (%d, %d)`\n%s", ETokType::to_str(type), token_str, Context.to_string() ); return false; } @@ -217,6 +241,7 @@ namespace Parser switch ( current ) { + // TODO : Need to handle the preprocessor as a separate pass. case '#': token.Text = scanner; token.Length = 1; @@ -681,16 +706,16 @@ namespace Parser #pragma region Helper Macros -# define check_parse_args( func, def ) \ -if ( def.Len <= 0 ) \ -{ \ - log_failure( "gen::" stringize(func) ": length must greater than 0" ); \ - return CodeInvalid; \ -} \ -if ( def.Ptr == nullptr ) \ -{ \ - log_failure( "gen::" stringize(func) ": def was null" ); \ - return CodeInvalid; \ +# define check_parse_args( def ) \ +if ( def.Len <= 0 ) \ +{ \ + log_failure( "gen::" stringize( __func__ ) ": length must greater than 0" ); \ + return CodeInvalid; \ +} \ +if ( def.Ptr == nullptr ) \ +{ \ + log_failure( "gen::" stringize( __func__ ) ": def was null" ); \ + return CodeInvalid; \ } # define nexttok Context.Tokens.next() @@ -701,16 +726,12 @@ if ( def.Ptr == nullptr ) \ # define check( Type_ ) ( left && currtok.Type == Type_ ) -// # define +# define push_scope() \ + StackNode scope { nullptr, NullToken, txt_StrC( __func__ ) }; \ + Context.push( & scope ) #pragma endregion Helper Macros -struct ParseContext -{ - ParseContext* Parent; - char const* Fn; -}; - internal Code parse_function_body(); internal Code parse_global_nspace(); @@ -735,6 +756,7 @@ internal inline Code parse_array_decl() { using namespace Parser; + push_scope(); if ( check( TokType::BraceSquare_Open ) ) { @@ -742,13 +764,13 @@ Code parse_array_decl() if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of typedef definition ( '[]' scope started )", stringize(parse_typedef) ); + log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", Context.to_string() ); return Code::Invalid; } if ( currtok.Type == TokType::BraceSquare_Close ) { - log_failure( "%s: Error, empty array expression in typedef definition", stringize(parse_typedef) ); + log_failure( "Error, empty array expression in typedef definition\n%s", Context.to_string() ); return Code::Invalid; } @@ -765,13 +787,13 @@ Code parse_array_decl() if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of type definition, expected ]", stringize(parse_typedef) ); + log_failure( "Error, unexpected end of array declaration, expected ]\n%s", Context.to_string() ); return Code::Invalid; } if ( currtok.Type != TokType::BraceSquare_Close ) { - log_failure( "%s: Error, expected ] in type definition, not %s", stringize(parse_typedef), ETokType::to_str( currtok.Type ) ); + log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", ETokType::to_str( currtok.Type ), Context.to_string() ); return Code::Invalid; } @@ -779,6 +801,7 @@ Code parse_array_decl() return array_expr; } + Context.pop(); return { nullptr }; } @@ -786,6 +809,7 @@ internal inline CodeAttributes parse_attributes() { using namespace Parser; + push_scope(); Token start; s32 len = 0; @@ -847,6 +871,7 @@ CodeAttributes parse_attributes() return def_attributes( attribute_txt ); } + pop_context(); return { nullptr }; } @@ -854,6 +879,8 @@ internal inline Parser::Token parse_identifier() { using namespace Parser; + push_scope(); + Token name = currtok; eat( TokType::Identifier ); @@ -864,13 +891,13 @@ Parser::Token parse_identifier() if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of type definition, expected identifier", Context.to_string() ); + log_failure( "Error, unexpected end of static symbol identifier\n%s", Context.to_string() ); return { nullptr, 0, TokType::Invalid }; } if ( currtok.Type != TokType::Identifier ) { - log_failure( "%s: Error, expected identifier in type definition, not %s", context, ETokType::to_str( currtok.Type ) ); + log_failure( "Error, expected static symbol identifier, not %s\n%s", ETokType::to_str( currtok.Type ), Context.to_string() ); return { nullptr, 0, TokType::Invalid }; } @@ -878,6 +905,33 @@ Parser::Token parse_identifier() eat( TokType::Identifier ); } + if ( check( TokType::Operator ) && currtok.Text[0] == '<' ) + { + // Template arguments can be complex so were not validating if they are correct. + s32 level = 0; + while ( left && (currtok.Text[0] != '>' || level > 0 ) ) + { + if ( currtok.Text[0] == '<' ) + level++; + + else if ( currtok.Text[0] == '>' && level > 0 ) + level--; + + eat( currtok.Type ); + } + + if ( left == 0 ) + { + log_failure( "Error, unexpected end of template arguments\n%s", Context.to_string() ); + return { nullptr, 0, TokType::Invalid }; + } + + eat( TokType::Operator ); + + name.Length = ( (sptr)prevtok.Text + (sptr)prevtok.Length ) - (sptr)name.Text; + } + + pop_context(); return name; } @@ -886,6 +940,7 @@ CodeParam parse_params( bool use_template_capture = false ) { using namespace Parser; using namespace ECode; + push_scope(); if ( ! use_template_capture ) eat( TokType::Capture_Start ); @@ -912,7 +967,7 @@ CodeParam parse_params( bool use_template_capture = false ) return param_varadic; } - type = parse_type( toks, context ); + type = parse_type(); if ( type == Code::Invalid ) return CodeInvalid; @@ -931,7 +986,7 @@ CodeParam parse_params( bool use_template_capture = false ) if ( currtok.Type == TokType::Statement_End ) { - log_failure( "gen::%s: Expected value after assignment operator", context ); + log_failure( "Expected value after assignment operator\n%s.", Context.to_string() ); return CodeInvalid; } @@ -995,7 +1050,7 @@ CodeParam parse_params( bool use_template_capture = false ) if ( currtok.Type == TokType::Statement_End ) { - log_failure( "gen::%s: Expected value after assignment operator", context ); + log_failure( "Expected value after assignment operator\n%s", Context.to_string() ); return CodeInvalid; } @@ -1031,12 +1086,13 @@ CodeParam parse_params( bool use_template_capture = false ) { if ( ! check( TokType::Operator) || currtok.Text[0] != '>' ) { - log_failure("gen::parse_params: expected '<' after 'template' keyword. %s", ETokType::to_str( currtok.Type )); + log_failure("Expected '<' after 'template' keyword\n%s", Context.to_string() ); return CodeInvalid; } eat( TokType::Operator ); } + Context.pop(); return result; # undef context } @@ -1052,6 +1108,7 @@ CodeFn parse_function_after_name( ) { using namespace Parser; + push_scope(); CodeParam params = parse_params( toks, stringize(parse_function) ); @@ -1090,7 +1147,7 @@ CodeFn parse_function_after_name( default: { - log_failure("gen::def_function: body must be either of Function_Body or Untyped type. %s", body.debug_str()); + log_failure("Body must be either of Function_Body or Untyped type, %s\n%s", body.debug_str(), Context.to_string()); return CodeInvalid; } } @@ -1111,26 +1168,28 @@ CodeFn parse_function_after_name( if ( params ) result->Params = params; + Context.pop(); return result; } internal inline -CodeOperator parse_operator_after_ret_type( - ModuleFlag mflags +CodeOperator parse_operator_after_ret_type( + ModuleFlag mflags , CodeAttributes attributes - , CodeSpecifiers specifiers + , CodeSpecifiers specifiers , CodeType ret_type ) { using namespace Parser; using namespace EOperator; + push_scope(); // Parse Operator eat( TokType::Decl_Operator ); if ( ! check( TokType::Operator ) ) { - log_failure( "gen::%s: Expected operator after 'operator' keyword", context ); + log_failure( "Expected operator after 'operator' keyword\n%s", Context.to_string() ); return CodeInvalid; } @@ -1316,7 +1375,7 @@ CodeOperator parse_operator_after_ret_type( if ( op == Invalid ) { - log_failure( "gen::%s: Invalid operator '%s'", context, currtok.Text ); + log_failure( "Invalid operator '%s'\n%s", currtok.Text, Context.to_string() ); return CodeInvalid; } @@ -1346,6 +1405,7 @@ CodeOperator parse_operator_after_ret_type( // OpValidateResult check_result = operator__validate( op, params, ret_type, specifiers ); CodeOperator result = def_operator( op, params, ret_type, body, specifiers, attributes, mflags ); + Context.pop(); return result; } @@ -1357,14 +1417,13 @@ CodeVar parse_variable_after_name( ,CodeSpecifiers specifiers , CodeType type , StrC name - , Parser::TokArray& toks - , char const* context ) +) { using namespace Parser; + push_scope(); - Code array_expr = parse_array_decl( toks, stringize(parse_variable) ); - - Code expr = { nullptr }; + Code array_expr = parse_array_decl(); + Code expr = { nullptr }; if ( currtok.IsAssign ) { @@ -1374,7 +1433,7 @@ CodeVar parse_variable_after_name( if ( currtok.Type == TokType::Statement_End ) { - log_failure( "gen::parse_variable: expected expression after assignment operator" ); + log_failure( "Expected expression after assignment operator\n%s", Context.to_string() ); return CodeInvalid; } @@ -1411,6 +1470,7 @@ CodeVar parse_variable_after_name( if ( expr ) result->Value = expr; + Context.pop(); return result; } @@ -1418,6 +1478,7 @@ internal inline Code parse_variable_assignment() { using namespace Parser; + push_scope(); Code expr = Code::Invalid; @@ -1429,7 +1490,7 @@ Code parse_variable_assignment() if ( currtok.Type == TokType::Statement_End ) { - log_failure( "gen::parse_variable: expected expression after assignment operator" ); + log_failure( "Expected expression after assignment operator\n%s", Context.to_string() ); return Code::Invalid; } @@ -1442,6 +1503,7 @@ Code parse_variable_assignment() expr = untyped_str( expr_tok ); } + Context.pop(); return expr; } @@ -1449,6 +1511,7 @@ internal inline Code parse_operator_function_or_variable( bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers ) { using namespace Parser; + push_scope(); Code result = Code::Invalid; @@ -1460,7 +1523,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes if ( check( TokType::Operator) ) { // Dealing with an operator overload - result = parse_operator_after_ret_type( ModuleFlag::None, attributes, specifiers, type, toks, context ); + result = parse_operator_after_ret_type( ModuleFlag::None, attributes, specifiers, type ); } else { @@ -1471,21 +1534,22 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes { // Dealing with a function - result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, type, name, toks, context ); + result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, type, name ); } else { if ( expects_function ) { - log_failure( "gen::parse_operator_function_or_variable: expected function declaration (consteval was used)" ); + log_failure( "Expected function declaration (consteval was used)\n%s", Context.to_string() ); return Code::Invalid; } // Dealing with a variable - result = parse_variable_after_name( ModuleFlag::None, attributes, specifiers, type, name, toks, context ); + result = parse_variable_after_name( ModuleFlag::None, attributes, specifiers, type, name ); } } + Context.pop(); return result; } @@ -1494,6 +1558,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) { using namespace Parser; using namespace ECode; + push_scope(); eat( TokType::BraceCurly_Open ); @@ -1579,7 +1644,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) case TokType::Decl_GNU_Attribute: case TokType::Decl_MSVC_Attribute: #define Entry( attribute, str ) case TokType::attribute: - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { attributes = parse_attributes(); @@ -1615,7 +1680,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) break; default: - log_failure( "gen::parse_class_struct_body: invalid specifier " "%s" " for variable", ESpecifier::to_str(spec) ); + log_failure( "Invalid specifier %s for variable\n%s", ESpecifier::to_str(spec), Context.to_string() ); return CodeInvalid; } @@ -1640,7 +1705,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) case TokType::Type_int: case TokType::Type_double: { - member = parse_operator_function_or_variable( expects_function, attributes, specifiers, toks, context ); + member = parse_operator_function_or_variable( expects_function, attributes, specifiers ); } break; @@ -1659,7 +1724,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) if ( member == Code::Invalid ) { - log_failure( "gen::parse_variable: failed to parse member" ); + log_failure( "Failed to parse member\n%s", Context.to_string() ); return CodeInvalid; } @@ -1667,6 +1732,7 @@ CodeBody parse_class_struct_body( Parser::TokType which ) } eat( TokType::BraceCurly_Close ); + Context.pop(); return result; } @@ -1674,10 +1740,11 @@ internal Code parse_class_struct( Parser::TokType which ) { using namespace Parser; + push_scope(); if ( which != TokType::Decl_Class && which != TokType::Decl_Struct ) { - log_failure( "%s: Error, expected class or struct, not %s", context, ETokType::to_str( which ) ); + log_failure( "Error, expected class or struct, not %s\n%s", ETokType::to_str( which ), Context.to_string() ); return CodeInvalid; } @@ -1755,6 +1822,7 @@ Code parse_class_struct( Parser::TokType which ) ); interfaces.free(); + Context.pop(); return result; } @@ -1763,6 +1831,7 @@ Code parse_function_body() { using namespace Parser; using namespace ECode; + push_scope(); eat( TokType::BraceCurly_Open ); @@ -1794,6 +1863,8 @@ Code parse_function_body() } eat( TokType::BraceCurly_Close ); + + Context.pop(); return result; } @@ -1802,6 +1873,7 @@ CodeBody parse_global_nspace( CodeT which ) { using namespace Parser; using namespace ECode; + push_scope(); if ( which != Namespace_Body && which != Global_Body && which != Export_Body && which != Extern_Linkage_Body ) return CodeInvalid; @@ -1838,7 +1910,7 @@ CodeBody parse_global_nspace( CodeT which ) case TokType::Decl_Extern_Linkage: if ( which == Extern_Linkage_Body ) - log_failure( "gen::parse_global_nspace: nested extern linkage" ); + log_failure( "Nested extern linkage\n%s", Context.to_string() ); member = parse_extern_link_body( toks, context ); break; @@ -1869,7 +1941,7 @@ CodeBody parse_global_nspace( CodeT which ) case TokType::Module_Export: if ( which == Export_Body ) - log_failure( "gen::parse_global_nspace: nested export declaration" ); + log_failure( "Nested export declaration\n%s", Context.to_string() ); member = parse_export_body( toks, context ); break; @@ -1882,7 +1954,7 @@ CodeBody parse_global_nspace( CodeT which ) case TokType::Decl_GNU_Attribute: case TokType::Decl_MSVC_Attribute: #define Entry( attribute, str ) case TokType::attribute: - GEN_Define_Attribute_Tokens + GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { attributes = parse_attributes(); @@ -1919,7 +1991,7 @@ CodeBody parse_global_nspace( CodeT which ) break; default: - log_failure( "gen::parse_global_nspace: invalid specifier " "%s" " for variable", ESpecifier::to_str(spec) ); + log_failure( "Invalid specifier %s for variable\n%s", ESpecifier::to_str(spec), Context.to_string() ); return CodeInvalid; } @@ -1950,7 +2022,7 @@ CodeBody parse_global_nspace( CodeT which ) if ( member == Code::Invalid ) { - log_failure( "gen::%s: failed to parse member", context ); + log_failure( "Failed to parse member\n%s", Context.to_string() ); return CodeInvalid; } @@ -1960,13 +2032,17 @@ CodeBody parse_global_nspace( CodeT which ) if ( which != Global_Body ) eat( TokType::BraceCurly_Close ); + Context.pop(); return result; } internal CodeClass parse_class() { - return (CodeClass) parse_class_struct( Parser::TokType::Decl_Class, toks, context ); + push_scope(); + CodeClass result = (CodeClass) parse_class_struct( Parser::TokType::Decl_Class, toks, context ); + Context.pop(); + return result; } CodeClass parse_class( StrC def ) @@ -1978,7 +2054,10 @@ CodeClass parse_class( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return (CodeClass) parse_class_struct( TokType::Decl_Class, toks, stringize(parse_class) ); + push_scope(); + CodeClass result = (CodeClass) parse_class_struct( TokType::Decl_Class ); + Context.pop(); + return result; } internal @@ -1986,6 +2065,7 @@ CodeEnum parse_enum() { using namespace Parser; using namespace ECode; + push_scope(); SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; @@ -2012,7 +2092,7 @@ CodeEnum parse_enum() if ( currtok.Type != TokType::Identifier ) { - log_failure( "gen::parse_enum: expected identifier for enum name" ); + log_failure( "Expected identifier for enum name\n%s", Context.to_string() ); return CodeInvalid; } @@ -2085,6 +2165,7 @@ CodeEnum parse_enum() if ( type ) result->UnderlyingType = type; + Context.pop(); return result; } @@ -2097,13 +2178,16 @@ CodeEnum parse_enum( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_enum( toks, stringize(parse_enum) ); + return parse_enum(); } internal inline CodeBody parse_export_body() { - return parse_global_nspace( ECode::Export_Body, toks, context ); + push_scope(); + CodeBody result = parse_global_nspace( ECode::Export_Body ); + Context.pop(); + return result; } CodeBody parse_export_body( StrC def ) @@ -2115,19 +2199,23 @@ CodeBody parse_export_body( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_export_body( toks, stringize(parse_export_body) ); + return parse_export_body(); } internal inline -CodeBody parse_extern_link_body( Parser::TokArray& toks, char const* context ) +CodeBody parse_extern_link_body() { - return parse_global_nspace( ECode::Extern_Linkage_Body, toks, context ); + push_scope(); + CodeBody result = parse_global_nspace( ECode::Extern_Linkage_Body ); + Context.pop(); + return result; } internal -CodeExtern parse_extern_link( Parser::TokArray& toks, char const* context ) +CodeExtern parse_extern_link() { using namespace Parser; + push_scope(); eat( TokType::Decl_Extern_Linkage ); @@ -2145,12 +2233,13 @@ CodeExtern parse_extern_link( Parser::TokArray& toks, char const* context ) Code entry = parse_extern_link_body( toks, context ); if ( entry == Code::Invalid ) { - log_failure( "gen::parse_extern_link: failed to parse body" ); + log_failure( "Failed to parse body\n%s", Context.to_string() ); return result; } result->Body = entry; + Context.pop(); return result; } @@ -2163,14 +2252,15 @@ CodeExtern parse_extern_link( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_extern_link( toks, stringize(parse_extern_link) ); + return parse_extern_link(); } internal -CodeFriend parse_friend( Parser::TokArray& toks, char const* context ) +CodeFriend parse_friend() { using namespace Parser; using namespace ECode; + push_scope(); eat( TokType::Decl_Friend ); @@ -2211,6 +2301,7 @@ CodeFriend parse_friend( Parser::TokArray& toks, char const* context ) else result->Declaration = type; + Context.pop(); return result; } @@ -2223,13 +2314,14 @@ CodeFriend parse_friend( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_friend( toks, stringize(parse_friend) ); + return parse_friend(); } internal -CodeFn parse_functon( Parser::TokArray& toks, char const* context ) +CodeFn parse_functon() { using namespace Parser; + push_scope(); SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; @@ -2261,7 +2353,7 @@ CodeFn parse_functon( Parser::TokArray& toks, char const* context ) break; default: - log_failure( "gen::parse_functon: invalid specifier " "%s" " for functon", ESpecifier::to_str(spec) ); + log_failure( "Invalid specifier %s for functon\n%s", ESpecifier::to_str(spec), Context.to_string() ); return CodeInvalid; } @@ -2286,22 +2378,22 @@ CodeFn parse_functon( Parser::TokArray& toks, char const* context ) if ( ! name ) return CodeInvalid; - CodeFn result = parse_function_after_name( mflags, attributes, specifiers, ret_type, name, toks, context ); + CodeFn result = parse_function_after_name( mflags, attributes, specifiers, ret_type, name ); + Context.pop(); return result; } CodeFn parse_function( StrC def ) { - using namespace Parser; - check_parse_args( parse_function, def ); + using namespace Parser; TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return CodeInvalid; - return (CodeFn) parse_functon( toks, stringize(parse_function) ); + return (CodeFn) parse_functon(); } CodeBody parse_global_body( StrC def ) @@ -2313,13 +2405,18 @@ CodeBody parse_global_body( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_global_nspace( ECode::Global_Body, toks, stringize(parse_global_body) ); + push_scope(); + CodeBody result = parse_global_nspace( ECode::Global_Body ); + Context.pop(); + return result; } internal CodeNamespace parse_namespace( Parser::TokArray& toks, char const* context ) { using namespace Parser; + push_scope(); + eat( TokType::Decl_Namespace ); Token name = parse_identifier( toks, stringize(parse_namespace) ); @@ -2335,6 +2432,7 @@ CodeNamespace parse_namespace( Parser::TokArray& toks, char const* context ) result->Body = body; + Context.pop(); return result; } @@ -2347,13 +2445,14 @@ CodeNamespace parse_namespace( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_namespace( toks, stringize(parse_namespace) ); + return parse_namespace(); } internal -CodeOperator parse_operator( Parser::TokArray& toks, char const* context ) +CodeOperator parse_operator() { using namespace Parser; + push_scope(); CodeAttributes attributes = { nullptr }; CodeSpecifiers specifiers = { nullptr }; @@ -2383,7 +2482,7 @@ CodeOperator parse_operator( Parser::TokArray& toks, char const* context ) break; default: - log_failure( "gen::parse_operator: invalid specifier " "%s" " for operator", ESpecifier::to_str(spec) ); + log_failure( "Invalid specifier " "%s" " for operator\n%s", ESpecifier::to_str(spec), Context.to_string() ); return CodeInvalid; } @@ -2403,7 +2502,9 @@ CodeOperator parse_operator( Parser::TokArray& toks, char const* context ) // Parse Return Type CodeType ret_type = parse_type( toks, stringize(parse_operator) ); - CodeOperator result = parse_operator_after_ret_type( mflags, attributes, specifiers, ret_type, toks, context ); + CodeOperator result = parse_operator_after_ret_type( mflags, attributes, specifiers, ret_type ); + + Context.pop(); return result; } @@ -2416,12 +2517,13 @@ CodeOperator parse_operator( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return (CodeOperator) parse_operator( toks, stringize(parse_operator) ); + return (CodeOperator) parse_operator(); } -CodeOpCast parse_operator_cast( Parser::TokArray& toks, char const* context ) +CodeOpCast parse_operator_cast() { using namespace Parser; + push_scope(); eat( TokType::Decl_Operator ); @@ -2479,6 +2581,7 @@ CodeOpCast parse_operator_cast( Parser::TokArray& toks, char const* context ) result->ValueType = type; + Context.pop(); return result; } @@ -2491,13 +2594,16 @@ CodeOpCast parse_operator_cast( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_operator_cast( toks, stringize(parse_operator_cast) ); + return parse_operator_cast(); } internal inline -CodeStruct parse_struct( Parser::TokArray& toks, char const* context ) +CodeStruct parse_struct() { - return (CodeStruct) parse_class_struct( Parser::TokType::Decl_Struct, toks, stringize(parse_struct) ); + push_scope(); + CodeStruct result = (CodeStruct) parse_class_struct(); + Context.pop(); + return result; } CodeStruct parse_struct( StrC def ) @@ -2509,15 +2615,16 @@ CodeStruct parse_struct( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return (CodeStruct) parse_class_struct( TokType::Decl_Struct, toks, stringize(parse_struct) ); + return (CodeStruct) parse_class_struct( TokType::Decl_Struct ); } internal -CodeTemplate parse_template( Parser::TokArray& toks, char const* context ) +CodeTemplate parse_template() { # define UseTemplateCapture true using namespace Parser; + push_scope(); ModuleFlag mflags = ModuleFlag::None; @@ -2592,7 +2699,7 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context ) break; default: - log_failure( "gen::parse_template: invalid specifier %s for variable or function", ESpecifier::to_str( spec ) ); + log_failure( "Invalid specifier %s for variable or function\n%s", ESpecifier::to_str( spec ), Context.to_string() ); return CodeInvalid; } @@ -2621,6 +2728,7 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context ) result->Declaration = definition; result->ModuleFlags = mflags; + Context.pop(); return result; # undef UseTemplateCapture } @@ -2634,13 +2742,14 @@ CodeTemplate parse_template( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_template( toks, stringize(parse_template) ); + return parse_template(); } internal -CodeType parse_type( Parser::TokArray& toks, char const* context ) +CodeType parse_type() { using namespace Parser; + push_scope(); Token context_tok = prevtok; @@ -2658,7 +2767,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) if ( spec != ESpecifier::Const ) { - log_failure( "gen::parse_type: Error, invalid specifier used in type definition: %s", currtok.Text ); + log_failure( "Error, invalid specifier used in type definition: %s\n%s", currtok.Text, Context.to_string() ); return CodeInvalid; } @@ -2669,7 +2778,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) if ( left == 0 ) { - log_failure( "%s: Error, unexpected end of type definition", context ); + log_failure( "Error, unexpected end of type definition\n%s", Context.to_string() ); return CodeInvalid; } @@ -2734,7 +2843,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) && spec != ESpecifier::Ref && spec != ESpecifier::RValue ) { - log_failure( "%s: Error, invalid specifier used in type definition: %s", context, currtok.Text ); + log_failure( "Error, invalid specifier used in type definition: %s\n%s", currtok.Text, Context.to_string() ); return CodeInvalid; } @@ -2809,6 +2918,7 @@ CodeType parse_type( Parser::TokArray& toks, char const* context ) if ( attributes ) result->Attributes = attributes; + Context.pop(); return result; } @@ -2821,15 +2931,14 @@ CodeType parse_type( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - CodeType result = parse_type( toks, stringize(parse_type) ); - - return result; + return parse_type(); } internal -CodeTypedef parse_typedef( Parser::TokArray& toks, char const* context ) +CodeTypedef parse_typedef() { using namespace Parser; + push_scope(); Token name = { nullptr, 0, TokType::Invalid }; Code array_expr = { nullptr }; @@ -2862,7 +2971,7 @@ CodeTypedef parse_typedef( Parser::TokArray& toks, char const* context ) if ( ! check( TokType::Identifier ) ) { - log_failure( "gen::parse_typedef: Error, expected identifier for typedef" ); + log_failure( "Error, expected identifier for typedef\n%s", Context.to_string() ); return CodeInvalid; } @@ -2886,6 +2995,7 @@ CodeTypedef parse_typedef( Parser::TokArray& toks, char const* context ) if ( type->Type == Typename && array_expr && array_expr->Type != Invalid ) type.cast()->ArrExpr = array_expr; + Context.pop(); return result; } @@ -2898,13 +3008,14 @@ CodeTypedef parse_typedef( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_typedef( toks, stringize(parse_typedef) ); + return parse_typedef(); } internal -CodeUnion parse_union( Parser::TokArray& toks, char const* context ) +CodeUnion parse_union() { using namespace Parser; + push_scope(); ModuleFlag mflags = ModuleFlag::None; @@ -2958,6 +3069,7 @@ CodeUnion parse_union( Parser::TokArray& toks, char const* context ) if ( attributes ) result->Attributes = attributes; + Context.pop(); return result; } @@ -2970,13 +3082,14 @@ CodeUnion parse_union( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_union( toks, stringize(parse_union) ); + return parse_union(); } internal -CodeUsing parse_using( Parser::TokArray& toks, char const* context ) +CodeUsing parse_using() { using namespace Parser; + push_scope(); SpecifierT specs_found[16] { ESpecifier::Invalid }; s32 NumSpecifiers = 0; @@ -3044,6 +3157,8 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context ) if ( attributes ) result->Attributes = attributes; } + + Context.pop(); return result; } @@ -3056,13 +3171,14 @@ CodeUsing parse_using( StrC def ) if ( toks.Arr == nullptr ) return CodeInvalid; - return parse_using( toks, stringize(parse_using) ); + return parse_using(); } internal CodeVar parse_variable() { using namespace Parser; + push_scope(); SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; s32 NumSpecifiers = 0; @@ -3099,7 +3215,7 @@ CodeVar parse_variable() break; default: - log_failure( "gen::parse_variable: invalid specifier %s for variable", ESpecifier::to_str( spec ) ); + log_failure( "Invalid specifier %s for variable\n%s", ESpecifier::to_str( spec ), Context.to_string() ); return CodeInvalid; } @@ -3125,33 +3241,30 @@ CodeVar parse_variable() Context.Scope->Name = current; eat( TokType::Identifier ); - CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, Context.Scope->Name, Context.Tokens, Context.Scope->ProcName ); + CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, Context.Scope->Name ); + Context.pop(); return result; } CodeVar parse_variable( StrC def ) { - using namespace Parser; check_parse_args( parse_variable, def ); + using namespace Parser; TokArray toks = lex( def ); if ( toks.Arr == nullptr ) return CodeInvalid; - Context.Tokens = toks; - Parser::StackNode root - { - toks.current(), - { nullptr, 0, TokType::Invalid }, - name(parse_variable) - }; - return parse_variable(); } // Undef helper macros # undef check_parse_args -# undef curr_tok +# undef nexttok +# undef currtok +# undef prevtok # undef eat # undef left +# undef check +# undef push_scope diff --git a/project/components/gen.interface.upfront.cpp b/project/components/interface.upfront.cpp similarity index 100% rename from project/components/gen.interface.upfront.cpp rename to project/components/interface.upfront.cpp diff --git a/project/components/gen.data.cpp b/project/components/static_data.cpp similarity index 100% rename from project/components/gen.data.cpp rename to project/components/static_data.cpp diff --git a/project/components/gen.types.hpp b/project/components/types.hpp similarity index 96% rename from project/components/gen.types.hpp rename to project/components/types.hpp index 20f201f..0e9d6a3 100644 --- a/project/components/gen.types.hpp +++ b/project/components/types.hpp @@ -68,7 +68,7 @@ ModuleFlag operator|( ModuleFlag A, ModuleFlag B) Override these to change the attribute to your own unique identifier convention. - The tokenizer identifies attribute defines with the GEN_Define_Attribute_Tokens macros. + The tokenizer identifies attribute defines with the GEN_DEFINE_ATTRIBUTE_TOKENS macros. See the example below and the Define_TokType macro used in gen.cpp to know the format. While the library can parse raw attributes, most projects use defines to wrap them for compiler platform indendence. The token define allows support for them without having to modify the library. diff --git a/project/components/gen.untyped.cpp b/project/components/untyped.cpp similarity index 97% rename from project/components/gen.untyped.cpp rename to project/components/untyped.cpp index d33ca23..12f4971 100644 --- a/project/components/gen.untyped.cpp +++ b/project/components/untyped.cpp @@ -12,8 +12,7 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va ) char tok_map_mem[ TokenFmt_TokenMap_MemSize ]; tok_map_arena = Arena::init_from_memory( tok_map_mem, sizeof(tok_map_mem) ); - - tok_map = HashTable::init( tok_map_arena ); + tok_map = HashTable::init( tok_map_arena ); s32 left = num_tokens - 1; diff --git a/project/dependencies/gen.basic_types.hpp b/project/dependencies/basic_types.hpp similarity index 100% rename from project/dependencies/gen.basic_types.hpp rename to project/dependencies/basic_types.hpp diff --git a/project/dependencies/gen.containers.hpp b/project/dependencies/containers.hpp similarity index 100% rename from project/dependencies/gen.containers.hpp rename to project/dependencies/containers.hpp diff --git a/project/dependencies/gen.debug.cpp b/project/dependencies/debug.cpp similarity index 100% rename from project/dependencies/gen.debug.cpp rename to project/dependencies/debug.cpp diff --git a/project/dependencies/gen.debug.hpp b/project/dependencies/debug.hpp similarity index 100% rename from project/dependencies/gen.debug.hpp rename to project/dependencies/debug.hpp diff --git a/project/dependencies/gen.file_handling.cpp b/project/dependencies/file_handling.cpp similarity index 100% rename from project/dependencies/gen.file_handling.cpp rename to project/dependencies/file_handling.cpp diff --git a/project/dependencies/gen.file_handling.hpp b/project/dependencies/file_handling.hpp similarity index 100% rename from project/dependencies/gen.file_handling.hpp rename to project/dependencies/file_handling.hpp diff --git a/project/dependencies/gen.hashing.cpp b/project/dependencies/hashing.cpp similarity index 100% rename from project/dependencies/gen.hashing.cpp rename to project/dependencies/hashing.cpp diff --git a/project/dependencies/gen.hashing.hpp b/project/dependencies/hashing.hpp similarity index 100% rename from project/dependencies/gen.hashing.hpp rename to project/dependencies/hashing.hpp diff --git a/project/dependencies/gen.header_start.hpp b/project/dependencies/header_start.hpp similarity index 100% rename from project/dependencies/gen.header_start.hpp rename to project/dependencies/header_start.hpp diff --git a/project/dependencies/gen.impl_start.cpp b/project/dependencies/impl_start.cpp similarity index 100% rename from project/dependencies/gen.impl_start.cpp rename to project/dependencies/impl_start.cpp diff --git a/project/dependencies/gen.macros.hpp b/project/dependencies/macros.hpp similarity index 100% rename from project/dependencies/gen.macros.hpp rename to project/dependencies/macros.hpp diff --git a/project/dependencies/gen.memory.cpp b/project/dependencies/memory.cpp similarity index 100% rename from project/dependencies/gen.memory.cpp rename to project/dependencies/memory.cpp diff --git a/project/dependencies/gen.memory.hpp b/project/dependencies/memory.hpp similarity index 100% rename from project/dependencies/gen.memory.hpp rename to project/dependencies/memory.hpp diff --git a/project/dependencies/gen.parsing.cpp b/project/dependencies/parsing.cpp similarity index 100% rename from project/dependencies/gen.parsing.cpp rename to project/dependencies/parsing.cpp diff --git a/project/dependencies/gen.parsing.hpp b/project/dependencies/parsing.hpp similarity index 100% rename from project/dependencies/gen.parsing.hpp rename to project/dependencies/parsing.hpp diff --git a/project/dependencies/gen.printing.cpp b/project/dependencies/printing.cpp similarity index 100% rename from project/dependencies/gen.printing.cpp rename to project/dependencies/printing.cpp diff --git a/project/dependencies/gen.printing.hpp b/project/dependencies/printing.hpp similarity index 86% rename from project/dependencies/gen.printing.hpp rename to project/dependencies/printing.hpp index 9914e97..567bee9 100644 --- a/project/dependencies/gen.printing.hpp +++ b/project/dependencies/printing.hpp @@ -10,12 +10,15 @@ struct FileInfo; char* str_fmt_buf ( char const* fmt, ... ); char* str_fmt_buf_va ( char const* fmt, va_list va ); sw str_fmt_va ( char* str, sw n, char const* fmt, va_list va ); -sw str_fmt_file ( FileInfo* f, char const* fmt, ... ); -sw str_fmt_file_va ( FileInfo* f, char const* fmt, va_list va ); sw str_fmt_out_va ( char const* fmt, va_list va ); sw str_fmt_out_err ( char const* fmt, ... ); sw str_fmt_out_err_va( char const* fmt, va_list va ); +// TODO : Move these to file handling. + +sw str_fmt_file ( FileInfo* f, char const* fmt, ... ); +sw str_fmt_file_va ( FileInfo* f, char const* fmt, va_list va ) + constexpr char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED"; diff --git a/project/dependencies/gen.string.cpp b/project/dependencies/string.cpp similarity index 100% rename from project/dependencies/gen.string.cpp rename to project/dependencies/string.cpp diff --git a/project/dependencies/gen.string.hpp b/project/dependencies/string.hpp similarity index 100% rename from project/dependencies/gen.string.hpp rename to project/dependencies/string.hpp diff --git a/project/dependencies/gen.string_ops.cpp b/project/dependencies/string_ops.cpp similarity index 100% rename from project/dependencies/gen.string_ops.cpp rename to project/dependencies/string_ops.cpp diff --git a/project/dependencies/gen.string_ops.hpp b/project/dependencies/string_ops.hpp similarity index 100% rename from project/dependencies/gen.string_ops.hpp rename to project/dependencies/string_ops.hpp diff --git a/project/dependencies/gen.timing.cpp b/project/dependencies/timing.cpp similarity index 100% rename from project/dependencies/gen.timing.cpp rename to project/dependencies/timing.cpp diff --git a/project/dependencies/gen.timing.hpp b/project/dependencies/timing.hpp similarity index 100% rename from project/dependencies/gen.timing.hpp rename to project/dependencies/timing.hpp diff --git a/project/components/AttributeTokens.csv b/project/enums/AttributeTokens.csv similarity index 100% rename from project/components/AttributeTokens.csv rename to project/enums/AttributeTokens.csv diff --git a/project/components/ECode.csv b/project/enums/ECode.csv similarity index 100% rename from project/components/ECode.csv rename to project/enums/ECode.csv diff --git a/project/components/EOperator.csv b/project/enums/EOperator.csv similarity index 100% rename from project/components/EOperator.csv rename to project/enums/EOperator.csv diff --git a/project/components/ESpecifier.csv b/project/enums/ESpecifier.csv similarity index 100% rename from project/components/ESpecifier.csv rename to project/enums/ESpecifier.csv diff --git a/project/components/ETokType.csv b/project/enums/ETokType.csv similarity index 100% rename from project/components/ETokType.csv rename to project/enums/ETokType.csv diff --git a/project/filesystem/gen.builder.cpp b/project/file_processors/builder.cpp similarity index 100% rename from project/filesystem/gen.builder.cpp rename to project/file_processors/builder.cpp diff --git a/project/filesystem/gen.builder.hpp b/project/file_processors/builder.hpp similarity index 100% rename from project/filesystem/gen.builder.hpp rename to project/file_processors/builder.hpp diff --git a/project/filesystem/gen.editor.hpp b/project/file_processors/editor.hpp similarity index 100% rename from project/filesystem/gen.editor.hpp rename to project/file_processors/editor.hpp diff --git a/project/filesystem/gen.scanner.hpp b/project/file_processors/scanner.hpp similarity index 100% rename from project/filesystem/gen.scanner.hpp rename to project/file_processors/scanner.hpp diff --git a/project/gen.bootstrap.cpp b/project/gen.bootstrap.cpp index fac2c4c..0a706ed 100644 --- a/project/gen.bootstrap.cpp +++ b/project/gen.bootstrap.cpp @@ -2,8 +2,8 @@ #define GEN_ENFORCE_STRONG_CODE_TYPES #define GEN_EXPOSE_BACKEND #include "gen.cpp" -#include "filesystem/gen.scanner.hpp" -#include "helpers/gen.helper.hpp" +#include "filesystem/scanner.hpp" +#include "helpers/helper.hpp" using namespace gen; @@ -33,25 +33,27 @@ int gen_main() { gen::init(); - Code push_ignores = scan_file( "helpers/gen.push_ignores.inline.hpp" ); - Code pop_ignores = scan_file( "helpers/gen.pop_ignores.inline.hpp" ); + Code push_ignores = scan_file( "helpers/push_ignores.inline.hpp" ); + Code pop_ignores = scan_file( "helpers/pop_ignores.inline.hpp" ); // gen_dep.hpp { - Code header_start = scan_file( "dependencies/gen.header_start.hpp" ); + Code header_start = scan_file( "dependencies/header_start.hpp" ); Code nspace_macro = untyped_str( namespace_by_default ? nspace_default : nspace_non_default ); - Code macros = scan_file( "dependencies/gen.macros.hpp" ); - Code basic_types = scan_file( "dependencies/gen.basic_types.hpp" ); - Code debug = scan_file( "dependencies/gen.debug.hpp" ); - Code memory = scan_file( "dependencies/gen.memory.hpp" ); - Code string_ops = scan_file( "dependencies/gen.string_ops.hpp" ); - Code printing = scan_file( "dependencies/gen.printing.hpp" ); - Code containers = scan_file( "dependencies/gen.containers.hpp" ); - Code hashing = scan_file( "dependencies/gen.hashing.hpp" ); - Code string = scan_file( "dependencies/gen.string.hpp" ); - Code file_handling = scan_file( "dependencies/gen.file_handling.hpp" ); - Code parsing = scan_file( "dependencies/gen.parsing.hpp" ); - Code timing = scan_file( "dependencies/gen.timing.hpp" ); + Code macros = scan_file( "dependencies/macros.hpp" ); + Code basic_types = scan_file( "dependencies/basic_types.hpp" ); + Code debug = scan_file( "dependencies/debug.hpp" ); + Code memory = scan_file( "dependencies/memory.hpp" ); + Code string_ops = scan_file( "dependencies/string_ops.hpp" ); + Code printing = scan_file( "dependencies/printing.hpp" ); + Code containers = scan_file( "dependencies/containers.hpp" ); + Code hashing = scan_file( "dependencies/hashing.hpp" ); + Code string = scan_file( "dependencies/string.hpp" ); + Code parsing = scan_file( "dependencies/parsing.hpp" ); + Code timing = scan_file( "dependencies/timing.hpp" ); + + // TOOD : Make this optional + Code file_handling = scan_file( "dependencies/file_handling.hpp" ); Builder deps_header; @@ -82,15 +84,15 @@ int gen_main() // gen_dep.cpp { CodeInclude header = def_include( txt_StrC("gen_dep.hpp") ); - Code impl_start = scan_file( "dependencies/gen.impl_start.cpp" ); - Code debug = scan_file( "dependencies/gen.debug.cpp" ); - Code string_ops = scan_file( "dependencies/gen.string_ops.cpp" ); - Code printing = scan_file( "dependencies/gen.printing.cpp" ); - Code memory = scan_file( "dependencies/gen.memory.cpp" ); - Code parsing = scan_file( "dependencies/gen.parsing.cpp" ); - Code hashing = scan_file( "dependencies/gen.hashing.cpp" ); - Code string = scan_file( "dependencies/gen.string.cpp" ); - Code timing = scan_file( "dependencies/gen.timing.cpp" ); + Code impl_start = scan_file( "dependencies/impl_start.cpp" ); + Code debug = scan_file( "dependencies/debug.cpp" ); + Code string_ops = scan_file( "dependencies/string_ops.cpp" ); + Code printing = scan_file( "dependencies/printing.cpp" ); + Code memory = scan_file( "dependencies/memory.cpp" ); + Code parsing = scan_file( "dependencies/parsing.cpp" ); + Code hashing = scan_file( "dependencies/hashing.cpp" ); + Code string = scan_file( "dependencies/string.cpp" ); + Code timing = scan_file( "dependencies/timing.cpp" ); Builder deps_impl; @@ -115,18 +117,19 @@ int gen_main() // gen.hpp { - Code header_start = scan_file( "components/gen.header_start.hpp" ); + Code header_start = scan_file( "components/header_start.hpp" ); Code nspace_macro = untyped_str( namespace_by_default ? nspace_default : nspace_non_default ); - Code types = scan_file( "components/gen.types.hpp" ); - Code data_structs = scan_file( "components/gen.data_structures.hpp" ); - Code interface = scan_file( "components/gen.interface.hpp" ); - Code header_end = scan_file( "components/gen.header_end.hpp" ); + Code types = scan_file( "components/types.hpp" ); + Code data_structs = scan_file( "components/data_structures.hpp" ); + Code interface = scan_file( "components/interface.hpp" ); + Code header_end = scan_file( "components/header_end.hpp" ); - CodeBody ecode = gen_ecode( "./components/ECode.csv" ); - CodeBody eoperator = gen_eoperator( "./components/EOperator.csv" ); - CodeBody especifier = gen_especifier( "./components/ESpecifier.csv" ); + CodeBody ecode = gen_ecode( "enums/ECode.csv" ); + CodeBody eoperator = gen_eoperator( "enums/EOperator.csv" ); + CodeBody especifier = gen_especifier( "enums/ESpecifier.csv" ); - Code builder = scan_file( "filesystem/gen.builder.hpp" ); + // TODO : Make this optional to include + Code builder = scan_file( "file_proecessors/builder.hpp" ); Builder header; @@ -159,18 +162,19 @@ int gen_main() { Code impl_start = scan_file( "components/gen.impl_start.cpp" ); CodeInclude header = def_include( txt_StrC("gen.hpp") ); - Code data = scan_file( "components/gen.data.cpp" ); - Code ast_case_macros = scan_file( "components/gen.ast_case_macros.cpp" ); - Code ast = scan_file( "components/gen.ast.cpp" ); - Code interface = scan_file( "components/gen.interface.cpp" ); - Code upfront = scan_file( "components/gen.interface.upfront.cpp" ); - Code parsing = scan_file( "components/gen.interface.parsing.cpp" ); - Code untyped = scan_file( "components/gen.untyped.cpp" ); + Code data = scan_file( "components/static_data.cpp" ); + Code ast_case_macros = scan_file( "components/ast_case_macros.cpp" ); + Code ast = scan_file( "components/ast.cpp" ); + Code interface = scan_file( "components/interface.cpp" ); + Code upfront = scan_file( "components/interface.upfront.cpp" ); + Code parsing = scan_file( "components/interface.parsing.cpp" ); + Code untyped = scan_file( "components/untyped.cpp" ); - CodeBody etoktype = gen_etoktype( "components/ETokType.csv", "components/AttributeTokens.csv" ); + CodeBody etoktype = gen_etoktype( "enums/ETokType.csv", "enums/AttributeTokens.csv" ); CodeNamespace parser_nspace = def_namespace( name(Parser), def_namespace_body( args(etoktype)) ); - Code builder = scan_file( "filesystem/gen.builder.cpp" ); + // TODO : Make this optional to include + Code builder = scan_file( "file_proecessors/builder.cpp" ); Builder impl; diff --git a/project/gen.cpp b/project/gen.cpp index a90f524..27668b5 100644 --- a/project/gen.cpp +++ b/project/gen.cpp @@ -1,4 +1,4 @@ -#include "helpers/gen.push_ignores.inline.hpp" +#include "helpers/push_ignores.inline.hpp" // ReSharper disable CppClangTidyClangDiagnosticSwitchEnum @@ -16,19 +16,19 @@ GEN_NS_BEGIN -#include "components/gen.data.cpp" +#include "components/static_data.cpp" -#include "components/gen.ast_case_macros.cpp" -#include "components/gen.ast.cpp" +#include "components/ast_case_macros.cpp" +#include "components/ast.cpp" -#include "components/gen.interface.cpp" -#include "components/gen.interface.upfront.cpp" -#include "components/gen.etoktype.cpp" -#include "components/gen.interface.parsing.cpp" -#include "components/gen.untyped.cpp" +#include "components/interface.cpp" +#include "components/interface.upfront.cpp" +#include "components/etoktype.cpp" +#include "components/interface.parsing.cpp" +#include "components/untyped.cpp" -#include "filesystem/gen.builder.cpp" +#include "file_proecessors/builder.cpp" GEN_NS_END -#include "helpers/gen.pop_ignores.inline.hpp" +#include "helpers/pop_ignores.inline.hpp" diff --git a/project/gen.dep.cpp b/project/gen.dep.cpp index a7589b7..e7b4f92 100644 --- a/project/gen.dep.cpp +++ b/project/gen.dep.cpp @@ -1,19 +1,19 @@ // This file is intended to be included within gen.cpp (There is no pragma diagnostic ignores) #include "gen.dep.hpp" -#include "dependencies/gen.impl_start.cpp" +#include "dependencies/impl_start.cpp" GEN_NS_BEGIN -#include "dependencies/gen.debug.cpp" -#include "dependencies/gen.string_ops.cpp" -#include "dependencies/gen.printing.cpp" -#include "dependencies/gen.memory.cpp" -#include "dependencies/gen.parsing.cpp" -#include "dependencies/gen.hashing.cpp" -#include "dependencies/gen.string.cpp" -#include "dependencies/gen.timing.cpp" +#include "dependencies/debug.cpp" +#include "dependencies/string_ops.cpp" +#include "dependencies/printing.cpp" +#include "dependencies/memory.cpp" +#include "dependencies/parsing.cpp" +#include "dependencies/hashing.cpp" +#include "dependencies/string.cpp" +#include "dependencies/timing.cpp" -#include "dependencies/gen.file_handling.cpp" +#include "dependencies/file_handling.cpp" GEN_NS_END diff --git a/project/gen.dep.hpp b/project/gen.dep.hpp index b6d6303..f315397 100644 --- a/project/gen.dep.hpp +++ b/project/gen.dep.hpp @@ -1,7 +1,7 @@ // This file is intended to be included within gen.hpp (There is no pragma diagnostic ignores) #pragma once -#include "dependencies/gen.header_start.hpp" +#include "dependencies/header_start.hpp" #ifdef GEN_DONT_USE_NAMESPACE # define GEN_NS_BEGIN @@ -13,16 +13,17 @@ GEN_NS_BEGIN -#include "dependencies/gen.macros.hpp" -#include "dependencies/gen.basic_types.hpp" -#include "dependencies/gen.debug.hpp" -#include "dependencies/gen.memory.hpp" -#include "dependencies/gen.string_ops.hpp" -#include "dependencies/gen.printing.hpp" -#include "dependencies/gen.containers.hpp" -#include "dependencies/gen.string.hpp" -#include "dependencies/gen.file_handling.hpp" -#include "dependencies/gen.parsing.hpp" -#include "dependencies/gen.timing.hpp" +#include "dependencies/macros.hpp" +#include "dependencies/basic_types.hpp" +#include "dependencies/debug.hpp" +#include "dependencies/memory.hpp" +#include "dependencies/string_ops.hpp" +#include "dependencies/printing.hpp" +#include "dependencies/containers.hpp" +#include "dependencies/string.hpp" +#include "dependencies/parsing.hpp" +#include "dependencies/timing.hpp" + +#include "dependencies/file_handling.hpp" GEN_NS_END diff --git a/project/gen.hpp b/project/gen.hpp index de4836d..e3abc5b 100644 --- a/project/gen.hpp +++ b/project/gen.hpp @@ -8,8 +8,8 @@ */ #pragma once -#include "helpers/gen.push_ignores.inline.hpp" -#include "components/gen.header_start.hpp" +#include "helpers/push_ignores.inline.hpp" +#include "components/header_start.hpp" #ifdef GEN_DONT_USE_NAMESPACE # define GEN_NS_BEGIN @@ -21,16 +21,16 @@ GEN_NS_BEGIN -#include "components/gen.types.hpp" -#include "components/gen.ecode.hpp" -#include "components/gen.eoperator.hpp" -#include "components/gen.especifier.hpp" -#include "components/gen.data_structures.hpp" -#include "components/gen.interface.hpp" -#include "components/gen.header_end.hpp" +#include "components/types.hpp" +#include "components/ecode.hpp" +#include "components/eoperator.hpp" +#include "components/especifier.hpp" +#include "components/data_structures.hpp" +#include "components/interface.hpp" +#include "components/header_end.hpp" -#include "filesystem/gen.builder.hpp" +#include "file_processors/builder.hpp" GEN_NS_END -#include "helpers/gen.pop_ignores.inline.hpp" +#include "helpers/pop_ignores.inline.hpp" diff --git a/project/helpers/gen.helper.hpp b/project/helpers/helper.hpp similarity index 100% rename from project/helpers/gen.helper.hpp rename to project/helpers/helper.hpp diff --git a/project/helpers/gen.pop_ignores.inline.hpp b/project/helpers/pop_ignores.inline.hpp similarity index 100% rename from project/helpers/gen.pop_ignores.inline.hpp rename to project/helpers/pop_ignores.inline.hpp diff --git a/project/helpers/gen.push_ignores.inline.hpp b/project/helpers/push_ignores.inline.hpp similarity index 100% rename from project/helpers/gen.push_ignores.inline.hpp rename to project/helpers/push_ignores.inline.hpp diff --git a/project/helpers/gen.undef.macros.hpp b/project/helpers/undef.macros.hpp similarity index 100% rename from project/helpers/gen.undef.macros.hpp rename to project/helpers/undef.macros.hpp diff --git a/scripts/Readme.md b/scripts/Readme.md index b4060d6..ec5b2de 100644 --- a/scripts/Readme.md +++ b/scripts/Readme.md @@ -1,12 +1,34 @@ # Scripts -Build and cleanup scripts for the test directory are found here along with `natvis` and `natstepfilter` files for debugging. +Generation, testing, and cleanup scripts for the test directory are found here along with `natvis` and `natstepfilter` files for debugging. -The build works as follows: +## Refactoring -* Compile and run the meta-program, it will dump files to the `test/gen` directory. -* Format the files using clang-format -* Build a program that uses some the generated definitions. (Have not done yet) +`refactor.ps1` Provides a way to run the [refactor](github.com/Ed94/refactor) program. It uses the `gencpp.refactor` script to complete a mass refactor of all content within the files of the specified within the script. -The `test/gen` directory has the meson.build config for the meta-program -The `test` directory has the one for the dependent-program. +Currently `refactor` only supports naive sort of *find and replace* feature set and will not be able to rename identifiers excluisvely to a specific context (such as only renaming member names of a specific struct, etc). + +## Build & Run Scripts + +**`clean.ps1`** +Remove any generated content from the repository. + +**`bootstrap.ps1`** +Generate a version of gencpp where components are inlined directly to `gen.` and `gen. ` +Any heavily preprocessed code is not inlined and are instead generated using the code in the `helpers` directory. + +**`singlheader.build.ps1`** +Generate a single-header version of the library where all code that would normally good in the usual four files (see bootstrap) are inlined into a single `gen.hpp` file. +As with the bootstrap, any heavily preprocessed code is not inlined and instead generated with helper code. + +**`test.gen.build.ps1`** +Build the metaprogram for generating the test code. + +**`test.gen.ps1`** +Build (if not already) the metaprogram for generating test code, then run it to generate code. + +**`test.build.ps1`** +Build and run metaprogram, build test program. + +**`test.run.ps1`** +Build and run metaprogram, build and run test program. diff --git a/scripts/get_sources.ps1 b/scripts/get_sources.ps1 deleted file mode 100644 index e5a2681..0000000 --- a/scripts/get_sources.ps1 +++ /dev/null @@ -1,12 +0,0 @@ -[string[]] $include = 'gen.cpp' #'*.c', '*.cc', '*.cpp' -# [string[]] $exclude = - -$path_root = git rev-parse --show-toplevel -$path_proj = Join-Path $path_root project - -$files = Get-ChildItem -Recurse -Path $path_proj -Include $include -Exclude $exclude - -$sources = $files | Select-Object -ExpandProperty FullName | Resolve-Path -Relative -$sources = $sources.Replace( '\', '/' ) - -return $sources diff --git a/scripts/build.ci.ps1 b/scripts/test.gen_build.ci.ps1 similarity index 100% rename from scripts/build.ci.ps1 rename to scripts/test.gen_build.ci.ps1 diff --git a/scripts/build.ps1 b/scripts/test.gen_build.ps1 similarity index 100% rename from scripts/build.ps1 rename to scripts/test.gen_build.ps1 diff --git a/scripts/gen.ps1 b/scripts/test.gen_run.ps1 similarity index 100% rename from scripts/gen.ps1 rename to scripts/test.gen_run.ps1