WIP: Restructuring project

.vscode/c_cpp_properties.json

@@ -3,7 +3,7 @@
         {
             "name": "Bootstrap",
             "includePath": [
-                "${workspaceFolder}/project/**"
+                "${workspaceFolder}/base/**"
             ],
             "defines": [
                 "_DEBUG",

base/Readme.md

@@ -0,0 +1,68 @@
+# Documentation
+
+The library is fragmented into a series of headers and source files meant to be scanned in and then generated to a standard target format, or a user's desires.
+
+Standard formats:
+
+* **base**: Files are in granular pieces separated into four directories:
+  * **dependencies**: Originally from the c-zpl library and modified thereafter.
+  * **components**: The essential definitions of the library.
+  * **helpers**: Contains helper functionality used by base and other libraries to regenerate or generate the other library formats.
+    * `base_codegen.hpp`: Helps with self-hosted code generation of enums, and operator overload inlines of the code types.
+    * `<push/pop>.<name>.inline.<hpp>`: macros that are meant to be injected at specific locations of the library.
+    * `misc.hpp`:
+    * `undef.macros.h`: Undefines all macros from library that original were intended to leak into user code.
+  * **auxillary**: Non-essential tooling:
+    * `Builder`: Similar conceptually to Jai programming language's *builder*, just opens a file and prepares a string buffer to serialize code into (`builder_print`, `builder_print_fmt`). Then write & close the file when completed (`builder_write`).
+    * **`Scanner`**: Interface to load up `Code` from files two basic funcctions are currently provided.
+      * `scan_file`: Used mainly by the library format generators to directly scan files into untyped `Code` (raw string content, pre-formatted no AST parsed).
+      * `parse_file`: Used to read file and then parsed to populate a `CodeBody` AST.
+* **gen_segemetned**: Dependencies go into gen.dep.{hpp/cpp} and components into gen.{hpp/cpp}
+* **gen_singleheader**: Everything into a single file: gen.hpp
+* **gen_unreal_engine**: Like gen_segemented but the library is modified slightly to compile as a thirdparty library within an Unreal Engine plugin or module.
+* **gen_c_library**: The library is heavily modifed into C11 compliant code. A segemented and single-header set of variants are generatd.
+
+
+Code not making up the core library is located in `auxiliary/<auxiliary_name>.<hpp/cpp>`. These are optional extensions or tools for the library.
+
+Feature Macros:
+
+* `GEN_DEFINE_ATTRIBUTE_TOKENS` : Allows user to define their own attribute macros for use in parsing.
+  * This is auto-generated if using the bootstrap or single-header generation
+  * *Note: The user will use the `AttributeTokens.csv` when the library is fully self-hosting.*
+* `GEN_DEFINE_LIBRARY_CORE_CONSTANTS` : Optional typename codes as they are non-standard to C/C++ and not necessary to library usage
+* `GEN_DONT_ENFORCE_GEN_TIME_GUARD` : By default, the library ( gen.hpp/ gen.cpp ) expects the macro `GEN_TIME` to be defined, this disables that.
+* `GEN_ENFORCE_STRONG_CODE_TYPES` : Enforces casts to filtered code types.
+* `GEN_EXPOSE_BACKEND` : Will expose symbols meant for internal use only.
+* `GEN_ROLL_OWN_DEPENDENCIES` : Optional override so that user may define the dependencies themselves.
+* `GEN_DONT_ALLOW_INVALID_CODE` (Not implemented yet) : Will fail when an invalid code is constructed, parsed, or serialized.
+* `GEN_C_LIKE_PP` : Will prevent usage of function defnitions using references and structs with member functions.
+Structs will still have user-defined operator conversions, for-range support, and other operator overloads
+
+*Note: A variant of the C++ library could be generated where those additonal support features are removed (see gen_c_library implementation for an idea of how)*
+
+## On multi-threading
+
+Currently unsupported. I want the library to be *stable* and *correct*, with the addition of exhausting all basic single-threaded optimizations before I consider multi-threading.
+
+## Extending the library
+
+This library is relatively very small (for parsing C++), and can be extended without much hassle.
+
+The convention you'll see used throughout the interface of the library is as follows:
+
+1. Check name or parameters to make sure they are valid for the construction requested
+2. Create a code object using `make_code`.
+3. Populate immediate fields (Name, Type, ModuleFlags, etc)
+4. Populate sub-entires using `add_entry`. If using the default serialization function `to_string`, follow the order at which entires are expected to appear (there is a strong ordering expected).
+
+Names or Content fields are interned strings and thus showed be cached using `get_cached_string` if its desired to preserve that behavior.
+
+`def_operator` is the most sophisticated upfront constructor as it has multiple permutations of definitions that could be created that are not trivial to determine if valid.
+
+The parser is documented under `docs/Parsing.md` and `docs/Parser_Algo.md`. Extending it is more serious, but resolution of a parse for a given internal parse procedure is well documented.
+
+## A note on compilation and runtime generation speed
+
+The library is designed to be fast to compile and generate code at runtime as fast as resonable possible on a debug build.
+Its recommended that your metaprogam be compiled using a single translation unit (unity build).

base/auxillary/builder.cpp

@@ -2,6 +2,8 @@
 #	include "builder.hpp"
 #endif
 
+#pragma region Builder
+
 Builder builder_open( char const* path )
 {
 	Builder result;
@@ -54,3 +56,4 @@ void builder_write(Builder* builder)
 	string_free(& builder->Buffer);
 }
 
+#pragma endregion Builder

base/auxillary/builder.hpp

@@ -1,8 +1,22 @@
 #ifdef GEN_INTELLISENSE_DIRECTIVES
 #	pragma once
-#	include "gen.hpp"
+#	include "helpers/push_ignores.inline.hpp"
+#	include "components/header_start.hpp"
+#	include "components/types.hpp"
+#	include "components/gen/ecode.hpp"
+#	include "components/gen/eoperator.hpp"
+#	include "components/gen/especifier.hpp"
+#	include "components/ast.hpp"
+#	include "components/code_types.hpp"
+#	include "components/ast_types.hpp"
+#	include "components/interface.hpp"
+#	include "components/inlines.hpp"
+#	include "components/gen/ast_inlines.hpp"
+#	include "components/header_end.hpp"
 #endif
 
+#pragma region Builder
+
 struct Builder;
 typedef struct Builder Builder;
 
@@ -51,3 +65,5 @@ void    builder_print_fmt( Builder& builder, char const* fmt, ...) {
 	va_end( va );
 }
 #endif
+
+#pragma endregion Builder

base/auxillary/gen_template.hpp

@@ -1,6 +1,18 @@
 #ifdef GEN_INTELLISENSE_DIRECTIVES
 #	pragma once
-#	include "../gen.hpp"
+#	include "helpers/push_ignores.inline.hpp"
+#	include "components/header_start.hpp"
+#	include "components/types.hpp"
+#	include "components/gen/ecode.hpp"
+#	include "components/gen/eoperator.hpp"
+#	include "components/gen/especifier.hpp"
+#	include "components/ast.hpp"
+#	include "components/code_types.hpp"
+#	include "components/ast_types.hpp"
+#	include "components/interface.hpp"
+#	include "components/inlines.hpp"
+#	include "components/gen/ast_inlines.hpp"
+#	include "components/header_end.hpp"
 #endif
 
 /*

base/auxillary/scanner.hpp

@@ -1,8 +1,22 @@
 #ifdef GEN_INTELLISENSE_DIRECTIVES
 #	pragma once
-#	include "gen.hpp"
+#	include "helpers/push_ignores.inline.hpp"
+#	include "components/header_start.hpp"
+#	include "components/types.hpp"
+#	include "components/gen/ecode.hpp"
+#	include "components/gen/eoperator.hpp"
+#	include "components/gen/especifier.hpp"
+#	include "components/ast.hpp"
+#	include "components/code_types.hpp"
+#	include "components/ast_types.hpp"
+#	include "components/interface.hpp"
+#	include "components/inlines.hpp"
+#	include "components/gen/ast_inlines.hpp"
+#	include "components/header_end.hpp"
 #endif
 
+#pragma region Scanner
+
 // This is a simple file reader that reads the entire file into memory.
 // It has an extra option to skip the first few lines for undesired includes.
 // This is done so that includes can be kept in dependency and component files so that intellisense works.
@@ -119,41 +133,53 @@ Code scan_file( char const* path )
 	return untyped_str( string_to_strc(str) );
 }
 
-#if 0
-struct CodeFile
+CodeBody parse_file( const char* path )
 {
-	using namespace Parser;
+	FileContents file = file_read_contents( GlobalAllocator, true, path );
+	CodeBody     code = parse_global_body( { file.size, (char const*)file.data } );
+	log_fmt("\nParsed: %s\n", path);
+	return code;
+}
 
-	String              FilePath;
-	TokArray            Tokens;
-	Array<ParseFailure> ParseFailures;
-	Code                CodeRoot;
+// The follow is basic support for light csv parsing (use it as an example)
+// Make something robust if its more serious.
+
+typedef struct CSV_Column CSV_Column;
+struct CSV_Column {
+	CSV_Object Owner;
+	Array<ADT_Node> Content;
 };
 
-namespace Parser
-{
-	struct ParseFailure
-	{
-		String Reason;
-		Code   Node;
-	};
-}
+typedef struct CSV_Columns2 CSV_Columns2;
+struct CSV_Columns2 {
+	CSV_Object Owner;
+	Array<ADT_Node> Col_1;
+	Array<ADT_Node> Col_2;
+};
 
-CodeFile scan_file( char const* path )
-{
-	using namespace Parser;
+CSV_Column parse_csv_one_column(AllocatorInfo allocator, char const* path) {
+	char scratch_mem[kilobytes(32)];
+	Arena scratch = arena_init_from_memory( scratch_mem, sizeof(scratch_mem) );
 
-	CodeFile
-	result = {};
-	result.FilePath = String::make( GlobalAllocator, path );
-
-	Code code = scan_file( path );
-	result.CodeRoot = code;
-
-	ParseContext context = parser_get_last_context();
-	result.Tokens        = context.Tokens;
-	result.ParseFailures = context.Failures;
+	file_read_contents( arena_allocator_info( & scratch), file_zero_terminate, path );
 
+	CSV_Column result;
+	csv_parse( & result.owner, scratch_mem, allocator, false );
+	result.Content = csv_nodes.nodes[0].nodes;
 	return result;
 }
-#endif
+
+CSV_Columns2 parse_csv_two_columns(AllocatorInfo allocator, char const* path) {
+	char scratch_mem[kilobytes(32)];
+	Arena scratch = arena_init_from_memory( scratch_mem, sizeof(scratch_mem) );
+
+	file_read_contents( arena_allocator_info( & scratch), file_zero_terminate, path );
+
+	CSV_Columns2 result;
+	csv_parse( & result.owner, scratch_mem, allocator, false );
+	result.Col_1 = csv_nodes.nodes[0].nodes;
+	result.Col_2 = csv_nodes.nodes[1].nodes;
+	return result;
+}
+
+#pragma endregion Scanner

base/base.cpp

@@ -0,0 +1,56 @@
+#define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
+#define GEN_ENFORCE_STRONG_CODE_TYPES
+#define GEN_EXPOSE_BACKEND
+#define GEN_C_LIKE_CPP 1
+#include "../project/gen.cpp"
+
+#include "helpers/push_ignores.inline.hpp"
+#include "helpers/helper.hpp"
+
+GEN_NS_BEGIN
+#include "helpers/push_container_defines.inline.hpp"
+#include "dependencies/parsing.cpp"
+#include "helpers/pop_container_defines.inline.hpp"
+GEN_NS_END
+
+#include "auxillary/builder.hpp"
+#include "auxillary/builder.cpp"
+#include "auxillary/scanner.hpp"
+#include "auxillary/misc.hpp"
+
+using namespace gen;
+
+constexpr char const* path_format_style = "../scripts/.clang-format";
+constexpr char const* scratch_file      = "gen/scratch.hpp";
+
+Code format( Code code ) {
+	return code_refactor_and_format(code, scratch_file, nullptr, path_format_style );
+}
+
+int gen_main()
+{
+	CodeBody ecode       = gen_ecode     ( "enums/ECodeTypes.csv" );
+	CodeBody eoperator   = gen_eoperator ( "enums/EOperator.csv" );
+	CodeBody especifier  = gen_especifier( "enums/ESpecifier.csv" );
+	CodeBody ast_inlines = gen_ast_inlines();
+
+	Builder header_ecode = builder_open( "components/gen/ecode.hpp" );
+	builder_print( & header_ecode, gen_component_header );
+	builder_print( & header_ecode, format(ecode) );
+	builder_write( & header_ecode);
+
+	Builder header_eoperator = builder_open( "components/gen/eoperator.hpp" );
+	builder_print( & header_eoperator, gen_component_header );
+	builder_print( & header_eoperator, format(eoperator) );
+	builder_write( & header_eoperator );
+
+	Builder header_especifier = builder_open( "components/gen/especifier.hpp" );
+	builder_print( & header_especifier, gen_component_header );
+	builder_print( & header_especifier, format(especifier) );
+	builder_write( & header_especifier);
+
+	Builder header_ast_inlines = builder_open( "components/gen/ast_inlines.hpp" );
+	builder_print( & header_ast_inlines, gen_component_header );
+	builder_print( & header_ast_inlines, format(ast_inlines) );
+	builder_write( & header_ast_inlines);
+}