Setup testing and library for getting the parse api done.

project/gen.cpp

@@ -1,8 +1,10 @@
 // ReSharper disable CppClangTidyClangDiagnosticSwitchEnum
+
+#ifdef gen_time
+
 #include "Bloat.hpp"
 #include "gen.hpp"
 
-#ifdef gen_time
 namespace gen
 {
 	ZPL_TABLE_DEFINE( StringTable, str_tbl_, String );
@@ -244,7 +246,7 @@ namespace gen
 
 	String AST::to_string()
 	{
-#	define ProcessModuleFlags() \
+	#	define ProcessModuleFlags() \
 		if ( bitfield_is_equal( u32, ModuleFlags, ModuleFlag::Export ))  \
 			result.append( "export " );             \
                                                                          \
@@ -992,10 +994,6 @@ namespace gen
 			str_tbl_init ( & StringMap, Allocator_StringTable );
 			if ( StringMap.entries == nullptr )
 				fatal( "gen::init: Failed to initialize the StringMap");
-
-			//type_tbl_init( & TypeMap,   Allocator_TypeTable );
-			//if ( TypeMap.entries == nullptr )
-			//	fatal( "gen::init: Failed to initialize the TypeMap" );
 		}
 
 		Code::Global          = make_code();
@@ -2886,6 +2884,35 @@ namespace gen
 		return result;
 	}
 
+	Code def_specifiers( s32 num, SpecifierT* specs )
+	{
+		if ( num <= 0 )
+		{
+			log_failure("gen::def_specifiers: num cannot be zero or less");
+			return Code::Invalid;
+		}
+
+		if ( num > AST::ArrSpecs_Cap )
+		{
+			log_failure("gen::def_specifiers: num of speciifers to define AST larger than AST specicifier capacity - %d", num);
+			return Code::Invalid;
+		}
+
+		Code
+		result       = make_code();
+		result->Type = ECode::Specifiers;
+
+		s32 idx = 0;
+		do
+		{
+			result->add_specifier( specs[idx] );
+			idx++;
+		}
+		while ( --num, num );
+
+		return result;
+	}
+
 	Code def_struct_body( s32 num, ... )
 	{
 		def_body_start( def_struct_body );
@@ -3027,6 +3054,7 @@ namespace gen
 		Entry( Comma,               "," )               \
 		Entry( Decl_Class,          "class" )           \
 		Entry( Decl_Enum,           "enum" )            \
+		Entry( Decl_Extern_Linkage, "extern" )          \
 		Entry( Decl_Friend,         "friend" )          \
 		Entry( Decl_Module,         "module" )          \
 		Entry( Decl_Namespace,      "namespace" )       \
@@ -3040,13 +3068,14 @@ namespace gen
 		Entry( Number,              "number" )          \
 		Entry( Operator,            "operator" )        \
 		Entry( Spec_Alignas,        "alignas" )         \
-		Entry( Spec_CLinkage,       "extern \"C\"" )    \
 		Entry( Spec_Const,          "const" )           \
 		Entry( Spec_Consteval,      "consteval" )       \
 		Entry( Spec_Constexpr,      "constexpr" )       \
 		Entry( Spec_Constinit,      "constinit" )       \
 		Entry( Spec_Extern,         "extern" )          \
 		Entry( Spec_Inline,         "inline" )          \
+		Entry( Spec_LocalPersist,   "local_persist" )   \
+		Entry( Spec_Mutable,        "mutable" )         \
 		Entry( Spec_Static,         "static" )          \
 		Entry( Spec_ThreadLocal,    "thread_local" )    \
 		Entry( Spec_Volatile,       "volatile")         \
@@ -3130,8 +3159,7 @@ namespace gen
 		inline
 		bool tok_is_specifier( Token const& tok )
 		{
-			return tok.Type <= TokType::Star
+			return (tok.Type <= TokType::Star && tok.Type >= TokType::Spec_Alignas)
-				|| tok.Type >= TokType::Spec_Alignas
 				|| tok.Type == TokType::Ampersand
 				|| tok.Type == TokType::Ampersand_DBL
 			;
@@ -3175,6 +3203,7 @@ namespace gen
 				}
 
 				Idx++;
+				return true;
 			}
 		};
 
@@ -3790,14 +3819,11 @@ namespace gen
 			if ( value )
 				param->add_entry( value );
 
-			param.lock();
-
 			result->add_entry( param );
 			eat( TokType::Comma );
 		}
 		eat( TokType::Capture_End );
 
-		result.lock();
 		return result;
 	#	undef context
 	}
@@ -3911,6 +3937,7 @@ namespace gen
 				case TokType::Spec_Static:
 				case TokType::Spec_ThreadLocal:
 				case TokType::Spec_Volatile:
+				{
 					Code specifiers = Code::Invalid;
 
 					SpecifierT specs_found[16] { ESpecifier::Num_Specifiers };
@@ -3946,7 +3973,7 @@ namespace gen
 					{
 						specifiers = def_specifiers( num_specifiers, specs_found );
 					}
-
+				}
 				case TokType::Identifier:
 				case TokType::Spec_Const:
 				case TokType::Type_Unsigned:
@@ -3954,6 +3981,8 @@ namespace gen
 				case TokType::Type_Short:
 				case TokType::Type_Long:
 				{
+					Code specifiers = Code::Invalid;
+
 					Code type = parse_type( toks, context );
 					if ( type == Code::Invalid )
 						return Code::Invalid;
@@ -4203,7 +4232,7 @@ namespace gen
 
 		// Parse alignment
 
-		Token name = parse_identifier( toks, context );
+		name = parse_identifier( toks, context );
 
 		if ( check( TokType::Assign_Classifer ) )
 		{
@@ -4324,7 +4353,6 @@ namespace gen
 		if ( type )
 			result->add_entry( type );
 
-		result.lock();
 		return result;
 	}
 
@@ -4394,8 +4422,6 @@ namespace gen
 
 			if ( params )
 				function->add_entry( params );
-
-			function.lock();
 		}
 
 		eat( TokType::Statement_End );
@@ -4410,7 +4436,6 @@ namespace gen
 		else
 			result->add_entry( type );
 
-		result.lock();
 		return result;
 	}
 
@@ -4548,7 +4573,6 @@ namespace gen
 		if ( params )
 			result->add_entry( params );
 
-		result.lock();
 		return result;
 	}
 
@@ -4588,7 +4612,6 @@ namespace gen
 
 		// }
 
-		result.lock();
 		return result;
 	}
 
@@ -4715,7 +4738,7 @@ namespace gen
 
 		if (num_specifiers)
 		{
-			Code specifiers = def_specifiers( num_specifiers, specs_found );
+			Code specifiers = def_specifiers( num_specifiers, (SpecifierT*)specs_found );
 
 			result->add_entry( specifiers );
 		}
@@ -4734,7 +4757,6 @@ namespace gen
 
 		Code result = parse_type( toks, txt(parse_type) );
 
-		result.lock();
 		return result;
 	}
 
@@ -4775,7 +4797,6 @@ namespace gen
 		if ( array_expr )
 			type->add_entry( array_expr );
 
-		result.lock();
 		return result;
 	}
 
@@ -4853,7 +4874,6 @@ namespace gen
 		if ( array_expr )
 			type->add_entry( array_expr );
 
-		result.lock();
 		return result;
 	}
 
@@ -5013,7 +5033,6 @@ namespace gen
 		if ( expr )
 			result->add_entry( expr );
 
-		result.lock();
 		return result;
 	}
 
@@ -5139,7 +5158,6 @@ namespace gen
 		tokmap_clear( & tok_map );
 
 		sw result = buf_size - remaining;
-		// buf[ result ] = '\0';
 
 		return result;
 	}
@@ -5236,7 +5254,6 @@ namespace gen
 		if ( result == false )
 			log_failure("gen::File::write - Failed to write to file: %s", file_name( & File ) );
 
-		// file_seek( & File, 0 );
 		file_close( & File );
 	}
 #pragma endregion Builder

project/gen.hpp

@@ -8,18 +8,18 @@
 */
 #pragma once
 
+#ifdef gen_time
+
 #include "Bloat.hpp"
 
 // Temporarily here for debugging purposes.
 #define GEN_DEFINE_LIBRARY_CODE_CONSTANTS
 // #define GEN_DONT_USE_FATAL
 
-// #define GEN_FEATURE_PARSING
+#define GEN_FEATURE_PARSING
 // #define GEN_FEATURE_EDITOR
 // #define GEN_FEATURE_SCANNER
 
-
-#ifdef gen_time
 namespace gen
 {
 	using LogFailType = sw(*)(char const*, ...);
@@ -583,11 +583,6 @@ namespace gen
 
 	/*
 		AST* typedef as to not constantly have to add the '*' as this is written often..
-
-		If GEN_ENFORCE_READONLY_AST is defined, readonly assertions will be done on any member dreference,
-		and the 'gen API' related functions. will set their created ASTs to readonly before returning.
-
-		Casting to AST* will bypass.
 	*/
 	struct Code
 	{
@@ -615,14 +610,6 @@ namespace gen
 				return Invalid;
 			}
 
-		#ifdef GEN_ENFORCE_READONLY_AST
-			if ( ast->Readonly )
-			{
-				log_failure("Attempted to a body AST from a readonly AST!");
-				return Invalid;
-			}
-		#endif
-
 			return * (Code*)( ast->body() );
 		}
 

scripts/build.ci.ps1

@@ -49,7 +49,8 @@ $path_scripts = Join-Path $path_root scripts
 	& ninja $args_ninja
 	Pop-Location
 
-	if ($true) {
+	# Test NonParsed
+	if ($false) {
 	$gencpp = Join-Path $path_gen_build gencpp.exe
 
 	Push-location $path_gen
@@ -74,6 +75,32 @@ $path_scripts = Join-Path $path_root scripts
 	Pop-Location
 	}
 
+	# Test Parsed
+	if ($true) {
+	$gencpp = Join-Path $path_gen_build gencpp_parsed.exe
+
+	Push-location $path_gen
+
+	Write-Host `nGenerating files -- using Parse API...
+	& $gencpp
+
+	Write-Host `nBeginning format...
+	$formatParams = @(
+		'-i'          # In-place
+		'-style=file' # Search for a .clang-format file in the parent directory of the source file.
+		'-verbose'
+	)
+
+	$include = @('*.gen.hpp', '*.gen.cpp')
+	$exclude = $null
+
+	$targetFiles = @(Get-ChildItem -Recurse -Path $path_gen -Include $include -Exclude $exclude | Select-Object -ExpandProperty FullName)
+
+	clang-format $formatParams $targetFiles
+	Write-Host "`nFormatting complete"
+	Pop-Location
+	}
+
 
 	# Build the program depending on generated files.
 	# if ( -not( Test-Path $path_test_build ) )k

test/Parsed/Array.Parsed.hpp

@@ -7,206 +7,251 @@ using namespace gen;
 
 Code gen__array_base()
 {
-	Code array_base = parse_struct( code(
+	return parse_global_body( code(
-		struct ArrayBase
+		struct ArrayHeader
 		{
-			struct Header
+			AllocatorInfo Allocator;
-			{
+			uw            Capacity;
-				AllocatorInfo Allocator;
+			uw            Num;
-				uw            Capacity;
-				uw            Num;
-			};
-
-			static inline
-			sw grow_formula( sw value )
-			{
-				return 2 * value * 8;
-			}
 		};
-	));
 
-	return array_base;
+		static inline uw array_grow_formula( uw value )
+		{
+			return 2 * value * 8;
+		}
+	));
 }
 
-Code gen__array( s32 length, char const* type_str, sw type_size )
+Code gen__array( StrC type, sw type_size )
 {
-	StrC tmpl = code(
+	StrC name;
-		struct Array_{type} : ArrayBase
+	{
-		{
+		char const* name_str = str_fmt_buf( "Array_%s\0", type.Ptr );
-			using Type = {type};
+		s32         name_len = str_len( name_str );
 
-			Type* Data;
+		name = { name_len, name_str };
+	};
 
-			static Array_{type} init( AllocatorInfo allocator )
+	Code array = parse_struct( token_fmt(
+		txt(
+			struct <ArrayType>
 			{
-				return init_reserve( allocator, grow_formula(0) );
+				using Header = ArrayHeader;
-			}
+				using Type   = <type>;
 
-			static Array_{type} init_reserve( AllocatorInfo allocator, uw capacity )
+				constexpr auto grow_formula = &array_glow_formula;
-			{
-				Header* header = rcast( Header*, alloc( allocator, sizeof(Header) + sizeof(Type) ));
 
-				if ( header == nullptr )
+				static
-					return false;
+				<ArrayType> init( AllocatorInfo allocator )
-
-				header->Allocator = allocator;
-				header->Capacity  = capacity;
-				header->Num       = 0;
-
-				Array_{type} array;
-				array.Data = rcast( Type*, header + 1 );
-
-				return array;
-			}
-
-			bool append( Type const& value )
-			{
-				Header& header = get_header();
-
-				if ( header.Num == header.Capacity )
 				{
-					if ( ! grow( header.Allocator ))
+					return init_reserve( allocator, grow_formula(0) );
-						return false;
 				}
 
-				data[ header.Num ] = value;
+				static
-				header.Num++;
+				<ArrayType> init_reserve( AllocatorInfo allocator, sw capacity )
+				{
+					Header* header = rcast( Header*, alloc( allocator, sizeof(Header) + sizeof(Type) ));
 
-				return true;
+					if ( header == nullptr )
-			}
+						return { nullptr };
 
-			Type& back()
+					header->Allocator = allocator;
-			{
+					header->Capacity  = capacity;
-				Header& header = get_header();
+					header->Num       = 0;
-				return data[ header.Num - 1 ];
-			}
 
-			void clear()
+					return { rcast( Type*, header + 1) };
-			{
+				}
-				Header& header = get_header();
-				header.Num = 0;
-			}
 
-			bool fill( uw begin, uw end, Type const& value )
+				bool append( Type value )
-			{
+				{
-				Header& header = get_header();
+					Header& header = get_header();
 
-				if ( begin < 0 || end >= header.Num )
+					if ( header.Num == header.Capacity )
-					return false;
+					{
+						if ( ! grow( header.Capacity ))
+							return false;
+					}
 
-				for ( uw idx = begin; idx < end; idx++ )
+					Data[ header.Num ] = value;
-					data[ idx ] = value;
+					header.Num++;
 
-				return true;
+					return true;
-			}
+				}
 
-			void free()
+				inline Type& back( void )
-			{
+				{
-				Header& header = get_header();
+					Header& header = get_header();
-				::free( header.Allocator, & header );
+					return Data[ header.Num - 1 ];
-			}
+				}
 
-			Header& get_header()
+				inline void clear( void )
-			{
+				{
-				return pcast( Header, Data - 1 );
+					Header& header = get_header();
-			}
+					header.Num     = 0;
+				}
 
-			bool grow( uw min_capacity )
+				bool fill( uw begin, uw end, Type value )
-			{
+				{
-				Header& header = get_header();
+					Header& header = get_header();
 
-				uw new_capacity = grow_formula( header.Capacity );
+					if ( begin < 0 || end >= header.Num )
-
-				if ( new_capacity < min_capacity )
-					new_capacity = min_capacity;
-
-				return set_capacity( new_capacity );
-			}
-
-			void pop()
-			{
-				Header& header = get_header();
-
-				assert_crash( header.Num > 0 );
-				header.Num--;
-			}
-
-			bool reserve( uw new_capacity )
-			{
-				Header& header = get_header();
-
-				if ( header.Capacity < new_capacity )
-					return set_capacity( new_capacity );
-
-				return true;
-			}
-
-			bool resize( uw num )
-			{
-				Header& header = get_header();
-
-				if ( header.Capacity < num )
-					if ( ! grow( num ))
 						return false;
 
-				header.Num = num;
+					for ( sw idx = begin; idx < end; idx++ )
-				return true;
+					{
-			}
+						Data[ idx ] = value;
+					}
 
-			bool set_capacity( uw new_capacity )
-			{
-				Header& header = get_header();
-
-				if ( new_capacity == header.Capacity )
 					return true;
+				}
 
-				if ( new_capacity < header.Num )
+				inline void free( void )
-					header.Num = new_capacity;
+				{
+					Header& header = get_header();
+					zpl::free( header.Allocator, &header );
+				}
 
-				sw      size       = sizeof(Header) + sizeof(Type) * new_capacity;
+				inline Header& get_header( void )
-				Header* new_header = rcast( Header*, alloc( header.Allocator, size ));
+				{
+					return *( reinterpret_cast< Header* >( Data ) - 1 );
+				}
 
-				if ( new_header == nullptr )
+				bool grow( uw min_capacity )
-					return false;
+				{
+					Header& header       = get_header();
+					uw      new_capacity = grow_formula( header.Capacity );
 
-				mem_move( new_header, & header, sizeof( Header ) + sizeof(Type) * header.Num );
+					if ( new_capacity < min_capacity )
+						new_capacity = 8;
 
-				new_header->Allocator = header.Allocator;
+					return set_capacity( new_capacity );
-				new_header->Num       = header.Num;
+				}
-				new_header->Capacity  = new_capacity;
 
-				::free( header );
+				inline uw num( void )
+				{
+					return get_header().Num;
+				}
 
-				*Data = new_header + 1;
+				inline bool pop( void )
+				{
+					Header& header = get_header();
 
-				return true;
+					ZPL_ASSERT( header.Num > 0 );
-			}
+					header.Num--;
+				}
+
+				inline void remove_at( uw idx )
+				{
+					Header* header = &get_header();
+					ZPL_ASSERT( idx < header->Num );
+
+					mem_move( header + idx, header + idx + 1, sizeof( Type ) * ( header->Num - idx - 1 ) );
+					header->Num--;
+				}
+
+				bool reserve( uw new_capacity )
+				{
+					Header& header = get_header();
+
+					if ( header.Capacity < new_capacity )
+						return set_capacity( new_capacity );
+
+					return true;
+				}
+
+				bool resize( uw num )
+				{
+					Header& header = get_header();
+
+					if ( num > header.Capacity )
+					{
+						if ( ! grow( header.Capacity ) )
+							return false;
+					}
+
+					header.Num = num;
+					return true;
+				}
+
+				bool set_capacity( uw new_capacity )
+				{
+					Header& header = get_header();
+
+					if ( new_capacity == header.Capacity )
+						return true;
+
+					if ( new_capacity < header.Num )
+						header.Num = new_capacity;
+
+					sw      size       = sizeof( Header ) + sizeof( Type ) * new_capacity;
+					Header* new_header = reinterpret_cast< Header* >( alloc( header.Allocator, size ) );
+
+					if ( new_header == nullptr )
+						return false;
+
+					mem_move( new_header, &header, sizeof( Header ) + sizeof( Type ) * header.Num );
+
+					new_header->Allocator = header.Allocator;
+					new_header->Num       = header.Num;
+					new_header->Capacity  = new_capacity;
+
+					zpl::free( header.Allocator, &header );
+
+					Data = ( Type* )new_header + 1;
+					return true;
+				}
+
+				Type* Data;
+			};
+		),
+		// Tokens
+			2
+			, "ArrayType", name
+			, "type",      type
+	));
+
+	Code op_ptr = untyped_str( code(
+		operator Type*()
+		{
+			return Data;
 		}
-	);
+	));
+	array.body()->add_entry( op_ptr );
 
-	char const* gen_from_tmpl     = token_fmt( tmpl.Ptr, 1, "type", type_str );
+	return array;
-	s32         gen_from_tmpl_len = str_len( gen_from_tmpl );
-
-	Code array = parse_struct( { gen_from_tmpl_len, gen_from_tmpl } );
 }
 
 
 struct GenArrayRequest
 {
-	s32         TypeLength;
+	StrC Dependency;
-	char const* Type;
+	StrC Type;
-	sw          Size;
+	sw   Size;
-	s32         DependencyLength;
-	char const* Dependency;
 };
 Array(GenArrayRequest) GenArrayRequests;
 
-void gen__array_request( s32 type_len, char const* type_str, sw type_size, s32 dep_len, char const* dep )
+void gen__array_request( StrC type, sw size, StrC dep = {} )
 {
-	GenArrayRequest request = { type_len, type_str, type_size, dep_len, dep };
+	do_once_start
+		array_init( GenArrayRequests, g_allocator );
+	do_once_end
+
+	// Make sure we don't already have a request for the type.
+	for ( sw idx = 0; idx < array_count( GenArrayRequests ); ++idx )
+	{
+		StrC const reqest_type = GenArrayRequests[ idx ].Type;
+
+		if ( reqest_type.Len != type.Len )
+			continue;
+
+		if ( str_compare( reqest_type.Ptr, type.Ptr, reqest_type.Len ) == 0 )
+			return;
+	}
+
+	GenArrayRequest request = { dep, type, size };
 	array_append( GenArrayRequests, request );
 }
-#define Gen_Array( type ) gen__array_request( txt_n_len( type ), sizeof(type) )
+#define gen_array( type ) gen__array_request( { txt_n_len(type) }, sizeof(type) )
 
 u32 gen_array_file()
 {
@@ -214,6 +259,11 @@ u32 gen_array_file()
 	gen_array_file;
 	gen_array_file.open( "array.gen.hpp" );
 
+	Code include_zpl = def_include( StrC::from("Bloat.hpp") );
+	gen_array_file.print( include_zpl );
+
+	Code array_base = gen__array_base();
+	gen_array_file.print( array_base );
 
 	GenArrayRequest* current = GenArrayRequests;
 	s32 left = array_count( GenArrayRequests );
@@ -221,7 +271,7 @@ u32 gen_array_file()
 	{
 		GenArrayRequest const& request = * current;
 
-		Code generated_array = gen__array( request.TypeLength, request.Type, request.Size );
+		Code generated_array = gen__array( request.Type, request.Size );
 
 		if ( request.Dependency )
 		{
@@ -229,7 +279,7 @@ u32 gen_array_file()
 			s32         cmt_len = str_len( cmt_str );
 
 			Code cmt     = def_comment( { cmt_len, cmt_str } );
-			Code include = def_include( { request.DependencyLength, request.Dependency } );
+			Code include = def_include( request.Dependency );
 
 			gen_array_file.print( cmt );
 			gen_array_file.print( include );

test/Parsed/Sanity.hpp

@@ -0,0 +1,29 @@
+#pragma once
+#ifdef gen_time
+#include "gen.hpp"
+
+using namespace gen;
+
+void gen_sanity()
+{
+	Builder
+	gen_sanity_file;
+	gen_sanity_file.open("./sanity.gen.hpp");
+
+	gen_sanity_file.print( def_comment( StrC::from(
+		"The following will show a series of base cases for the gen parsed api.\n"
+	)));
+
+	// Typedef
+	{
+		Code u8_typedef = parse_typedef( code(
+			typedef unsigned char u8;
+		));
+
+		gen_sanity_file.print(u8_typedef);
+	}
+
+	gen_sanity_file.write();
+}
+#endif
+

test/gen/meson.build

@@ -11,7 +11,8 @@ includes = include_directories(
 # get_sources = files('./get_sources.ps1')
 # sources     = files(run_command('powershell', get_sources, check: true).stdout().strip().split('\n'))
 
-sources = [ '../test.cpp' ]
+sources = [ '../test.NonParsed.cpp' ]
+sources_parsed = [ '../test.Parsed.cpp' ]
 
 if get_option('buildtype').startswith('debug')
 
@@ -24,4 +25,5 @@ endif
 
 add_project_arguments('-Dgen_time', language : ['c', 'cpp'])
 
-executable( 'gencpp', sources, include_directories : includes )
+# executable( 'gencpp', sources, include_directories : includes )
+executable( 'gencpp_parsed', sources_parsed, include_directories : includes )

test/test.Parsed.cpp

@@ -0,0 +1,48 @@
+#include "Bloat.cpp"
+// #include "Parsed\Array.Parsed.hpp"
+// #include "Parsed\Buffer.Parsed.hpp"
+// #include "Parsed\HashTable.Parsed.hpp"
+// #include "Parsed\Ring.Parsed.hpp"
+#include "Parsed\Sanity.hpp"
+
+
+#ifdef gen_time
+#include "gen.cpp"
+
+using namespace gen;
+
+
+int gen_main()
+{
+	Memory::setup();
+	gen::init();
+
+	gen_sanity();
+
+	// gen_array( u8 );
+	// gen_array( sw );
+
+	// gen_buffer( u8 );
+
+	// gen_hashtable( u32 );
+
+	// gen_ring( s16 );
+
+	// gen_array_file();
+	// gen_buffer_file();
+	// gen_hashtable_file();
+	// gen_ring_file();
+
+	gen::deinit();
+	Memory::cleanup();
+	return 0;
+}
+#endif
+
+
+#ifdef runtime
+int main()
+{
+	return 0;
+}
+#endif