Nativs support for gen::AST, fixes & update to library, getting gen_array working...

token_fmt returns StrC now. Did some fixes so it can properly function now!

Lots of changes to the Array non-parsed implementation.
It shows no errors in the edtitor static analysis, however I haven't tried it yet.
Unfortunately have some friction with how I would like to set it up like with everything in C++...
Can't inherit from ArrayBase to provide the common members to all Array defs, without introducing constructor complexity.

I may be open to using constructors in the future, depends on how I feel when trying out the context pattern used in jai and odin...

.vscode/gencpp.natvis

@@ -1,6 +1,14 @@
 <?xml version="1.0" encoding="utf-8"?>
 <AutoVisualizer xmlns="http://schemas.microsoft.com/vstudio/debugger/natvis/2010">
 
+	<Type Name="gen::Code">
+		<DisplayString>{ast.Name} {ast.Type}</DisplayString>
+	</Type>
+
+	<Type Name="gen::AST">
+		<DisplayString>{Name} {Type}</DisplayString>
+	</Type>
+
 	<Type Name="String">
 		<DisplayString Condition="Data == nullptr">null</DisplayString>
 		<DisplayString>{Data,na}</DisplayString>

project/gen.cpp

@@ -180,6 +180,12 @@ namespace gen
 				result->Content = Content;
 				return result;
 
+			case Specifiers:
+				// Need to memcpy the specifiers.
+				mem_copy( result->ArrSpecs, ArrSpecs, sizeof( ArrSpecs ) );
+				result->StaticIndex = StaticIndex;
+				return result;
+
 			// The main purpose of this is to make sure entires in the AST are unique,
 			// So that we can assign the new parent without corrupting the existing AST.
 			case Class:
@@ -206,7 +212,6 @@ namespace gen
 			case Operator_Member:
 			case Operator_Member_Fwd:
 			case Parameters:
-			case Specifiers:
 			case Struct:
 			case Struct_Fwd:
 			case Struct_Body:
@@ -397,15 +402,15 @@ namespace gen
 				{
 					s32 idx = 1;
 
-					AST const* Entry = entry( idx);
+					AST const* current = entry( idx);
 
-					if ( Entry->Type == Attributes )
+					if ( current->Type == Attributes )
 					{
 						result.append_fmt( "%s ", entry( idx )->to_string() );
 						idx++;
 					}
 
-					if ( Entry->Type == Typename)
+					if ( current->Type == Typename)
 					{
 						result.append_fmt( "%s : %s\n{\n"
 							, Name
@@ -476,7 +481,7 @@ namespace gen
 					}
 				}
 
-				result.append_fmt( "%s};"
+				result.append_fmt( "%s\n};"
 					, body()->to_string()
 				);
 			}
@@ -548,40 +553,40 @@ namespace gen
 				u32 idx  = 1;
 				u32 left = num_entries();
 
-				AST* Entry = entry( idx );
+				AST* current = entry( idx );
 
-				if ( Entry && Entry->Type == Attributes )
+				if ( current && current->Type == Attributes )
 				{
-					result.append_fmt( "%s ", Entry->to_string() );
+					result.append_fmt( "%s ", current->to_string() );
 					idx++;
 					left--;
-					Entry = entry( idx );
+					current = entry( idx );
 				}
 
-				if ( Entry && Entry->Type == Specifiers )
+				if ( current && current->Type == Specifiers )
 				{
-					result.append_fmt( "%s\n", Entry->to_string() );
+					result.append_fmt( "%s\n", current->to_string() );
 					idx++;
 					left--;
-					Entry = entry( idx );
+					current = entry( idx );
 				}
 
-				if ( Entry && Entry->Type == Typename )
+				if ( current && current->Type == Typename )
 				{
-					result.append_fmt( "%s ", Entry->to_string() );
+					result.append_fmt( "%s ", current->to_string() );
 					idx++;
 					left--;
-					Entry = entry( idx );
+					current = entry( idx );
 				}
 
 				result.append_fmt( "%s(", Name );
 
-				if ( left && Entry && Entry->Type == Parameters )
+				if ( left && current && current->Type == Parameters )
 				{
-					result.append_fmt("%s", Entry->to_string() );
+					result.append_fmt("%s", current->to_string() );
 					idx++;
 					left--;
-					Entry = entry( idx );
+					current = entry( idx );
 				}
 				else
 				{
@@ -605,40 +610,40 @@ namespace gen
 				u32 idx  = 0;
 				u32 left = num_entries();
 
-				AST* Entry = entry( idx );
+				AST* current = entry( idx );
 
-				if ( Entry && Entry->Type == Attributes )
+				if ( current && current->Type == Attributes )
 				{
-					result.append_fmt( "%s ", Entry->to_string() );
+					result.append_fmt( "%s ", current->to_string() );
 					idx++;
 					left--;
-					Entry = entry( idx );
+					current = entry( idx );
 				}
 
-				if ( Entry && Entry->Type == Specifiers )
+				if ( current && current->Type == Specifiers )
 				{
-					result.append_fmt( "%s\n", Entry->to_string() );
+					result.append_fmt( "%s\n", current->to_string() );
 					idx++;
 					left--;
-					Entry = entry( idx );
+					current = entry( idx );
 				}
 
-				if ( Entry && Entry->Type == Typename )
+				if ( current && current->Type == Typename )
 				{
-					result.append_fmt( "%s ", Entry->to_string() );
+					result.append_fmt( "%s ", current->to_string() );
 					idx++;
 					left--;
-					Entry = entry( idx );
+					current = entry( idx );
 				}
 
 				result.append_fmt( "%s(", Name );
 
-				if ( left && Entry && Entry->Type == Parameters )
+				if ( left && current && current->Type == Parameters )
 				{
-					result.append_fmt("%s", Entry->to_string() );
+					result.append_fmt("%s", current->to_string() );
 					idx++;
 					left--;
-					Entry = entry( idx );
+					current = entry( idx );
 				}
 				else
 				{
@@ -750,13 +755,16 @@ namespace gen
 				result.append_fmt( "%s %s", entry( 0 )->to_string(), Name );
 
 				s32 index = 1;
-				s32 left  = num_entries();
+				s32 left  = num_entries() - 1;
 
-				while ( left--, left > 0 )
+				while ( left-- )
+				{
 					result.append_fmt( ", %s %s"
 						, entry( index )->entry( 0 )->to_string()
 						, entry( index )->Name
 					);
+					index++;
+				}
 			}
 			break;
 
@@ -768,8 +776,11 @@ namespace gen
 			{
 				s32 idx  = 0;
 				s32 left = StaticIndex;
-				while ( left--, left > 0 )
+				while ( left-- )
-					result.append_fmt( "%s ", ESpecifier::to_str( ArrSpecs[idx]) );
+				{
+					result.append_fmt( "%s ", (char const*)ESpecifier::to_str( ArrSpecs[idx]) );
+					idx++;
+				}
 			}
 			break;
 
@@ -781,24 +792,43 @@ namespace gen
 
 				result.append( "struct ");
 
-				s32 idx = 1;
+				if ( num_entries() > 1 )
-
-				if ( entry( idx )->Type == Attributes )
 				{
-					result.append_fmt( "%s ", entry( idx )->to_string() );
+					s32 idx = 1;
-					idx++;
+
+					AST* current = entry( idx );
+
+					if ( current && current->Type == Attributes )
+					{
+						result.append_fmt( "%s ", entry( idx )->to_string() );
+						idx++;
+						current = entry( idx );
+					}
+
+					result.append_fmt( "%s", Name );
+
+					if ( current )
+					{
+						switch ( ParentAccess )
+						{
+							case AccessSpec::Private:
+							case AccessSpec::Protected:
+							case AccessSpec::Public:
+								result.append_fmt( " : %s %s", to_str(ParentAccess), current->to_string() );
+								idx++;
+								break;
+
+							default:
+								result.append_fmt( " : %s", current->to_string() );
+								break;
+						}
+					}
+
+					result.append_fmt( "\n{\n%s\n};", body()->to_string() );
+					break;
 				}
 
-				result.append_fmt( "%s ", Name );
+				result.append_fmt( "%s\n{\n%s\n};", Name, body()->to_string() );
-
-				if ( entry( idx ) )
-				{
-					char const* access_str = to_str( ParentAccess );
-
-					result.append_fmt( ": %s %s", access_str, entry( idx )->to_string() );
-				}
-
-				result.append_fmt( "\n{\n%s\n};", body()->to_string() );
 			}
 			break;
 
@@ -2253,6 +2283,18 @@ namespace gen
 		return result;
 	}
 
+	Code def_specifier( SpecifierT spec )
+	{
+		Code
+		result       = make_code();
+		result->Type = ECode::Specifiers;
+
+		result->add_specifier( spec );
+
+		result.lock();
+		return result;
+	}
+
 	Code def_struct( StrC name
 		, Code body
 		, Code parent, AccessSpec parent_access
@@ -2269,7 +2311,7 @@ namespace gen
 			return Code::Invalid;
 		}
 
-		if ( parent && parent->Type != Struct )
+		if ( parent && parent->Type != Typename )
 		{
 			log_failure( "gen::def_struct: parent was not a `Struct` type" );
 			return Code::Invalid;
@@ -5184,56 +5226,70 @@ namespace gen
 			}
 		}
 
-		sw   result  = 0;
 		char current = *fmt;
 
 		while ( current )
 		{
 			sw len = 0;
 
-			while ( current && current != '{' && remaining )
+			while ( current && current != '<' && remaining )
 			{
 				* buf = * fmt;
 				buf++;
 				fmt++;
+				remaining--;
 
 				current = * fmt;
 			}
 
-			if ( current == '{' )
+			if ( current == '<' )
 			{
-				char const* scanner = fmt;
+				char const* scanner = fmt + 1;
 
 				s32 tok_len = 0;
 
-				while ( *scanner != '}' )
+				while ( *scanner != '>' )
 				{
 					tok_len++;
 					scanner++;
 				}
 
-				char const* token = fmt;
+				char const* token = fmt + 1;
 
-				u32      key   = crc32( token, tok_len );
+				u32       key   = crc32( token, tok_len );
-				TokEntry value = * tokmap_get( & tok_map, key );
+				TokEntry* value = tokmap_get( & tok_map, key );
-				sw       left  = value.Length;
 
-				while ( left-- )
+				if ( value )
 				{
-					* buf = *value.Str;
+					sw          left = value->Length;
-					buf++;
+					char const* str  = value->Str;
-					value.Str++;
+
+					while ( left-- )
+					{
+						* buf = * str;
+						buf++;
+						remaining--;
+						str++;
+					}
+
+					scanner++;
+					fmt     = scanner;
+					current = * fmt;
+					continue;
 				}
 
-				scanner++;
+				* buf = * fmt;
-				fmt     = scanner;
+				buf++;
+				fmt++;
+				remaining--;
+
 				current = * fmt;
 			}
 		}
 
 		tokmap_clear( & tok_map );
 
-		return result;
+		return buf_size - remaining;
 	}
 
 	Code untyped_str( StrC content )

project/gen.hpp

@@ -210,6 +210,7 @@ namespace gen
 	*/
 
 		#define Define_Specifiers                    \
+		Entry( Invalid,          INVALID )           \
 		Entry( Const,            const )             \
 		Entry( Consteval,        consteval )         \
 		Entry( Constexpr,        constexpr )         \
@@ -233,7 +234,6 @@ namespace gen
 
 		enum Type : u32
 		{
-			Invalid,
 		#	define Entry( Specifier, Code ) Specifier,
 			Define_Specifiers
 		#	undef Entry
@@ -301,9 +301,9 @@ namespace gen
 		local_persist
 		char const* lookup[ (u32)AccessSpec::Num_AccessSpec ] = {
 			"",
-			"private",
-			"protected",
 			"public",
+			"protected",
+			"private",
 		};
 
 		if ( type > AccessSpec::Public )
@@ -862,17 +862,17 @@ namespace gen
 	sw token_fmt_va( char* buf, uw buf_size, char const* fmt, s32 num_tokens, va_list va );
 
 	inline
-	char const* token_fmt( char const* fmt, sw num_tokens, ... )
+	StrC token_fmt( char const* fmt, sw num_tokens, ... )
 	{
 		local_persist thread_local
 		char buf[ZPL_PRINTF_MAXLEN] = { 0 };
 
 		va_list va;
 		va_start(va, fmt);
-		token_fmt_va(buf, ZPL_PRINTF_MAXLEN, fmt, num_tokens, va);
+		sw result = token_fmt_va(buf, ZPL_PRINTF_MAXLEN, fmt, num_tokens, va);
 		va_end(va);
 
-		return buf;
+		return { result, buf };
 	}
 
 	Code untyped_str      ( StrC content);

scripts/build.ci.ps1

@@ -37,7 +37,7 @@ $path_scripts = Join-Path $path_root scripts
 		$args_meson += $path_gen_build
 
 		Push-Location $path_gen
-		& meson $args_meson 
+		& meson $args_meson
 		Pop-Location
 	}
 
@@ -49,6 +49,7 @@ $path_scripts = Join-Path $path_root scripts
 	& ninja $args_ninja
 	Pop-Location
 
+	if ($true) {
 	$gencpp = Join-Path $path_gen_build gencpp.exe
 
 	Push-location $path_gen
@@ -71,6 +72,7 @@ $path_scripts = Join-Path $path_root scripts
 	clang-format $formatParams $targetFiles
 	Write-Host "`nFormatting complete"
 	Pop-Location
+	}
 
 
 	# Build the program depending on generated files.

test/NonParsed/Array.NonParsed.hpp

@@ -23,7 +23,7 @@ Code gen__array_base()
 		Code params = def_param( t_uw, name(value));
 		Code body   = untyped_str( code( return 2 * value * 8; ));
 
-		Code spec = def_specifiers( ESpecifier::Static_Member, ESpecifier::Inline );
+		Code spec = def_specifiers( 2, ESpecifier::Static_Member, ESpecifier::Inline );
 
 		grow_formula = def_function( name(grow_formula), params, t_uw, body, spec );
 	}
@@ -36,15 +36,34 @@ Code gen__array_base()
 
 Code gen__array( StrC type, sw type_size )
 {
-	Code t_allocator_info = def_type( name(AllocatorInfo) );
+	static Code t_allocator_info = def_type( name(AllocatorInfo) );
-	Code v_nullptr        = untyped_str( code(nullptr));
+	static Code v_nullptr        = untyped_str( code(nullptr));
 
-	static Code ArrayBase = gen__array_base();
+	// static Code array_base = def_type( name(ArrayBase) );
+	static Code header;
+	do_once_start
+		Code allocator = def_variable( t_allocator_info, name(Allocator) );
+		Code capacity  = def_variable( t_uw,             name(Capacity) );
+		Code num       = def_variable( t_uw,             name(Num) );
+
+		Code body   = def_struct_body( 3, allocator, capacity, num );
+		     header = def_struct( name(Header), body );
+	do_once_end
+
+	static Code grow_formula;
+	do_once_start
+		Code params = def_param( t_uw, name(value));
+		Code body   = untyped_str( code( return 2 * value * 8; ));
+
+		Code spec = def_specifiers( 2, ESpecifier::Static_Member, ESpecifier::Inline );
+
+		grow_formula = def_function( name(grow_formula), params, t_uw, body, spec );
+	do_once_end
 
 	StrC name;
 	{
-		char const* name_str = str_fmt_buf( "Array_%s", type.Ptr );
+		char const* name_str = str_fmt_buf( "Array_%s\0", type.Ptr );
-		s32         name_len = str_len( name );
+		s32         name_len = str_len( name_str );
 
 		name = { name_len, name_str };
 	};
@@ -59,8 +78,8 @@ Code gen__array( StrC type, sw type_size )
 	Code t_header_ptr = def_type( name(Header), __, spec_ptr );
 	Code t_header_ref = def_type( name(Header), __, spec_ref );
 
-	Code spec_static_inline = def_specifiers( ESpecifier::Static_Member, ESpecifier::Inline );
+	static Code spec_static_inline = def_specifiers( 2, ESpecifier::Static_Member, ESpecifier::Inline );
-	Code spec_static        = def_specifiers( ESpecifier::Static_Member );
+	static Code spec_static        = def_specifier( ESpecifier::Static_Member );
 
 	Code array;
 	{
@@ -79,21 +98,25 @@ Code gen__array( StrC type, sw type_size )
 
 		Code init_reserve;
 		{
-			Code params = def_params( 2, t_allocator_info, txt_n_len(allocator), t_sw, txt_n_len(capacity) );
+			Code params = def_params( 2, (Code[2]){
-			Code body = untyped_str( code(
+				def_param( t_allocator_info, name(allocator) ),
+				def_param( t_sw, name(capacity) )
+			});
+
+			char const* tmpl = txt(
 				Header* header = rcast( Header*, alloc( allocator, sizeof(Header) + sizeof(Type) ));
 
 				if ( header == nullptr )
-					return false;
+					return (<type>){ nullptr };
 
 				header->Allocator = allocator;
 				header->Capacity  = capacity;
 				header->Num       = 0;
 
-				Data = rcast( Type*, header + 1 );
+				return (<type>){ rcast( Type*, header + 1) };
+			);
 
-				return true;
+			Code body = untyped_str( token_fmt( tmpl, 1, "type", (char const*)t_array_type->Name) );
-			));
 
 			init_reserve = def_function( name(init_reserve), params, t_array_type, body, spec_static );
 		}
@@ -107,11 +130,11 @@ Code gen__array( StrC type, sw type_size )
 
 				if ( header.Num == header.Capacity )
 				{
-					if ( ! grow( header.Allocator ))
+					if ( ! grow( header.Capacity ))
 						return false;
 				}
 
-				data[ header.Num ] = value;
+				Data[ header.Num ] = value;
 				header.Num++;
 
 				return true;
@@ -127,17 +150,21 @@ Code gen__array( StrC type, sw type_size )
 				return Data[ header.Num - 1 ];
 			));
 
-			back = def_function( name(back), __, t_type_ref, body );
+			back = def_function( name(back), __, t_type_ref, body, spec_inline );
 		}
 
 		Code clear = def_function( name(clear), __, t_void, untyped_str( code(
 			Header& header = get_header();
 			header.Num = 0;
-		)));
+		)), spec_inline );
 
 		Code fill;
 		{
-			Code params = def_params( 3, t_uw, txt_n_len(begin), t_uw, txt_n_len(end), t_type, txt_n_len(value) );
+			Code params = def_params( 3, (Code[3]){
+				def_param( t_uw, name(begin) ),
+				def_param( t_uw, name(end) ),
+				def_param( t_type, name(value) )
+			});
 
 			Code body = untyped_str( code(
 				Header& header = get_header();
@@ -147,7 +174,7 @@ Code gen__array( StrC type, sw type_size )
 
 				for ( sw idx = begin; idx < end; idx++ )
 				{
-					data[ idx ] = value;
+					Data[ idx ] = value;
 				}
 
 				return true;
@@ -167,7 +194,7 @@ Code gen__array( StrC type, sw type_size )
 		}
 
 		Code get_header = def_function( name(get_header), __, t_header_ref, untyped_str( code(
-			return pcast( Header, Data - 1 );
+			return * ( rcast( Header*, Data ) - 1 );
 		)));
 
 		Code grow;
@@ -177,7 +204,7 @@ Code gen__array( StrC type, sw type_size )
 			Code body = untyped_str( code(
 				Header& header = get_header();
 
-				t_uw new_capacity = grow_formula( header.Capacity );
+				uw new_capacity = grow_formula( header.Capacity );
 
 				if ( new_capacity < min_capacity )
 					new_capacity = 8;
@@ -193,11 +220,11 @@ Code gen__array( StrC type, sw type_size )
 			Code body = untyped_str( code(
 				Header& header = get_header();
 
-				assert_crash( header.Num > 0 );
+				ZPL_ASSERT( header.Num > 0 );
 				header.Num--;
 			));
 
-			pop = def_function( name(pop), __, t_bool, body );
+			pop = def_function( name(pop), __, t_bool, body, spec_inline );
 		}
 
 		Code reserve;
@@ -208,7 +235,7 @@ Code gen__array( StrC type, sw type_size )
 				Header& header = get_header();
 
 				if ( header.Capacity < new_capacity )
-					return set_capacity( capacity );
+					return set_capacity( new_capacity );
 
 				return true;
 			));
@@ -225,7 +252,7 @@ Code gen__array( StrC type, sw type_size )
 
 				if ( num > header.Capacity )
 				{
-					if ( ! grow( header.Allocator ))
+					if ( ! grow( header.Capacity ))
 						return false;
 				}
 
@@ -261,9 +288,9 @@ Code gen__array( StrC type, sw type_size )
 				new_header->Num       = header.Num;
 				new_header->Capacity  = new_capacity;
 
-				::free( header );
+				::free( header.Allocator, & header );
 
-				*Data = new_header + 1;
+				Data = (u8*) new_header + 1;
 
 				return true;
 			));
@@ -272,6 +299,8 @@ Code gen__array( StrC type, sw type_size )
 		}
 
 		Code body = def_struct_body( 17
+			, header
+			, grow_formula
 			, using_type
 
 			, init
@@ -309,10 +338,14 @@ Array(GenArrayRequest) GenArrayRequests;
 
 void gen__array_request( StrC type, sw size, StrC dep = {} )
 {
+	do_once_start
+		array_init( GenArrayRequests, g_allocator );
+	do_once_end
+
 	GenArrayRequest request = { type, dep, size };
 	array_append( GenArrayRequests, request );
 }
-#define Gen_Array( type ) gen__array_request( { txt_n_len(type) }, sizeof(type) )
+#define gen_array( type ) gen__array_request( { txt_n_len(type) }, sizeof(type) )
 
 u32 gen_array_file()
 {
@@ -320,6 +353,11 @@ u32 gen_array_file()
 	gen_array_file;
 	gen_array_file.open( "array.gen.hpp" );
 
+	Code include_zpl = def_include( StrC::from("../../thirdparty/zpl.h") );
+	gen_array_file.print( include_zpl );
+
+	// Code array_base = gen__array_base();
+	// gen_array_file.print( array_base );
 
 	GenArrayRequest* current = GenArrayRequests;
 	s32 left = array_count( GenArrayRequests );

test/NonParsed/Sanity.hpp

@@ -1,5 +1,3 @@
-
-
 #ifdef gen_time
 #include "gen.hpp"
 

test/test.cpp

@@ -14,8 +14,11 @@ int gen_main()
 	Memory::setup();
 	gen::init();
 
-	gen_sanity();
+	// gen_sanity();
-	// gen_array_file();
+
+	gen_array( u8 );
+	// gen_array( sw );
+	gen_array_file();
 
 	gen::deinit();
 	Memory::cleanup();