changing to just pass lib Context instead of ParseContext.

If I want to make it more granular I can later.
started to work on modularizing parser code paths
2025-07-01 03:11:03 -07:00 · 2025-02-19 15:48:56 -05:00 · 2025-02-19 12:10:11 -05:00 · 2025-02-19 11:22:00 -05:00 · 2025-02-19 11:08:24 -05:00 · 2025-02-19 10:50:55 -05:00
27 changed files with 485 additions and 228 deletions
--- a/base/auxiliary/builder.hpp
+++ b/base/auxiliary/builder.hpp
@ -3,7 +3,7 @@
 #	include "helpers/push_ignores.inline.hpp"
 #	include "components/header_start.hpp"
 #	include "components/types.hpp"
-#	include "components/gen/ecode.hpp"
+#	include "components/gen/ecodetypes.hpp"
 #	include "components/gen/eoperator.hpp"
 #	include "components/gen/especifier.hpp"
 #	include "components/ast.hpp"
--- a/base/auxiliary/scanner.hpp
+++ b/base/auxiliary/scanner.hpp
@ -3,7 +3,7 @@
 #	include "helpers/push_ignores.inline.hpp"
 #	include "components/header_start.hpp"
 #	include "components/types.hpp"
-#	include "components/gen/ecode.hpp"
+#	include "components/gen/ecodetypes.hpp"
 #	include "components/gen/eoperator.hpp"
 #	include "components/gen/especifier.hpp"
 #	include "components/ast.hpp"
--- a/base/components/ast.hpp
+++ b/base/components/ast.hpp
@ -1,9 +1,6 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #pragma once
-#include "types.hpp"
-#include "gen/ecode.hpp"
-#include "gen/eoperator.hpp"
-#include "gen/especifier.hpp"
+#include "parser_types.hpp"
 #endif

 /*
@ -406,7 +403,8 @@ struct AST
 				Code  PostNameMacro;    // Only used with parameters for specifically UE_REQUIRES (Thanks Unreal)
 			};
 		};
-		StrCached  Content;          // Attributes, Comment, Execution, Include
+		StrCached  Content;      // Attributes, Comment, Execution, Include
+		TokenSlice ContentToks; // TODO(Ed): Use a token slice for content
 		struct {
 			Specifier  ArrSpecs[AST_ArrSpecs_Cap]; // Specifiers
 			Code       NextSpecs;              // Specifiers; If ArrSpecs is full, then NextSpecs is used.
@ -422,7 +420,7 @@ struct AST
 		Code Next;
 		Code Back;
 	};
-	Token*            Token; // Reference to starting token, only available if it was derived from parsing.
+	Token*            Token; // Reference to starting token, only available if it was derived from parsing. // TODO(Ed): Change this to a token slice.
 	Code              Parent;
 	CodeType          Type;
 //	CodeFlag          CodeFlags;
--- a/base/components/code_serialization.cpp
+++ b/base/components/code_serialization.cpp
@ -38,13 +38,13 @@ void body_to_strbuilder_export( CodeBody body, StrBuilder* result )
 	GEN_ASSERT(result != nullptr);
 	strbuilder_append_fmt( result, "export\n{\n" );

-	Code curr = cast(Code, body);
+	Code curr = body->Front;
 	s32  left = body->NumEntries;
 	while ( left-- )
 	{
 		code_to_strbuilder_ref(curr, result);
 		// strbuilder_append_fmt( result, "%SB", code_to_strbuilder(curr) );
-		++curr;
+		curr = curr->Next;
 	}

 	strbuilder_append_fmt( result, "};\n" );
--- a/base/components/code_types.hpp
+++ b/base/components/code_types.hpp
@ -253,6 +253,7 @@ struct CodeSpecifiers
 	Using_Code( CodeSpecifiers );
 	bool       append( Specifier spec )            { return specifiers_append(* this, spec); }
 	s32        has( Specifier spec )               { return specifiers_has(* this, spec); }
+	s32        index_of( Specifier spec )          { return specifiers_index_of(* this, spec); }
 	s32        remove( Specifier to_remove )       { return specifiers_remove(* this, to_remove); }
 	StrBuilder to_strbuilder()                     { return specifiers_to_strbuilder(* this ); }
 	void       to_strbuilder( StrBuilder& result ) { return specifiers_to_strbuilder_ref(* this, & result); }
--- a/base/components/inlines.hpp
+++ b/base/components/inlines.hpp
@ -1,6 +1,6 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #pragma once
-#include "interface.hpp"
+#include "constants.hpp"
 #endif

 #pragma region Serialization
@ -38,7 +38,7 @@ void body_to_strbuilder_ref( CodeBody body, StrBuilder* result )
 	{
 		code_to_strbuilder_ref(curr, result);
 		// strbuilder_append_fmt( result, "%SB", code_to_strbuilder(curr) );
-		++curr;
+		curr = curr->Next;
 	}
 }

--- a/base/components/interface.cpp
+++ b/base/components/interface.cpp
@ -71,6 +71,14 @@ void* fallback_allocator_proc( void* allocator_data, AllocType type, ssize size,
 	return nullptr;
 }

+internal
+void fallback_logger(LogEntry entry) 
+{
+	GEN_ASSERT(entry.msg.Len > 0);
+	GEN_ASSERT(entry.msg.Ptr);
+	log_fmt("%S: %S", loglevel_to_str(entry.level), entry.msg);
+}
+
 internal
 void define_constants()
 {
@ -283,6 +291,19 @@ void init(Context* ctx)
 		ctx->InitSize_Fallback_Allocator_Bucket_Size = megabytes(8);
 	}

+	if (ctx->InitSize_StrCacheTable == 0)
+	{
+		ctx->InitSize_StrCacheTable = kilobytes(8);
+	}
+	if (ctx->InitSize_MacrosTable == 0)
+	{
+		ctx->InitSize_MacrosTable = kilobytes(8);
+	}
+
+	if (ctx->Logger == nullptr) {
+		ctx->Logger = & fallback_logger;
+	}
+
 	// Override the current context (user has to put it back if unwanted).
 	_ctx = ctx;

@ -298,7 +319,7 @@ void init(Context* ctx)
 	}
 	// Setup the code pool and code entries arena.
 	{
-		Pool code_pool = pool_init( ctx->Allocator_Pool, ctx->CodePool_NumBlocks, sizeof(AST) );
+		Pool code_pool = pool_init( ctx->Allocator_Pool, ctx->CodePool_NumBlocks, size_of(AST) );
 		if ( code_pool.PhysicalStart == nullptr )
 			GEN_FATAL( "gen::init: Failed to initialize the code pool" );
 		array_append( ctx->CodePools, code_pool );
@ -311,11 +332,11 @@ void init(Context* ctx)
 	}
 	// Setup the hash tables
 	{
-		ctx->StrCache = hashtable_init(StrCached, ctx->Allocator_DyanmicContainers);
+		ctx->StrCache = hashtable_init_reserve(StrCached, ctx->Allocator_DyanmicContainers, ctx->InitSize_StrCacheTable);
 		if ( ctx->StrCache.Entries == nullptr )
 			GEN_FATAL( "gen::init: Failed to initialize the StringCache");

-		ctx->Macros = hashtable_init(Macro, ctx->Allocator_DyanmicContainers);
+		ctx->Macros = hashtable_init_reserve(Macro, ctx->Allocator_DyanmicContainers, ctx->InitSize_MacrosTable);
 		if (ctx->Macros.Hashes == nullptr || ctx->Macros.Entries == nullptr) {
 			GEN_FATAL( "gen::init: Failed to initialize the PreprocessMacros table" );
 		}
--- a/base/components/interface.hpp
+++ b/base/components/interface.hpp
@ -15,23 +15,37 @@
  \▓▓▓▓▓▓  \▓▓▓▓▓▓▓\▓▓   \▓▓     \▓▓▓▓▓▓\▓▓   \▓▓   \▓▓▓▓  \▓▓▓▓▓▓▓\▓▓      \▓▓       \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓
 */

-#if 0
-enum LogLevel : u32
+enum LogLevel //: u32
 {
-	Info,
-	Warning,
-	Panic,
+	LL_Null,
+	LL_Note,
+	LL_Warning,
+	LL_Error,
+	LL_Fatal,
+	LL_UnderlyingType = GEN_U32_MAX,
 };
+typedef enum LogLevel LogLevel;
+
+Str loglevel_to_str(LogLevel level) 
+{
+	local_persist
+	Str lookup[] = {
+		{ "Null",    sizeof("Null")  - 1 },
+		{ "Note",    sizeof("Note")  - 1 },
+		{ "Warning", sizeof("Info")  - 1 },
+		{ "Error",   sizeof("Error") - 1 },
+		{ "Fatal",   sizeof("Fatal") - 1 },
+	};
+	return lookup[level];
+}

 struct LogEntry
 {
-	Str   msg;
-	u32   line_num;
-	void* data;
+	Str      msg;
+	LogLevel level;
 };

-typedef void LoggerCallback(LogEntry entry);
-#endif
+typedef void LoggerProc(LogEntry entry);

 // Note(Ed): This is subject to heavily change 
 // with upcoming changes to the library's fallback (default) allocations strategy;
@ -64,9 +78,16 @@ struct Context
 	u32 InitSize_LexerTokens;
 	u32 SizePer_StringArena;

+	u32 InitSize_StrCacheTable;
+	u32 InitSize_MacrosTable;
+
 // TODO(Ed): Symbol Table
 	// Keep track of all resolved symbols (naemspaced identifiers)

+// Logging
+
+	LoggerProc* Logger;
+
 // Parser

 	// Used by the lexer to persistently treat all these identifiers as preprocessor defines.
@ -104,6 +125,45 @@ struct Context
 // An implicit context interface will be provided instead as wrapper procedures as convience.
 GEN_API extern Context* _ctx;

+// By default this library will either crash or exit if an error is detected while generating codes.
+// Even if set to not use GEN_FATAL, GEN_FATAL will still be used for memory failures as the library is unusable when they occur.
+#ifdef GEN_DONT_USE_FATAL
+	#define log_failure log_fmt
+#else
+	#define log_failure GEN_FATAL
+#endif
+
+// TODO(Ed): Swap all usage of this with logger_fmt (then rename logger_fmt to log_fmt)
+inline
+ssize log_fmt(char const* fmt, ...)
+{
+	ssize res;
+	va_list va;
+
+	va_start(va, fmt);
+	res = c_str_fmt_out_va(fmt, va);
+	va_end(va);
+
+	return res;
+}
+
+inline
+void logger_fmt(Context* ctx, LogLevel level, char const* fmt, ...)
+{
+	local_persist thread_local
+	PrintF_Buffer buf = struct_zero(PrintF_Buffer);
+
+	va_list va;
+	va_start(va, fmt);
+	ssize res = c_str_fmt_va(buf, GEN_PRINTF_MAXLEN, fmt, va) -1;
+	va_end(va);
+
+	StrBuilder msg = strbuilder_make_length(ctx->Allocator_Temp, buf, res);
+
+	LogEntry entry = { strbuilder_to_str(msg), level };
+	ctx->Logger(entry);
+}
+
 // Initialize the library. There first ctx initialized must exist for lifetime of other contextes that come after as its the one that
 GEN_API void init(Context* ctx);

@ -114,7 +174,7 @@ GEN_API void deinit(Context* ctx);
 // Retrieves the active context (not usually needed, but here in case...)
 GEN_API Context* get_context();

-// Clears the allocations, but doesn't free the memoery, then calls init() again.
+// Clears the allocations, but doesn't free the memory, then calls init() again.
 // Ease of use.
 GEN_API void reset(Context* ctx);

@ -334,37 +394,42 @@ forceinline CodeBody         def_union_body      ( s32 num, Code* codes )

 #pragma region Parsing

-#if 0
-struct StackNode
+struct ParseStackNode
 {
-	StackNode* Prev;
+	ParseStackNode* Prev;

-	Token Start;
-	Token Name;       // The name of the AST node (if parsed)
-	Str  FailedProc; // The name of the procedure that failed
+	TokenSlice Tokens;
+	Token*     Start;
+	Str        Name;     // The name of the AST node (if parsed)
+	Str        ProcName; // The name of the procedure
+	Code       CodeRel;  // Relevant AST node
+	// TODO(Ed): When an error occurs, the parse stack is not released and instead the scope is left dangling.
 };
-// Stack nodes are allocated the error's allocator

-struct Error
+typedef struct ParseMessage ParseMessage;
+struct ParseMessage
 {
-	StrBuilder     message;
-	StackNode* context_stack;
+	ParseMessage*   Next;
+	ParseStackNode* Scope;
+	Str             Log;
+	LogLevel        Level;
 };

 struct ParseInfo
 {
-	Arena FileMem;
-	Arena TokMem;
-	Arena CodeMem;
-
-	FileContents FileContent;
-	Array<Token> Tokens;
-	Array<Error> Errors;
-	// Errors are allocated to a dedicated general arena.
+	ParseMessage* messages;
+	LexedInfo     lexed;
+	Code          result;
 };

-CodeBody parse_file( Str path );
-#endif
+struct ParseOpts
+{
+	AllocatorInfo backing_msgs;
+	AllocatorInfo backing_tokens;
+	AllocatorInfo backing_ast;
+};
+
+ParseInfo wip_parse_str( LexedInfo lexed, ParseOpts* opts GEN_PARAM_DEFAULT );

 GEN_API CodeClass       parse_class        ( Str class_def       );
 GEN_API CodeConstructor parse_constructor  ( Str constructor_def );
@ -395,9 +460,10 @@ GEN_API ssize token_fmt_va( char* buf, usize buf_size, s32 num_tokens, va_list v
 //! Do not use directly. Use the token_fmt macro instead.
 Str   token_fmt_impl( ssize, ... );

-GEN_API Code untyped_str( Str content);
+GEN_API Code untyped_str      ( Str content);
 GEN_API Code untyped_fmt      ( char const* fmt, ... );
 GEN_API Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... );
+GEN_API Code untyped_toks     ( TokenSlice tokens );

 #pragma endregion Untyped text

--- a/base/components/interface.parsing.cpp
+++ b/base/components/interface.parsing.cpp
@ -1,6 +1,6 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #pragma once
-#include "gen/etoktype.cpp"
+#include "gen/etoktype.hpp"
 #include "interface.upfront.cpp"
 #include "lexer.cpp"
 #include "parser.cpp"
@ -8,6 +8,29 @@

 // Publically Exposed Interface

+ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts)
+{
+	TokArray toks;
+	if (lexed.tokens.Num == 0 && lexed.tokens.Ptr == nullptr) {
+		check_parse_args(lexed.text);
+		toks = lex(lexed.text);
+
+		TokenSlice slice = { toks.Arr, scast(s32, array_num(toks.Arr)) };
+		lexed.tokens = slice;
+	}
+	ParseInfo info = struct_zero(ParseInfo);
+	info.lexed = lexed;
+
+	// TODO(Ed): ParseInfo should be set to the parser context.
+
+	_ctx->parser.Tokens = toks;
+	push_scope();
+	CodeBody result = parse_global_nspace(CT_Global_Body);
+	parser_pop(& _ctx->parser);
+
+	return info;
+}
+
 CodeClass parse_class( Str def )
 {
 	check_parse_args( def );
@ -326,6 +349,7 @@ CodeVar parse_variable( Str def )
 #undef left
 #undef check
 #undef push_scope
+#undef NullScope
 #undef def_assign

 // Here for C Variant
--- a/base/components/interface.untyped.cpp
+++ b/base/components/interface.untyped.cpp
@ -176,3 +176,16 @@ Code untyped_token_fmt( s32 num_tokens, char const* fmt, ... )

 	return result;
 }
+
+Code untyped_toks( TokenSlice tokens )
+{
+	if ( tokens.Num == 0 ) {
+		log_failure( "untyped_toks: empty token slice" );
+		return InvalidCode;
+	}
+	Code
+	result              = make_code();
+	result->Type        = CT_Untyped;
+	result->ContentToks = tokens;
+	return result;
+}
--- a/base/components/interface.upfront.cpp
+++ b/base/components/interface.upfront.cpp
@ -473,8 +473,10 @@ CodeComment def_comment( Str content )
 	return (CodeComment) result;
 }

-CodeConstructor def_constructor( Opts_def_constructor p )
+CodeConstructor def_constructor( Opts_def_constructor opt )
 {
+	Opts_def_constructor p = get_optional(opt);
+
 	if ( p.params && p.params->Type != CT_Parameters ) {
 		log_failure("gen::def_constructor: params must be of Parameters type - %s", code_debug_str((Code)p.params));
 		GEN_DEBUG_TRAP();
@ -510,8 +512,10 @@ CodeConstructor def_constructor( Opts_def_constructor p )
 	return result;
 }

-CodeClass def_class( Str name, Opts_def_struct p )
+CodeClass def_class( Str name, Opts_def_struct opt )
 {
+	Opts_def_struct p = get_optional(opt);
+
 	if ( ! name_check( def_class, name ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -561,8 +565,10 @@ CodeClass def_class( Str name, Opts_def_struct p )
 	return result;
 }

-CodeDefine def_define( Str name, MacroType type, Opts_def_define p )
+CodeDefine def_define( Str name, MacroType type, Opts_def_define opt )
 {
+	Opts_def_define p = get_optional(opt);
+
 	if ( ! name_check( def_define, name ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -585,8 +591,10 @@ CodeDefine def_define( Str name, MacroType type, Opts_def_define p )
 	return result;
 }

-CodeDestructor def_destructor( Opts_def_destructor p )
+CodeDestructor def_destructor( Opts_def_destructor opt )
 {
+	Opts_def_destructor p = get_optional(opt);
+
 	if ( p.specifiers && p.specifiers->Type != CT_Specifiers ) {
 		log_failure( "gen::def_destructor: specifiers was not a 'Specifiers' type: %s", code_debug_str(p.specifiers) );
 		GEN_DEBUG_TRAP();
@ -619,8 +627,10 @@ CodeDestructor def_destructor( Opts_def_destructor p )
 	return result;
 }

-CodeEnum def_enum( Str name, Opts_def_enum p )
+CodeEnum def_enum( Str name, Opts_def_enum opt )
 {
+	Opts_def_enum p = get_optional(opt);
+
 	if ( ! name_check( def_enum, name ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -742,8 +752,10 @@ CodeFriend def_friend( Code declaration )
 	return result;
 }

-CodeFn def_function( Str name, Opts_def_function p )
+CodeFn def_function( Str name, Opts_def_function opt )
 {
+	Opts_def_function p = get_optional(opt);
+
 	if ( ! name_check( def_function, name )) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -802,8 +814,10 @@ CodeFn def_function( Str name, Opts_def_function p )
 	return result;
 }

-CodeInclude def_include( Str path, Opts_def_include p )
+CodeInclude def_include( Str path, Opts_def_include opt )
 {
+	Opts_def_include p = get_optional(opt);
+
 	if ( path.Len <= 0 || path.Ptr == nullptr ) {
 		log_failure( "gen::def_include: Invalid path provided - %d" );
 		GEN_DEBUG_TRAP();
@ -821,8 +835,10 @@ CodeInclude def_include( Str path, Opts_def_include p )
 	return result;
 }

-CodeModule def_module( Str name, Opts_def_module p )
+CodeModule def_module( Str name, Opts_def_module opt )
 {
+	Opts_def_module p = get_optional(opt);
+
 	if ( ! name_check( def_module, name )) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -835,8 +851,10 @@ CodeModule def_module( Str name, Opts_def_module p )
 	return result;
 }

-CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace p )
+CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace opt )
 {
+	Opts_def_namespace p = get_optional(opt);
+
 	if ( ! name_check( def_namespace, name )) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -859,8 +877,10 @@ CodeNS def_namespace( Str name, CodeBody body, Opts_def_namespace p )
 	return result;
 }

-CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator p )
+CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator opt )
 {
+	Opts_def_operator p = get_optional(opt);
+
 	if ( p.attributes && p.attributes->Type != CT_PlatformAttributes ) {
 		log_failure( "gen::def_operator: PlatformAttributes was provided but its not of attributes type: %s", code_debug_str(p.attributes) );
 		GEN_DEBUG_TRAP();
@ -926,8 +946,10 @@ CodeOperator def_operator( Operator op, Str nspace, Opts_def_operator p )
 	return result;
 }

-CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast p )
+CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast opt )
 {
+	Opts_def_operator_cast p = get_optional(opt);
+
 	if ( ! null_check( def_operator_cast, type )) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -959,8 +981,10 @@ CodeOpCast def_operator_cast( CodeTypename type, Opts_def_operator_cast p )
 	return result;
 }

-CodeParams def_param( CodeTypename type, Str name, Opts_def_param p )
+CodeParams def_param( CodeTypename type, Str name, Opts_def_param opt )
 {
+	Opts_def_param p = get_optional(opt);
+
 	if ( ! name_check( def_param, name ) || ! null_check( def_param, type ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -1034,8 +1058,10 @@ CodeSpecifiers def_specifier( Specifier spec )
 	return result;
 }

-CodeStruct def_struct( Str name, Opts_def_struct p )
+CodeStruct def_struct( Str name, Opts_def_struct opt )
 {
+	Opts_def_struct p = get_optional(opt);
+
 	if ( p.attributes && p.attributes->Type != CT_PlatformAttributes ) {
 		log_failure( "gen::def_struct: attributes was not a `PlatformAttributes` type - %s", code_debug_str(cast(Code, p.attributes)) );
 		GEN_DEBUG_TRAP();
@ -1076,8 +1102,10 @@ CodeStruct def_struct( Str name, Opts_def_struct p )
 	return result;
 }

-CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_template p )
+CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_template opt )
 {
+	Opts_def_template p = get_optional(opt);
+
 	if ( ! null_check( def_template, declaration ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -1108,8 +1136,10 @@ CodeTemplate def_template( CodeParams params, Code declaration, Opts_def_templat
 	return result;
 }

-CodeTypename def_type( Str name, Opts_def_type p )
+CodeTypename def_type( Str name, Opts_def_type opt )
 {
+	Opts_def_type p = get_optional(opt);
+
 	if ( ! name_check( def_type, name )) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -1143,8 +1173,10 @@ CodeTypename def_type( Str name, Opts_def_type p )
 	return result;
 }

-CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef p )
+CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef opt )
 {
+	Opts_def_typedef p = get_optional(opt);
+
 	if ( ! null_check( def_typedef, type ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -1206,8 +1238,10 @@ CodeTypedef def_typedef( Str name, Code type, Opts_def_typedef p )
 	return result;
 }

-CodeUnion def_union( Str name, CodeBody body, Opts_def_union p )
+CodeUnion def_union( Str name, CodeBody body, Opts_def_union opt )
 {
+	Opts_def_union p = get_optional(opt);
+
 	if ( ! null_check( def_union, body ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -1233,8 +1267,10 @@ CodeUnion def_union( Str name, CodeBody body, Opts_def_union p )
 	return result;
 }

-CodeUsing def_using( Str name, CodeTypename type, Opts_def_using p )
+CodeUsing def_using( Str name, CodeTypename type, Opts_def_using opt )
 {
+	Opts_def_using p = get_optional(opt);
+
 	if ( ! name_check( def_using, name ) || null_check( def_using, type ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
@ -1274,8 +1310,10 @@ CodeUsing def_using_namespace( Str name )
 	return result;
 }

-CodeVar def_variable( CodeTypename type, Str name, Opts_def_variable p )
+CodeVar def_variable( CodeTypename type, Str name, Opts_def_variable opt )
 {
+	Opts_def_variable p = get_optional(opt);
+
 	if ( ! name_check( def_variable, name ) || ! null_check( def_variable, type ) ) {
 		GEN_DEBUG_TRAP();
 		return InvalidCode;
--- a/base/components/lexer.cpp
+++ b/base/components/lexer.cpp
@ -1,7 +1,7 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #pragma once
 #include "interface.upfront.cpp"
-#include "gen/etoktype.cpp"
+#include "gen/etoktype.hpp"
 #endif

 StrBuilder tok_to_strbuilder(Token tok)
@ -564,9 +564,12 @@ void lex_found_token( LexContext* ctx )
 	array_append( _ctx->Lexer_Tokens, ctx->token );
 }

+// TODO(Ed): We should dynamically allocate the lexer's array in Allocator_DyanmicContainers.
+
 // TODO(Ed): We need to to attempt to recover from a lex failure?
+
 neverinline
-// TokArray lex( Array<Token> tokens, Str content )
+// void lex( Array<Token> tokens, Str content )
 TokArray lex( Str content )
 {
 	LexContext c; LexContext* ctx = & c;
--- a/base/components/parser.cpp
+++ b/base/components/parser.cpp
@ -1,6 +1,6 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #pragma once
-#include "gen/etoktype.cpp"
+#include "gen/etoktype.hpp"
 #include "parser_case_macros.cpp"
 #include "interface.upfront.cpp"
 #include "lexer.cpp"
@ -11,7 +11,7 @@
 constexpr bool lex_dont_skip_formatting = false;
 constexpr bool      lex_skip_formatting = true;

-void parser_push( ParseContext* ctx, StackNode* node )
+void parser_push( ParseContext* ctx, ParseStackNode* node )
 {
 	node->Prev = ctx->Scope;
 	ctx->Scope = node;
@ -60,7 +60,7 @@ StrBuilder parser_to_strbuilder(ParseContext ctx)
 	else
 		strbuilder_append_fmt(& result, "\t(%d, %d)\n", last_valid.Line, last_valid.Column );

-	StackNode* curr_scope = ctx.Scope;
+	ParseStackNode* curr_scope = ctx.Scope;
 	s32 level = 0;
 	do
 	{
@ -181,18 +181,25 @@ bool _check_parse_args( Str def, char const* func_name )
 #	define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ )
 #	define check( Type_ )        ( left && currtok.Type        == Type_ )

-#	define push_scope()                                                                                                         \
-	Str null_name = {};                                                                                                         \
-	StackNode scope = { nullptr, lex_current( &  _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ) }; \
-	parser_push( & _ctx->parser, & scope )
+// TODO(Ed): Don't do this anymore, we need a better initializer.
+#	define push_scope()                                                                                                                                        \
+	Str null_name = {};                                                                                                                                        \
+	ParseStackNode scope = { nullptr, {nullptr, 0}, lex_current( &  _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ), { nullptr} }; \
+	parser_push( & _ctx->parser, & scope ) 
+
+#if GEN_COMPILER_CPP
+#	define NullScope { nullptr, {nullptr, 0}, lex_current( &  ctx->parser.Tokens, lex_dont_skip_formatting ), Str{nullptr, 0}, txt( __func__ ), { nullptr} }
+#else
+#	define NullScope (ParseStackNode){ nullptr, {nullptr, 0}, lex_current( &  ctx->parser.Tokens, lex_dont_skip_formatting ), (Str){nullptr, 0}, txt( __func__ ), { nullptr} }
+#endif

 #pragma endregion Helper Macros

 // Procedure Forwards ( Entire parser internal parser interface )

-internal Code               parse_array_decl                   ();
-internal CodeAttributes     parse_attributes                   ();
-internal CodeComment        parse_comment                      ();
+internal Code               parse_array_decl                   (Context* ctx);
+internal CodeAttributes     parse_attributes                   (Context* ctx);
+internal CodeComment        parse_comment                      (Context* ctx);
 internal Code               parse_complicated_definition       ( TokType which );
 internal CodeBody           parse_class_struct_body            ( TokType which, Token name );
 internal Code               parse_class_struct                 ( TokType which, bool inplace_def );
@ -490,10 +497,18 @@ StrBuilder parser_strip_formatting( Str raw_text, bool preserve_newlines )
 	return content;
 }

-internal
-Code parse_array_decl()
+StrBuilder parser_strip_formatting_2(TokenSlice tokens)
 {
-	push_scope();
+	// TODO(Ed): Use this to produce strings for validation purposes. We shouldn't serialize down from tokens once we start storing token slices for content.
+	StrBuilder result = struct_zero(StrBuilder);
+	return result;
+}
+
+internal
+Code parse_array_decl(Context* ctx)
+{
+	ParseStackNode scope = NullScope;
+	parser_push(& ctx->parser, & scope );

 	if ( check( Tok_Operator ) && currtok.Text.Ptr[0] == '[' && currtok.Text.Ptr[1] == ']' )
 	{
@ -501,7 +516,7 @@ Code parse_array_decl()
 		eat( Tok_Operator );
 		// []

-		parser_pop(& _ctx->parser);
+		parser_pop(& ctx->parser);
 		return array_expr;
 	}

@ -512,41 +527,45 @@ Code parse_array_decl()

 		if ( left == 0 )
 		{
-			log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", parser_to_strbuilder(_ctx->parser) );
-			parser_pop(& _ctx->parser);
+			log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%s", parser_to_strbuilder(ctx->parser) );
+			parser_pop(& ctx->parser);
 			return InvalidCode;
 		}

 		if ( currtok.Type == Tok_BraceSquare_Close )
 		{
-			log_failure( "Error, empty array expression in definition\n%s", parser_to_strbuilder(_ctx->parser) );
-			parser_pop(& _ctx->parser);
+			log_failure( "Error, empty array expression in definition\n%s", parser_to_strbuilder(ctx->parser) );
+			parser_pop(& ctx->parser);
 			return InvalidCode;
 		}

+		TokenSlice tokens = { & currtok, 1 };
 		Token untyped_tok = currtok;

 		while ( left && currtok.Type != Tok_BraceSquare_Close )
 		{
 			eat( currtok.Type );
+			++ tokens.Num;
 		}

-		untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr;
+		// untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr;
+		untyped_tok.Text = token_range_to_str(untyped_tok, prevtok);

 		Code array_expr = untyped_str( untyped_tok.Text );
+		// Code array_expr = untyped_toks( tokens ); // TODO(Ed): Use token slice instead of untyped strings.
 		// [ <Content>

 		if ( left == 0 )
 		{
-			log_failure( "Error, unexpected end of array declaration, expected ]\n%s", parser_to_strbuilder(_ctx->parser) );
-			parser_pop(& _ctx->parser);
+			log_failure( "Error, unexpected end of array declaration, expected ]\n%s", parser_to_strbuilder(ctx->parser) );
+			parser_pop(& ctx->parser);
 			return InvalidCode;
 		}

 		if ( currtok.Type != Tok_BraceSquare_Close )
 		{
-			log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(_ctx->parser) );
-			parser_pop(& _ctx->parser);
+			log_failure( "%s: Error, expected ] in array declaration, not %s\n%s", toktype_to_str( currtok.Type ), parser_to_strbuilder(ctx->parser) );
+			parser_pop(& ctx->parser);
 			return InvalidCode;
 		}

@ -556,24 +575,25 @@ Code parse_array_decl()
 		// Its a multi-dimensional array
 		if ( check( Tok_BraceSquare_Open ))
 		{
-			Code adjacent_arr_expr = parse_array_decl();
+			Code adjacent_arr_expr = parse_array_decl(ctx);
 			// [ <Content> ][ <Content> ]...

 			array_expr->Next = adjacent_arr_expr;
 		}

-		parser_pop(& _ctx->parser);
+		parser_pop(& ctx->parser);
 		return array_expr;
 	}

-	parser_pop(& _ctx->parser);
+	parser_pop(& ctx->parser);
 	return NullCode;
 }

 internal inline
-CodeAttributes parse_attributes()
+CodeAttributes parse_attributes(Context* ctx)
 {
-	push_scope();
+	ParseStackNode scope = NullScope;
+	parser_push(& ctx->parser, & scope);

 	Token start = currtok;
 	s32   len   = 0;
@ -664,7 +684,7 @@ CodeAttributes parse_attributes()
 	if ( len > 0 )
 	{
 		Str attribute_txt = { start.Text.Ptr, len };
-		parser_pop(& _ctx->parser);
+		parser_pop(& ctx->parser);

 		StrBuilder name_stripped = parser_strip_formatting( attribute_txt, parser_strip_formatting_dont_preserve_newlines );

@ -673,11 +693,10 @@ CodeAttributes parse_attributes()
 		result->Name    = cache_str( strbuilder_to_str(name_stripped) );
 		result->Content = result->Name;
 		// result->Token   =
-
 		return ( CodeAttributes )result;
 	}

-	parser_pop(& _ctx->parser);
+	parser_pop(& ctx->parser);
 	return NullCode;
 }

@ -708,7 +727,7 @@ Code parse_class_struct( TokType which, bool inplace_def )
 	eat( which );
 	// <ModuleFlags> <class/struct>

-	attributes = parse_attributes();
+	attributes = parse_attributes(_ctx);
 	// <ModuleFlags> <class/struct> <Attributes>

 	if ( check( Tok_Identifier ) ) {
@ -775,7 +794,7 @@ Code parse_class_struct( TokType which, bool inplace_def )
 		// <ModuleFlags> <class/struct> <Attributes> <Name> <final> : <Access Specifier> <Name>, ... { <Body> };

 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
+			inline_cmt = parse_comment(_ctx);
 		// <ModuleFlags> <class/struct> <Attributes> <Name> <final> : <Access Specifier> <Name>, ... { <Body> }; <InlineCmt>
 	}

@ -836,7 +855,7 @@ CodeBody parse_class_struct_body( TokType which, Token name )
 				break;
 			}
 			case Tok_Comment: {
-				member = cast(Code, parse_comment());
+				member = cast(Code, parse_comment(_ctx));
 				break;
 			}
 			case Tok_Access_Public: {
@ -995,7 +1014,7 @@ CodeBody parse_class_struct_body( TokType which, Token name )
 			GEN_DEFINE_ATTRIBUTE_TOKENS
 		#undef Entry
 			{
-				attributes = parse_attributes();
+				attributes = parse_attributes(_ctx);
 				// <Attributes>
 			}
 			//! Fallthrough intended
@ -1047,7 +1066,7 @@ CodeBody parse_class_struct_body( TokType which, Token name )
 				if ( tok_is_attribute(currtok) )
 				{
 					// Unfortuantely Unreal has code where there is attirbutes before specifiers
-					CodeAttributes more_attributes = parse_attributes();
+					CodeAttributes more_attributes = parse_attributes(_ctx);

 					if ( attributes )
 					{
@ -1135,9 +1154,10 @@ CodeBody parse_class_struct_body( TokType which, Token name )
 }

 internal
-CodeComment parse_comment()
-{
-	push_scope();
+CodeComment parse_comment(Context* ctx)
+{	
+	ParseStackNode scope = NullScope;
+	parser_push(& ctx->parser, & scope );

 	CodeComment
 	result          = (CodeComment) make_code();
@ -1146,7 +1166,7 @@ CodeComment parse_comment()
 	// result->Token   = currtok_noskip;
 	eat( Tok_Comment );

-	parser_pop(& _ctx->parser);
+	parser_pop(& ctx->parser);
 	return result;
 }

@ -1443,6 +1463,22 @@ CodeFn parse_function_after_name(
 	}
 	// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers>

+	// Check for trailing specifiers...
+	CodeAttributes post_rt_attributes = parse_attributes(_ctx);
+	if (post_rt_attributes)
+	{
+		if (attributes)
+		{
+			StrBuilder merged   =  strbuilder_fmt_buf(_ctx->Allocator_Temp, "%S %S", attributes->Content, post_rt_attributes->Content);
+			attributes->Content = cache_str(strbuilder_to_str(merged));
+		}
+		else
+		{
+			attributes = post_rt_attributes;
+		}
+	}
+	// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers> <Attributes> 
+
 	CodeBody    body       = NullCode;
 	CodeComment inline_cmt = NullCode;
 	if ( check( Tok_BraceCurly_Open ) )
@ -1480,18 +1516,20 @@ CodeFn parse_function_after_name(
 		eat( Tok_Statement_End );
 		
 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
-		// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers>; <InlineCmt>
+			inline_cmt = parse_comment(_ctx);
+			// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers> < = 0 or delete > ; <InlineCmt>
 	}
-	else
+
+
+	if (body == nullptr)
 	{
 		Token stmt_end = currtok;
 		eat( Tok_Statement_End );
-		// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers>;
+		// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers> <Attributes> < = 0 or delete > ;

 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
-			// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers>; <InlineCmt>
+			inline_cmt = parse_comment(_ctx);
+			// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers> < = 0 or delete > ; <InlineCmt>
 	}

 	StrBuilder
@ -1644,7 +1682,7 @@ CodeBody parse_global_nspace( CodeType which )
 			break;

 			case Tok_Comment:
-				member = cast(Code, parse_comment());
+				member = cast(Code, parse_comment(_ctx));
 			break;

 			case Tok_Decl_Class:
@ -1766,7 +1804,7 @@ CodeBody parse_global_nspace( CodeType which )

 			case Tok_Preprocess_Macro_Expr: 
 			{
-				if (tok_is_attribute(currtok))
+				if ( ! tok_is_attribute(currtok))
 				{
 					log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(_ctx->parser));
 					return InvalidCode;
@ -1780,7 +1818,7 @@ CodeBody parse_global_nspace( CodeType which )
 			GEN_DEFINE_ATTRIBUTE_TOKENS
 		#undef Entry
 			{
-				attributes = parse_attributes();
+				attributes = parse_attributes(_ctx);
 				// <Attributes>
 			}
 			//! Fallthrough intentional
@ -2049,11 +2087,20 @@ Token parse_identifier( bool* possible_member_function )

 	Macro* macro = lookup_macro(currtok.Text);
 	b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier );
+	b32 is_decarator         = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Identifier_Decorator );

 	// Typename can be: '::' <name>
 	// If that is the case first  option will be Tok_Access_StaticSymbol below
-	if (check(Tok_Identifier) || accept_as_identifier)
-		eat( Tok_Identifier );
+	if (check(Tok_Identifier) || accept_as_identifier) 
+	{
+		if (is_decarator) {
+			Code name_macro = parse_simple_preprocess(currtok.Type);
+			name.Text.Len   = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )name.Text.Ptr;
+		}	
+		else {
+			eat(Tok_Identifier);
+		}
+	}
 	// <Name>

 	parse_template_args( & name );
@ -2363,7 +2410,10 @@ CodeOperator parse_operator_after_ret_type(
 		case '(':
 		{
 			if ( currtok.Text.Ptr[1] == ')' )
+			{
 				op = Op_FunctionCall;
+				eat(Tok_Paren_Open);
+			}

 			else
 				op = Op_Invalid;
@ -2513,7 +2563,7 @@ CodeOperator parse_operator_after_ret_type(
 		// <ExportFlag> <Attributes> <Specifiers> <ReturnType> <Qualifier::...> operator <Op> ( <Parameters> ) <Specifiers>;

 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
+			inline_cmt = parse_comment(_ctx);
 			// <ExportFlag> <Attributes> <Specifiers> <ReturnType> <Qualifier::...> operator <Op> ( <Parameters> ) <Specifiers>; <InlineCmt>
 	}

@ -2544,7 +2594,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes
 	// <Attributes> <Specifiers> <ReturnType/ValueType>

 	// Thanks Unreal
-	CodeAttributes post_rt_attributes = parse_attributes();
+	CodeAttributes post_rt_attributes = parse_attributes(_ctx);
 	if (post_rt_attributes)
 	{
 		if (attributes)
@ -3260,7 +3310,7 @@ CodeVar parse_variable_after_name(
 {
 	push_scope();

-	Code array_expr    = parse_array_decl();
+	Code array_expr    = parse_array_decl(_ctx);
 	Code expr          = NullCode;
 	Code bitfield_expr = NullCode;

@ -3370,7 +3420,7 @@ CodeVar parse_variable_after_name(

 		// Check for inline comment : <type> <identifier> = <expression>; // <inline comment>
 		if ( left && ( currtok_noskip.Type == Tok_Comment ) && currtok_noskip.Line == stmt_end.Line ) {
-			inline_cmt = parse_comment();
+			inline_cmt = parse_comment(_ctx);
 			// <Attributes> <Specifiers> <ValueType> <Name> : <Expression>, ...; <InlineCmt>
 			// <Attributes> <Specifiers> <ValueType> <Name> = <Expression>, ...; <InlineCmt>
 			// <Attributes> <Specifiers> <ValueType> <Name> { <Expression> }, ...; <InlineCmt>
@ -3554,7 +3604,7 @@ CodeConstructor parser_parse_constructor( CodeSpecifiers specifiers )
 		// <Name> ( <Parameters> );

 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
+			inline_cmt = parse_comment(_ctx);
 		// <Name> ( <Parameters> ); <InlineCmt>
 	}

@ -3756,7 +3806,7 @@ CodeDestructor parser_parse_destructor( CodeSpecifiers specifiers )
 		// <Virtual Specifier> ~<Name>() <Pure Specifier>;

 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
+			inline_cmt = parse_comment(_ctx);
 		// <Virtual Specifier> ~<Name>() <Pure Specifier>; <InlineCmt>
 	}

@ -3810,7 +3860,7 @@ CodeEnum parser_parse_enum( bool inplace_def )
 		// enum class
 	}

-	attributes = parse_attributes();
+	attributes = parse_attributes(_ctx);
 	// enum <class> <Attributes>

 	if ( check( Tok_Identifier ) )
@ -3880,7 +3930,7 @@ CodeEnum parser_parse_enum( bool inplace_def )
 				break;

 				case Tok_Comment:
-					member = cast(Code, parse_comment());
+					member = cast(Code, parse_comment(_ctx));
 				break;

 				case Tok_Preprocess_Define:
@ -3995,7 +4045,7 @@ CodeEnum parser_parse_enum( bool inplace_def )
 		// enum <class> <Attributes> <Name> : <UnderlyingType> { <Body> };

 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
+			inline_cmt = parse_comment(_ctx);
 			// enum <class> <Attributes> <Name> : <UnderlyingType> { <Body> }; <InlineCmt>
 	}

@ -4173,7 +4223,7 @@ CodeFriend parser_parse_friend()
 		// friend <ReturnType> <Name> ( <Parameters> );

 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
+			inline_cmt = parse_comment(_ctx);
 		// friend <Type>; <InlineCmt>
 		// friend <ReturnType> <Name> ( <Parameters> ); <InlineCmt>
 	}
@ -4213,7 +4263,7 @@ CodeFn parser_parse_function()
 	}
 	// <export>

-	attributes = parse_attributes();
+	attributes = parse_attributes(_ctx);
 	// <export> <Attributes>

 	while ( left && tok_is_specifier(currtok) )
@ -4314,7 +4364,7 @@ CodeOperator parser_parse_operator()
 	}
 	// <export>

-	attributes = parse_attributes();
+	attributes = parse_attributes(_ctx);
 	// <export> <Attributes>

 	while ( left && tok_is_specifier(currtok) )
@ -4442,7 +4492,7 @@ CodeOpCast parser_parse_operator_cast( CodeSpecifiers specifiers )
 		// <Specifiers> <Qualifier> :: ... operator <UnderlyingType>() <const>;

 		if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-			inline_cmt = parse_comment();
+			inline_cmt = parse_comment(_ctx);
 			// <Specifiers> <Qualifier> :: ... operator <UnderlyingType>() <const>; <InlineCmt>
 	}

@ -4545,7 +4595,7 @@ CodeTemplate parser_parse_template()
 		Specifier specs_found[ 16 ] = { Spec_NumSpecifiers };
 		s32        NumSpecifiers = 0;

-		attributes = parse_attributes();
+		attributes = parse_attributes(_ctx);
 		// <export> template< <Parameters> > <Attributes>

 		// Specifiers Parsing
@ -4676,7 +4726,7 @@ CodeTypename parser_parse_type( bool from_template, bool* typedef_is_function )
 	ETypenameTag tag = Tag_None;

 	// Attributes are assumed to be before the type signature
-	CodeAttributes attributes = parse_attributes();
+	CodeAttributes attributes = parse_attributes(_ctx);
 	// <Attributes>

 	// Prefix specifiers
@ -4714,21 +4764,37 @@ CodeTypename parser_parse_type( bool from_template, bool* typedef_is_function )
 	else if ( currtok.Type == Tok_Decl_Class || currtok.Type == Tok_Decl_Enum || currtok.Type == Tok_Decl_Struct
 			|| currtok.Type == Tok_Decl_Union )
 	{
-		switch (currtok.Type) {
-			case Tok_Decl_Class  : tag = Tag_Class;  break;
-			case Tok_Decl_Enum   : tag = Tag_Enum;   break;
-			case Tok_Decl_Struct : tag = Tag_Struct; break;
-			case Tok_Decl_Union  : tag = Tag_Union;  break;
-			default:
-				break;
+		Token next = nexttok;
+
+		if (next.Type == Tok_Identifier)
+		{
+			switch (currtok.Type) {
+				case Tok_Decl_Class  : tag = Tag_Class;  break;
+				case Tok_Decl_Enum   : tag = Tag_Enum;   break;
+				case Tok_Decl_Struct : tag = Tag_Struct; break;
+				case Tok_Decl_Union  : tag = Tag_Union;  break;
+				default:
+					break;
+			}
+			eat( currtok.Type );
+			// <Attributes> <Specifiers> <class, enum, struct, union>
+
+			name = parse_identifier(nullptr);
+			// <Attributes> <Specifiers> <class, enum, struct, union> <Name> 
 		}
-		eat( currtok.Type );
-		// <Attributes> <Specifiers> <class, enum, struct, union>
+		else if  (next.Type == Tok_BraceCurly_Open)
+		{
+			name = currtok;
+			// We have an inplace definition, we need to consume that...

-		name = parse_identifier(nullptr);
+			// TODO(Ed): we need to add a way for AST_CodeTypename to track an implace definition..
+			b32 const inplace = true;
+			Code indplace_def = cast(Code, parser_parse_struct(inplace));

-		// name.Length = ( ( sptr )currtok.Text + currtok.Length ) - ( sptr )name.Text;
-		// eat( Tok_Identifier );
+			// For now we lose the structural information, 
+			name.Text.Len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )name.Text.Ptr;
+			// <Attributes> <Specifiers> <class, enum, struct, union> <inplace def>
+		}
 		_ctx->parser.Scope->Name = name.Text;
 		// <Attributes> <Specifiers> <class, enum, struct, union> <Name>
 	}
@ -5298,7 +5364,7 @@ CodeTypedef parser_parse_typedef()
 			return InvalidCode;
 		}

-		array_expr = parse_array_decl();
+		array_expr = parse_array_decl(_ctx);
 		// <UnderlyingType> + <ArrayExpr>
 	}

@ -5308,7 +5374,7 @@ CodeTypedef parser_parse_typedef()

 	CodeComment inline_cmt = NullCode;
 	if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line )
-		inline_cmt = parse_comment();
+		inline_cmt = parse_comment(_ctx);
 		// <ModuleFalgs> typedef <UnderlyingType> <Name> <ArrayExpr>; <InlineCmt>

 	CodeTypedef
@ -5361,7 +5427,7 @@ CodeUnion parser_parse_union( bool inplace_def )
 	eat( Tok_Decl_Union );
 	// <ModuleFlags> union

-	CodeAttributes attributes = parse_attributes();
+	CodeAttributes attributes = parse_attributes(_ctx);
 	// <ModuleFlags> union <Attributes>

 	Str name = { nullptr, 0 };
@ -5397,7 +5463,7 @@ CodeUnion parser_parse_union( bool inplace_def )
 				break;

 				case Tok_Comment:
-					member = cast(Code, parse_comment());
+					member = cast(Code, parse_comment(_ctx));
 				break;

 				// TODO(Ed) : Unions can have constructors and destructors
@ -5530,12 +5596,12 @@ CodeUsing parser_parse_using()

 	if ( ! is_namespace )
 	{
-		attributes = parse_attributes();
+		attributes = parse_attributes(_ctx);
 		// <ModuleFlags> using <Name> <Attributes>
 		
 		if ( bitfield_is_set( u32, currtok.Flags, TF_Assign ))
 		{
-			attributes = parse_attributes();
+			attributes = parse_attributes(_ctx);
 			// <ModuleFlags> using <Name> <Attributes>

 			eat( Tok_Operator );
@ -5544,7 +5610,7 @@ CodeUsing parser_parse_using()
 			type = parser_parse_type(parser_not_from_template, nullptr);
 			// <ModuleFlags> using <Name> <Attributes> = <UnderlyingType>

-			array_expr = parse_array_decl();
+			array_expr = parse_array_decl(_ctx);
 			// <UnderlyingType> + <ArrExpr>
 		}
 	}
@ -5555,7 +5621,7 @@ CodeUsing parser_parse_using()

 	CodeComment inline_cmt = NullCode;
 	if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) {
-		inline_cmt = parse_comment();
+		inline_cmt = parse_comment(_ctx);
 	}
 	// <ModuleFlags> using <namespace> <Attributes> <Name> = <UnderlyingType>; <InlineCmt>

@ -5607,7 +5673,7 @@ CodeVar parser_parse_variable()
 	}
 	// <ModuleFlags>

-	attributes = parse_attributes();
+	attributes = parse_attributes(_ctx);
 	// <ModuleFlags> <Attributes>

 	while ( left && tok_is_specifier(currtok) )
--- a/base/components/parser_types.hpp
+++ b/base/components/parser_types.hpp
@ -1,7 +1,7 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #pragma once
 #include "types.hpp"
-#include "gen/ecode.hpp"
+#include "gen/ecodetypes.hpp"
 #include "gen/eoperator.hpp"
 #include "gen/especifier.hpp"
 #include "gen/etoktype.hpp"
@ -91,6 +91,27 @@ bool tok_is_end_definition(Token tok) {

 StrBuilder tok_to_strbuilder(Token tok);

+struct TokenSlice
+{
+	Token* Ptr;
+	s32    Num;
+
+#if GEN_COMPILER_CPP
+	forceinline operator Token* ()               const { return Ptr; }
+	forceinline Token& operator[]( ssize index ) const { return Ptr[index]; }
+#endif
+};
+
+forceinline 
+Str token_range_to_str(Token start, Token end)
+{
+	Str result = {
+		start.Text.Ptr,
+		(scast(sptr, rcast(uptr, end.Text.Ptr)) + end.Text.Len) - scast(sptr, rcast(uptr, start.Text.Ptr))
+	};
+	return result;
+}
+
 struct TokArray 
 {
 	Array(Token) Arr;
@ -104,23 +125,21 @@ struct LexContext
 	char const*     scanner;
 	s32             line;
 	s32             column;
-	// StringTable     defines;
 	Token           token;
 };

-struct StackNode
+struct LexedInfo
 {
-	StackNode* Prev;
-
-	Token* Start;
-	Str    Name;          // The name of the AST node (if parsed)
-	Str    ProcName;    // The name of the procedure
+	Str        text;
+	TokenSlice tokens;
 };

+typedef struct ParseStackNode ParseStackNode;
+
 struct ParseContext
 {
-	TokArray   Tokens;
-	StackNode* Scope;
+	TokArray        Tokens;
+	ParseStackNode* Scope;
 };

 enum MacroType : u16
@ -168,26 +187,36 @@ Str macrotype_to_str( MacroType type )

 enum EMacroFlags : u16
 {
-	MF_Functional          = bit(0), // Macro has parameters (args expected to be passed)
-	MF_Expects_Body        = bit(1), // Expects to assign a braced scope to its body.
+	// Macro has parameters (args expected to be passed)
+	MF_Functional          = bit(0), 
+
+	// Expects to assign a braced scope to its body.
+	MF_Expects_Body        = bit(1), 

 	// lex__eat wil treat this macro as an identifier if the parser attempts to consume it as one.
-	//  ^^^ This is a kludge because we don't support push/pop macro pragmas rn.
+	// This is a kludge because we don't support push/pop macro pragmas rn.
 	MF_Allow_As_Identifier = bit(2), 

+	// When parsing identifiers, it will allow the consumption of the macro parameters (as its expected to be a part of constructing the identifier)
+	// Example of a decarator macro from stb_sprintf.h: 
+	// STBSP__PUBLICDEC int STB_SPRINTF_DECORATE(sprintf)(char* buf, char const *fmt, ...) STBSP__ATTRIBUTE_FORMAT(2,3);
+	//                       ^^ STB_SPRINTF_DECORATE is decorating sprintf
+	MF_Identifier_Decorator = bit(3), 
+
 	// lex__eat wil treat this macro as an attribute if the parser attempts to consume it as one.
-	//  ^^^ This a kludge because unreal has a macro that behaves as both a 'statement' and an attribute (UE_DEPRECATED, PRAGMA_ENABLE_DEPRECATION_WARNINGS, etc)
+	// This a kludge because unreal has a macro that behaves as both a 'statement' and an attribute (UE_DEPRECATED, PRAGMA_ENABLE_DEPRECATION_WARNINGS, etc)
 	// TODO(Ed): We can keep the MF_Allow_As_Attribute flag for macros, however, we need to add the ability of AST_Attributes to chain themselves.
 	// Its thats already a thing in the standard language anyway
 	// & it would allow UE_DEPRECATED, (UE_PROPERTY / UE_FUNCTION) to chain themselves as attributes of a resolved member function/variable definition
-	MF_Allow_As_Attribute  = bit(3),
+	MF_Allow_As_Attribute  = bit(4),

 	// When a macro is encountered after attributes and specifiers while parsing a function, or variable:
-	// It will consume the macro and treat it as resolving the definition. (Yes this is for Unreal Engine)
+	// It will consume the macro and treat it as resolving the definition.
 	// (MUST BE OF MT_Statement TYPE)
-	MF_Allow_As_Definition = bit(4),
+	MF_Allow_As_Definition = bit(5),

-	MF_Allow_As_Specifier = bit(5), // Created for Unreal's PURE_VIRTUAL
+	// Created for Unreal's PURE_VIRTUAL
+	MF_Allow_As_Specifier = bit(6),

 	MF_Null           = 0,
 	MF_UnderlyingType = GEN_U16_MAX,
--- a/base/components/static_data.cpp
+++ b/base/components/static_data.cpp
@ -1,6 +1,6 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #pragma once
-#include "../gen.hpp"
+#include "interface.hpp"
 #endif

 #pragma region StaticData
--- a/base/components/types.hpp
+++ b/base/components/types.hpp
@ -1,6 +1,18 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #pragma once
-#include "header_start.hpp"
+#include "dependencies/platform.hpp"
+#include "dependencies/macros.hpp"
+#include "dependencies/basic_types.hpp"
+#include "dependencies/debug.hpp"
+#include "dependencies/memory.hpp"
+#include "dependencies/string_ops.hpp"
+#include "dependencies/printing.hpp"
+#include "dependencies/containers.hpp"
+#include "dependencies/hashing.hpp"
+#include "dependencies/strings.hpp"
+#include "dependencies/filesystem.hpp"
+#include "dependencies/timing.hpp"
+#include "dependencies/parsing.hpp"
 #endif

 /*
@ -19,16 +31,6 @@

 */

-using LogFailType = ssize(*)(char const*, ...);
-
-// By default this library will either crash or exit if an error is detected while generating codes.
-// Even if set to not use GEN_FATAL, GEN_FATAL will still be used for memory failures as the library is unusable when they occur.
-#ifdef GEN_DONT_USE_FATAL
-	#define log_failure log_fmt
-#else
-	#define log_failure GEN_FATAL
-#endif
-
 enum AccessSpec : u32
 {
 	AccessSpec_Default,
--- a/base/dependencies/debug.hpp
+++ b/base/dependencies/debug.hpp
@ -1,9 +1,6 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #	pragma once
-#	include "dependencies/platform.hpp"
-#	include "dependencies/macros.hpp"
 #	include "basic_types.hpp"
-#	include "macros.hpp"
 #endif

 #pragma region Debug
--- a/base/dependencies/filesystem.hpp
+++ b/base/dependencies/filesystem.hpp
@ -187,7 +187,7 @@ struct FileContents
 {
 	AllocatorInfo allocator;
 	void*         data;
-	ssize            size;
+	ssize         size;
 };

 constexpr b32 file_zero_terminate    = true;
--- a/base/dependencies/macros.hpp
+++ b/base/dependencies/macros.hpp
@ -198,21 +198,16 @@
 #ifndef        forceinline
 #	if GEN_COMPILER_MSVC
 #		define forceinline __forceinline
-#		define neverinline __declspec( noinline )
 #	elif GEN_COMPILER_GCC
 #		define forceinline inline __attribute__((__always_inline__))
-#		define neverinline __attribute__( ( __noinline__ ) )
 #	elif GEN_COMPILER_CLANG
 #	if __has_attribute(__always_inline__)
 #		define forceinline inline __attribute__((__always_inline__))
-#		define neverinline __attribute__( ( __noinline__ ) )
 #	else
 #		define forceinline
-#		define neverinline
 #	endif
 #	else
 #		define forceinline
-#		define neverinline
 #	endif
 #endif

@ -303,10 +298,20 @@
 #	define GEN_PARAM_DEFAULT
 #endif

-#if GEN_COMPILER_CPP
-    #define struct_init(type, value) {value}
-#else
-    #define struct_init(type, value) {value}
+#ifndef struct_init
+#	if GEN_COMPILER_CPP
+#		define struct_init(type, value) value
+#	else
+#		define struct_init(type, value) (type) value
+#	endif
+#endif
+
+#ifndef struct_zero
+#	if GEN_COMPILER_CPP
+#		define struct_zero(type) {}
+#	else
+#		define struct_zero(type) {0}
+#	endif
 #endif

 #if 0
@ -319,4 +324,12 @@
 #	define GEN_OPITMIZE_MAPPINGS_END
 #endif

+#ifndef get_optional
+#	if GEN_COMPILER_C
+#		define get_optional(opt) opt ? *opt : (typeof(*opt)){0}
+#	else
+#		define get_optional(opt) opt
+#	endif
+#endif
+
 #pragma endregion Macros
--- a/base/dependencies/memory.hpp
+++ b/base/dependencies/memory.hpp
@ -134,12 +134,6 @@ GEN_API void* heap_allocator_proc( void* allocator_data, AllocType type, ssize s
 //! The heap allocator backed by operating system's memory manager.
 constexpr AllocatorInfo heap( void ) { AllocatorInfo allocator = { heap_allocator_proc, nullptr }; return allocator; }

-//! Helper to allocate memory using heap allocator.
-#define malloc( sz ) alloc( heap(), sz )
-
-//! Helper to free memory allocated by heap allocator.
-#define mfree( ptr ) free( heap(), ptr )
-
 struct VirtualMemory
 {
 	void*  data;
@ -185,6 +179,8 @@ void  arena_check         (Arena* arena);
 void  arena_free          (Arena* arena);
 ssize arena_size_remaining(Arena* arena, ssize alignment);

+// TODO(Ed): Add arena_pos, arena_pop, and arena_pop_to
+
 struct Arena
 {
 	AllocatorInfo Backing;
--- a/base/dependencies/printing.cpp
+++ b/base/dependencies/printing.cpp
@ -1,6 +1,6 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #	pragma once
-#	include "strbuilder_ops.cpp"
+#	include "string_ops.cpp"
 #endif

 #pragma region Printing
--- a/base/dependencies/printing.hpp
+++ b/base/dependencies/printing.hpp
@ -1,6 +1,6 @@
 #ifdef INTELLISENSE_DIRECTIVES
 #	pragma once
-#	include "strbuilder_ops.hpp"
+#	include "string_ops.hpp"
 #endif

 #pragma region Printing
@ -26,17 +26,4 @@ GEN_API ssize  c_str_fmt_file_va   ( FileInfo* f, char const* fmt, va_list va );
 constexpr
 char const* Msg_Invalid_Value = "INVALID VALUE PROVIDED";

-inline
-ssize log_fmt(char const* fmt, ...)
-{
-	ssize res;
-	va_list va;
-
-	va_start(va, fmt);
-	res = c_str_fmt_out_va(fmt, va);
-	va_end(va);
-
-	return res;
-}
-
 #pragma endregion Printing
--- a/base/dependencies/strings.hpp
+++ b/base/dependencies/strings.hpp
@ -320,7 +320,7 @@ inline
 StrBuilder strbuilder_fmt_buf(AllocatorInfo allocator, char const* fmt, ...)
 {
 	local_persist thread_local
-	PrintF_Buffer buf = struct_init(PrintF_Buffer, {0});
+	PrintF_Buffer buf = struct_zero(PrintF_Buffer);

 	va_list va;
 	va_start(va, fmt);
--- a/base/gen.hpp
+++ b/base/gen.hpp
@ -11,9 +11,6 @@
 #include "helpers/push_ignores.inline.hpp"
 #include "components/header_start.hpp"

-// Has container defines pushed
-#include "gen.dep.hpp"
-
 GEN_NS_BEGIN

 #include "components/types.hpp"
--- a/gen_c_library/c_library.cpp
+++ b/gen_c_library/c_library.cpp
@ -1159,6 +1159,8 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 				Str actual_name = { fn->Name.Ptr + prefix.Len, fn->Name.Len  - prefix.Len };
 				Str new_name    = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "def__%S", actual_name ).to_str();

+				opt_param->ValueType->Specs = def_specifier(Spec_Ptr);
+
 				// Resolve define's arguments
 				b32    has_args   = fn->Params->NumEntries > 1;
 				StrBuilder params_str = StrBuilder::make_reserve(_ctx->Allocator_Temp, 32);
@ -1172,10 +1174,10 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 				}
 				char const* tmpl_fn_macro = nullptr;
 				if (params_str.length() > 0 ) {
-					tmpl_fn_macro= "#define <def_name>( <params> ... ) <def__name>( <params> (<opts_type>) { __VA_ARGS__ } )\n";
+					tmpl_fn_macro= "#define <def_name>( <params> ... ) <def__name>( <params> & (<opts_type>) { __VA_ARGS__ } )\n";
 				}
 				else {
-					tmpl_fn_macro= "#define <def_name>( ... ) <def__name>( (<opts_type>) { __VA_ARGS__ } )\n";
+					tmpl_fn_macro= "#define <def_name>( ... ) <def__name>( & (<opts_type>) { __VA_ARGS__ } )\n";
 				}
 				Code fn_macro = untyped_str(token_fmt(
 						"def_name",  fn->Name
@ -1504,6 +1506,7 @@ R"(#define <interface_name>( code ) _Generic( (code), \
 				Str actual_name = { fn->Name.Ptr + prefix.Len, fn->Name.Len  - prefix.Len };
 				Str new_name    = StrBuilder::fmt_buf(_ctx->Allocator_Temp, "def__%S", actual_name ).to_str();
 				fn->Name = cache_str(new_name);
+				opt_param->ValueType->Specs = def_specifier(Spec_Ptr);
 			}
 			src_upfront.append(fn);
 		}
--- a/gen_c_library/c_library.refactor
+++ b/gen_c_library/c_library.refactor
@ -53,6 +53,7 @@ word enum_underlying,              gen_enum_underlying
 word nullptr,                      gen_nullptr
 word struct_init,                  gen_struct_init
 word hash,                         gen_hash
+word txt,                          gen_txt

 // Basic Types

@ -410,6 +411,8 @@ namespace var_,        gen_var_

 word _ctx, gen__ctx

+word get_context, gen_get_context
+
 word init,   gen_init
 word deinit, gen_deinit
 word reset,  gen_reset
@ -532,7 +535,7 @@ namespace Lexer_, gen_Lexer_
 word LexContext, gen_LexContext
 word lex,        gen_lex

-word StackNode,    gen_StackNode
+word ParseStackNode,    gen_ParseStackNode
 word ParseContext, gen_ParseContext

 // namespace parse_, gen_parse_
--- a/scripts/helpers/vendor_toolchain.ps1
+++ b/scripts/helpers/vendor_toolchain.ps1
@ -404,7 +404,7 @@ if ( $vendor -match "msvc" )
 	$flag_optimize_intrinsics		  = '/Oi'
 	$flag_optimized_debug_forceinline = '/d2Obforceinline'
 	$flag_optimized_debug			  = '/Zo'
-	$flag_
+	# $flag_
 	# $flag_out_name                    = '/OUT:'
 	$flag_path_interm                 = '/Fo'
 	$flag_path_debug                  = '/Fd'
Author	SHA1	Message	Date
Ed_	441a46daaa	changing to just pass lib Context instead of ParseContext. If I want to make it more granular I can later.	2025-02-19 15:48:56 -05:00
Ed_	26623075ad	started to work on modularizing parser code paths	2025-02-19 12:10:11 -05:00
Ed_	7ea90ef349	intellisense still broken...	2025-02-19 11:22:00 -05:00
Ed_	6d531fdf97	Fixing intellisense	2025-02-19 11:08:24 -05:00
Ed_	3b81eea688	Doing some initial prep for parser overhaul	2025-02-19 10:50:55 -05:00
Ed_	844d431e1c	Setup Context's logger and fallback. (Not yet used in implementation).	2025-02-19 10:50:36 -05:00
Ed_	727b54c341	Added CodeSpecifiers::index_of for cpp, fix token bug with () operator in parse_operator_after_ret_type	2025-02-15 21:22:33 -05:00
Ed_	ef72d27f3e	made optionals for c11 actually work...	2025-02-11 19:24:42 -05:00
Ed_	75b1d42cca	changes based off of metadesk genc_c11	2025-02-11 18:59:09 -05:00