preparing to revamp lexer

base/components/interface.cpp

@@ -3,8 +3,8 @@
 #include "code_serialization.cpp"
 #endif
 
-internal void parser_init();
+internal void parser_init(Context* ctx);
-internal void parser_deinit();
+internal void parser_deinit(Context* ctx);
 
 internal
 void* fallback_allocator_proc( void* allocator_data, AllocType type, ssize size, ssize alignment, void* old_memory, ssize old_size, u64 flags )
@@ -343,7 +343,7 @@ void init(Context* ctx)
 	}
 
 	define_constants();
-	parser_init();
+	parser_init(ctx);
 
 	++ context_counter;
 }
@@ -392,7 +392,7 @@ void deinit(Context* ctx)
 		while ( left--, left );
 		array_free( ctx->Fallback_AllocatorBuckets);
 	}
-	parser_deinit();
+	parser_deinit(ctx);
 
 	if (_ctx == ctx) 
 		_ctx = nullptr;

base/components/interface.hpp

@@ -15,38 +15,6 @@
   \▓▓▓▓▓▓  \▓▓▓▓▓▓▓\▓▓   \▓▓     \▓▓▓▓▓▓\▓▓   \▓▓   \▓▓▓▓  \▓▓▓▓▓▓▓\▓▓      \▓▓       \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓
 */
 
-enum LogLevel //: u32
-{
-	LL_Null,
-	LL_Note,
-	LL_Warning,
-	LL_Error,
-	LL_Fatal,
-	LL_UnderlyingType = GEN_U32_MAX,
-};
-typedef enum LogLevel LogLevel;
-
-Str loglevel_to_str(LogLevel level) 
-{
-	local_persist
-	Str lookup[] = {
-		{ "Null",    sizeof("Null")  - 1 },
-		{ "Note",    sizeof("Note")  - 1 },
-		{ "Warning", sizeof("Info")  - 1 },
-		{ "Error",   sizeof("Error") - 1 },
-		{ "Fatal",   sizeof("Fatal") - 1 },
-	};
-	return lookup[level];
-}
-
-struct LogEntry
-{
-	Str      msg;
-	LogLevel level;
-};
-
-typedef void LoggerProc(LogEntry entry);
-
 // Note(Ed): This is subject to heavily change 
 // with upcoming changes to the library's fallback (default) allocations strategy;
 // and major changes to lexer/parser context usage.
@@ -411,7 +379,7 @@ struct ParseMessage
 {
 	ParseMessage*   Next;
 	ParseStackNode* Scope;
-	Str             Log;
+	Str             Content;
 	LogLevel        Level;
 };
 

base/components/interface.parsing.cpp

@@ -10,6 +10,9 @@
 
 ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts)
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	TokArray toks;
 	if (lexed.tokens.Num == 0 && lexed.tokens.Ptr == nullptr) {
 		check_parse_args(lexed.text);
@@ -23,31 +26,44 @@ ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts)
 
 	// TODO(Ed): ParseInfo should be set to the parser context.
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	push_scope();
-	CodeBody result = parse_global_nspace(CT_Global_Body);
-	parser_pop(& _ctx->parser);
 
+	ParseStackNode scope = NullScope;
+	parser_push(& ctx->parser, & scope);
+
+	CodeBody result = parse_global_nspace(ctx,CT_Global_Body);
+
+	parser_pop(& ctx->parser);
 	return info;
 }
 
 CodeClass parse_class( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+		ctx->parser.Tokens = toks;
-	push_scope();
+
-	CodeClass result = (CodeClass) parse_class_struct( Tok_Decl_Class, parser_not_inplace_def );
+	ParseStackNode scope = NullScope;
-	parser_pop(& _ctx->parser);
+	parser_push(& ctx->parser, & scope);
+
+	CodeClass result = (CodeClass) parse_class_struct( ctx, Tok_Decl_Class, parser_not_inplace_def );
+
+	parser_pop(& ctx->parser);
 	return result;
 }
 
-CodeConstructor parse_constructor( Str def )
+CodeConstructor parse_constructor(Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
@@ -80,8 +96,8 @@ CodeConstructor parse_constructor( Str def )
 				break;
 
 			default :
-				log_failure( "Invalid specifier %s for variable\n%S", spec_to_str( spec ), parser_to_strbuilder(_ctx->parser) );
+				log_failure( "Invalid specifier %s for variable\n%S", spec_to_str( spec ), parser_to_strbuilder(ctx->parser, ctx->Allocator_Temp) );
-				parser_pop(& _ctx->parser);
+				parser_pop(& ctx->parser);
 				return InvalidCode;
 		}
 
@@ -100,28 +116,38 @@ CodeConstructor parse_constructor( Str def )
 		// <specifiers> ...
 	}
 
-	_ctx->parser.Tokens         = toks;
+	ctx->parser.Tokens     = toks;
-	CodeConstructor result = parser_parse_constructor( specifiers );
+	CodeConstructor result = parser_parse_constructor(ctx, specifiers);
 	return result;
 }
 
 CodeDefine parse_define( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+	
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	push_scope();
+
-	CodeDefine result = parser_parse_define();
+	ParseStackNode scope = NullScope;
-	parser_pop(& _ctx->parser);
+	parser_push(& ctx->parser, & scope);
+
+	CodeDefine result = parser_parse_define(ctx);
+
+	parser_pop(& ctx->parser);
 	return result;
 }
 
 CodeDestructor parse_destructor( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
@@ -131,210 +157,269 @@ CodeDestructor parse_destructor( Str def )
 	// TODO(Ed): Destructors can have prefix attributes
 	// TODO(Ed): Destructors can have virtual
 
-	_ctx->parser.Tokens        = toks;
+	ctx->parser.Tokens   = toks;
-	CodeDestructor result = parser_parse_destructor(NullCode);
+	CodeDestructor result = parser_parse_destructor(ctx, NullCode);
 	return result;
 }
 
 CodeEnum parse_enum( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
+	ParseStackNode scope = NullScope;
+	parser_push(& ctx->parser, & scope);
+
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 	{
-		parser_pop(& _ctx->parser);
+		parser_pop(& ctx->parser);
 		return InvalidCode;
 	}
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_enum( parser_not_inplace_def);
+	return parser_parse_enum(ctx, parser_not_inplace_def);
 }
 
 CodeBody parse_export_body( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_export_body();
+	return parser_parse_export_body(ctx);
 }
 
 CodeExtern parse_extern_link( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_extern_link();
+	return parser_parse_extern_link(ctx);
 }
 
 CodeFriend parse_friend( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_friend();
+	return parser_parse_friend(ctx);
 }
 
 CodeFn parse_function( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return (CodeFn) parser_parse_function();
+	return (CodeFn) parser_parse_function(ctx);
 }
 
 CodeBody parse_global_body( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	push_scope();
+
-	CodeBody result = parse_global_nspace( CT_Global_Body );
+	ParseStackNode scope = NullScope;
-	parser_pop(& _ctx->parser);
+	parser_push(& ctx->parser, & scope);
+
+	CodeBody result = parse_global_nspace(ctx, CT_Global_Body );
+
+	parser_pop(& ctx->parser);
 	return result;
 }
 
 CodeNS parse_namespace( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_namespace();
+	return parser_parse_namespace(ctx);
 }
 
 CodeOperator parse_operator( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return (CodeOperator) parser_parse_operator();
+	return (CodeOperator) parser_parse_operator(ctx);
 }
 
 CodeOpCast parse_operator_cast( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_operator_cast(NullCode);
+	return parser_parse_operator_cast(ctx, NullCode);
 }
 
 CodeStruct parse_struct( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	push_scope();
+
-	CodeStruct result = (CodeStruct) parse_class_struct( Tok_Decl_Struct, parser_not_inplace_def );
+	ParseStackNode scope = NullScope;
-	parser_pop(& _ctx->parser);
+	parser_push(& ctx->parser, & scope);
+
+	CodeStruct result = (CodeStruct) parse_class_struct( ctx, Tok_Decl_Struct, parser_not_inplace_def );
+
+	parser_pop(& ctx->parser);
 	return result;
 }
 
 CodeTemplate parse_template( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_template();
+	return parser_parse_template(ctx);
 }
 
 CodeTypename parse_type( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_type( parser_not_from_template, nullptr);
+	return parser_parse_type( ctx, parser_not_from_template, nullptr);
 }
 
 CodeTypedef parse_typedef( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_typedef();
+	return parser_parse_typedef(ctx);
 }
 
 CodeUnion parse_union( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_union( parser_not_inplace_def);
+	return parser_parse_union(ctx, parser_not_inplace_def);
 }
 
 CodeUsing parse_using( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_using();
+	return parser_parse_using(ctx);
 }
 
 CodeVar parse_variable( Str def )
 {
+	// TODO(Ed): Lift this.
+	Context* ctx = _ctx;
+
 	check_parse_args( def );
 
 	TokArray toks = lex( def );
 	if ( toks.Arr == nullptr )
 		return InvalidCode;
 
-	_ctx->parser.Tokens = toks;
+	ctx->parser.Tokens = toks;
-	return parser_parse_variable();
+	return parser_parse_variable(ctx);
 }
 
 // Undef helper macros

base/components/lexer.cpp

@@ -17,7 +17,7 @@ StrBuilder tok_to_strbuilder(Token tok)
 	return result;
 }
 
-bool lex__eat( TokArray* self, TokType type );
+bool lex__eat(Context* ctx, TokArray* self, TokType type );
 
 Token* lex_current(TokArray* self, bool skip_formatting )
 {
@@ -568,6 +568,23 @@ void lex_found_token( LexContext* ctx )
 
 // TODO(Ed): We need to to attempt to recover from a lex failure?
 
+neverinline
+LexedInfo lex_WIP(Context* lib_ctx, Str content)
+{
+	LexedInfo result = struct_zero();
+	result.text   = content;
+	result.tokens = array_init_reserve(Token, ctx->Allocator_DyanmicContainers, ctx->InitSize_LexerTokens );
+
+	LexContext c = struct_zero(); LexContext* ctx = & c;
+	c.content = content;
+	c.left    = content.Len;
+	c.scanner = content.Ptr;
+
+	
+
+	return result;
+}
+
 neverinline
 // void lex( Array<Token> tokens, Str content )
 TokArray lex( Str content )

base/components/parser_types.hpp

@@ -128,10 +128,19 @@ struct LexContext
 	Token           token;
 };
 
+typedef struct LexerMessage LexerMessage;
+struct LexerMessage
+{
+	LexerMessage* next;
+	Str           content;
+	LogLevel      level;
+};
+
 struct LexedInfo
 {
-	Str        text;
+	LexerMessage messages;
-	TokenSlice tokens;
+	Str          text;
+	TokenSlice   tokens;
 };
 
 typedef struct ParseStackNode ParseStackNode;

base/components/types.hpp

@@ -31,6 +31,38 @@
 
 */
 
+enum LogLevel //: u32
+{
+	LL_Null,
+	LL_Note,
+	LL_Warning,
+	LL_Error,
+	LL_Fatal,
+	LL_UnderlyingType = GEN_U32_MAX,
+};
+typedef enum LogLevel LogLevel;
+
+Str loglevel_to_str(LogLevel level) 
+{
+	local_persist
+	Str lookup[] = {
+		{ "Null",    sizeof("Null")  - 1 },
+		{ "Note",    sizeof("Note")  - 1 },
+		{ "Warning", sizeof("Info")  - 1 },
+		{ "Error",   sizeof("Error") - 1 },
+		{ "Fatal",   sizeof("Fatal") - 1 },
+	};
+	return lookup[level];
+}
+
+struct LogEntry
+{
+	Str      msg;
+	LogLevel level;
+};
+
+typedef void LoggerProc(LogEntry entry);
+
 enum AccessSpec : u32
 {
 	AccessSpec_Default,

base/dependencies/containers.hpp

@@ -26,10 +26,8 @@ template <class TType> using TRemovePtr = typename RemovePtr<TType>::Type;
 
 struct ArrayHeader;
 
-#if GEN_COMPILER_CPP
+template<class Type> struct Array;
-	template<class Type> struct Array;
+#define get_array_underlying_type(array) typename TRemovePtr<typeof(array)>:: DataType
-#	define get_array_underlying_type(array) typename TRemovePtr<typeof(array)>:: DataType
-#endif
 
 usize array_grow_formula(ssize value);
 
@@ -59,12 +57,12 @@ struct ArrayHeader {
 	usize         Num;
 };
 
-#if GEN_COMPILER_CPP
 template<class Type>
 struct Array
 {
 	Type* Data;
 
+#if ! GEN_C_LIKE_CPP
 #pragma region Member Mapping
 	forceinline static Array  init(AllocatorInfo allocator)                         { return array_init<Type>(allocator); }
 	forceinline static Array  init_reserve(AllocatorInfo allocator, ssize capacity) { return array_init_reserve<Type>(allocator, capacity); }
@@ -88,6 +86,7 @@ struct Array
 	forceinline bool         resize(usize num)                                 { return array_resize<Type>(this, num); }
 	forceinline bool         set_capacity(usize new_capacity)                  { return array_set_capacity<Type>(this, new_capacity); }
 #pragma endregion Member Mapping
+#endif
 
 	forceinline operator Type*()             { return Data; }
 	forceinline operator Type const*() const { return Data; }
@@ -99,9 +98,8 @@ struct Array
 
 	using DataType = Type;
 };
-#endif
 
-#if GEN_COMPILER_CPP && 0
+#if 0
 template<class Type> bool append(Array<Type>& array, Array<Type> other)                         { return append( & array, other ); }
 template<class Type> bool append(Array<Type>& array, Type value)                                { return append( & array, value ); }
 template<class Type> bool append(Array<Type>& array, Type* items, usize item_num)               { return append( & array, items, item_num ); }