preparing to revamp lexer

base/components/interface.hpp

@@ -15,38 +15,6 @@
   \▓▓▓▓▓▓  \▓▓▓▓▓▓▓\▓▓   \▓▓     \▓▓▓▓▓▓\▓▓   \▓▓   \▓▓▓▓  \▓▓▓▓▓▓▓\▓▓      \▓▓       \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓ \▓▓▓▓▓▓▓
 */
 
-enum LogLevel //: u32
-{
-	LL_Null,
-	LL_Note,
-	LL_Warning,
-	LL_Error,
-	LL_Fatal,
-	LL_UnderlyingType = GEN_U32_MAX,
-};
-typedef enum LogLevel LogLevel;
-
-Str loglevel_to_str(LogLevel level) 
-{
-	local_persist
-	Str lookup[] = {
-		{ "Null",    sizeof("Null")  - 1 },
-		{ "Note",    sizeof("Note")  - 1 },
-		{ "Warning", sizeof("Info")  - 1 },
-		{ "Error",   sizeof("Error") - 1 },
-		{ "Fatal",   sizeof("Fatal") - 1 },
-	};
-	return lookup[level];
-}
-
-struct LogEntry
-{
-	Str      msg;
-	LogLevel level;
-};
-
-typedef void LoggerProc(LogEntry entry);
-
 // Note(Ed): This is subject to heavily change 
 // with upcoming changes to the library's fallback (default) allocations strategy;
 // and major changes to lexer/parser context usage.
@@ -411,7 +379,7 @@ struct ParseMessage
 {
 	ParseMessage*   Next;
 	ParseStackNode* Scope;
-	Str             Log;
+	Str             Content;
 	LogLevel        Level;
 };
 

base/components/lexer.cpp

@@ -568,6 +568,23 @@ void lex_found_token( LexContext* ctx )
 
 // TODO(Ed): We need to to attempt to recover from a lex failure?
 
+neverinline
+LexedInfo lex_WIP(Context* lib_ctx, Str content)
+{
+	LexedInfo result = struct_zero();
+	result.text   = content;
+	result.tokens = array_init_reserve(Token, ctx->Allocator_DyanmicContainers, ctx->InitSize_LexerTokens );
+
+	LexContext c = struct_zero(); LexContext* ctx = & c;
+	c.content = content;
+	c.left    = content.Len;
+	c.scanner = content.Ptr;
+
+	
+
+	return result;
+}
+
 neverinline
 // void lex( Array<Token> tokens, Str content )
 TokArray lex( Str content )

base/components/parser_types.hpp

@@ -128,10 +128,19 @@ struct LexContext
 	Token           token;
 };
 
+typedef struct LexerMessage LexerMessage;
+struct LexerMessage
+{
+	LexerMessage* next;
+	Str           content;
+	LogLevel      level;
+};
+
 struct LexedInfo
 {
-	Str        text;
+	LexerMessage messages;
-	TokenSlice tokens;
+	Str          text;
+	TokenSlice   tokens;
 };
 
 typedef struct ParseStackNode ParseStackNode;

base/components/types.hpp

@@ -31,6 +31,38 @@
 
 */
 
+enum LogLevel //: u32
+{
+	LL_Null,
+	LL_Note,
+	LL_Warning,
+	LL_Error,
+	LL_Fatal,
+	LL_UnderlyingType = GEN_U32_MAX,
+};
+typedef enum LogLevel LogLevel;
+
+Str loglevel_to_str(LogLevel level) 
+{
+	local_persist
+	Str lookup[] = {
+		{ "Null",    sizeof("Null")  - 1 },
+		{ "Note",    sizeof("Note")  - 1 },
+		{ "Warning", sizeof("Info")  - 1 },
+		{ "Error",   sizeof("Error") - 1 },
+		{ "Fatal",   sizeof("Fatal") - 1 },
+	};
+	return lookup[level];
+}
+
+struct LogEntry
+{
+	Str      msg;
+	LogLevel level;
+};
+
+typedef void LoggerProc(LogEntry entry);
+
 enum AccessSpec : u32
 {
 	AccessSpec_Default,