Doing some initial prep for parser overhaul

base/components/ast.hpp

@@ -406,7 +406,8 @@ struct AST
 				Code  PostNameMacro;    // Only used with parameters for specifically UE_REQUIRES (Thanks Unreal)
 			};
 		};
-		StrCached  Content;          // Attributes, Comment, Execution, Include
+		StrCached  Content;      // Attributes, Comment, Execution, Include
+		// TokenSlice Content;   // TODO(Ed): Use a token slice for content
 		struct {
 			Specifier  ArrSpecs[AST_ArrSpecs_Cap]; // Specifiers
 			Code       NextSpecs;              // Specifiers; If ArrSpecs is full, then NextSpecs is used.
@@ -422,7 +423,7 @@ struct AST
 		Code Next;
 		Code Back;
 	};
-	Token*            Token; // Reference to starting token, only available if it was derived from parsing.
+	Token*            Token; // Reference to starting token, only available if it was derived from parsing. // TODO(Ed): Change this to a token slice.
 	Code              Parent;
 	CodeType          Type;
 //	CodeFlag          CodeFlags;

base/components/interface.hpp

@@ -391,37 +391,41 @@ forceinline CodeBody         def_union_body      ( s32 num, Code* codes )
 
 #pragma region Parsing
 
-#if 0
+struct ParseStackNode
-struct StackNode
 {
-	StackNode* Prev;
+	ParseStackNode* Prev;
 
-	Token Start;
+	TokenSlice tokens;
-	Token Name;       // The name of the AST node (if parsed)
+	Token*     Start;
-	Str  FailedProc; // The name of the procedure that failed
+	Str        Name;     // The name of the AST node (if parsed)
+	Str        ProcName; // The name of the procedure
+	Code       code;     // Relevant AST node
+	// TODO(Ed): When an error occurs, the parse stack is not released and instead the scope is left dangling.
 };
-// Stack nodes are allocated the error's allocator
 
-struct Error
+struct ParseMessage
 {
-	StrBuilder     message;
+	ParseMessage*   Next;
-	StackNode* context_stack;
+	ParseStackNode* Scope;
+	Str             Log;
+	LogLevel        Level;
 };
 
 struct ParseInfo
 {
-	Arena FileMem;
+	ParseMessage* messages;
-	Arena TokMem;
+	LexedInfo     lexed;
-	Arena CodeMem;
+	Code          result;
-
-	FileContents FileContent;
-	Array<Token> Tokens;
-	Array<Error> Errors;
-	// Errors are allocated to a dedicated general arena.
 };
 
-CodeBody parse_file( Str path );
+struct Opts_parse
-#endif
+{
+	AllocatorInfo backing_msgs;
+	AllocatorInfo backing_tokens;
+	AllocatorInfo backing_ast;
+};
+
+ParseInfo wip_parse_str( LexedInfo lexed, Opts_parse opts GEN_PARAM_DEFAULT );
 
 GEN_API CodeClass       parse_class        ( Str class_def       );
 GEN_API CodeConstructor parse_constructor  ( Str constructor_def );

base/components/interface.parsing.cpp

@@ -8,6 +8,29 @@
 
 // Publically Exposed Interface
 
+ParseInfo wip_parse_str(LexedInfo lexed, Opts_parse opts)
+{
+	TokArray toks;
+	if (lexed.tokens.Num == 0 && lexed.tokens.Ptr == nullptr) {
+		check_parse_args(lexed.text);
+		toks = lex(lexed.text);
+
+		TokenSlice slice = { toks.Arr, scast(s32, array_num(toks.Arr)) };
+		lexed.tokens = slice;
+	}
+	ParseInfo info = struct_zero(ParseInfo);
+	info.lexed = lexed;
+
+	// TODO(Ed): ParseInfo should be set to the parser context.
+
+	_ctx->parser.Tokens = toks;
+	push_scope();
+	CodeBody result = parse_global_nspace(CT_Global_Body);
+	parser_pop(& _ctx->parser);
+
+	return info;
+}
+
 CodeClass parse_class( Str def )
 {
 	check_parse_args( def );

base/components/parser.cpp

@@ -11,7 +11,7 @@
 constexpr bool lex_dont_skip_formatting = false;
 constexpr bool      lex_skip_formatting = true;
 
-void parser_push( ParseContext* ctx, StackNode* node )
+void parser_push( ParseContext* ctx, ParseStackNode* node )
 {
 	node->Prev = ctx->Scope;
 	ctx->Scope = node;
@@ -60,7 +60,7 @@ StrBuilder parser_to_strbuilder(ParseContext ctx)
 	else
 		strbuilder_append_fmt(& result, "\t(%d, %d)\n", last_valid.Line, last_valid.Column );
 
-	StackNode* curr_scope = ctx.Scope;
+	ParseStackNode* curr_scope = ctx.Scope;
 	s32 level = 0;
 	do
 	{
@@ -181,9 +181,10 @@ bool _check_parse_args( Str def, char const* func_name )
 #	define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ )
 #	define check( Type_ )        ( left && currtok.Type        == Type_ )
 
-#	define push_scope()                                                                                                         \
+// TODO(Ed): Don't do this anymore, we need a better initializer.
-	Str null_name = {};                                                                                                         \
+#	define push_scope()                                                                                                                                        \
-	StackNode scope = { nullptr, lex_current( &  _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ) }; \
+	Str null_name = {};                                                                                                                                        \
+	ParseStackNode scope = { nullptr, {nullptr, 0}, lex_current( &  _ctx->parser.Tokens, lex_dont_skip_formatting ), null_name, txt( __func__ ), { nullptr} }; \
 	parser_push( & _ctx->parser, & scope )
 
 #pragma endregion Helper Macros

base/components/parser_types.hpp

@@ -97,6 +97,12 @@ struct TokArray
 	s32          Idx;
 };
 
+struct TokenSlice
+{
+	Token* Ptr;
+	s32    Num;
+};
+
 struct LexContext
 {
 	Str             content;
@@ -108,19 +114,18 @@ struct LexContext
 	Token           token;
 };
 
-struct StackNode
+struct LexedInfo
 {
-	StackNode* Prev;
+	Str        text;
-
+	TokenSlice tokens;
-	Token* Start;
-	Str    Name;          // The name of the AST node (if parsed)
-	Str    ProcName;    // The name of the procedure
 };
 
+typedef struct ParseStackNode ParseStackNode;
+
 struct ParseContext
 {
-	TokArray   Tokens;
+	TokArray        Tokens;
-	StackNode* Scope;
+	ParseStackNode* Scope;
 };
 
 enum MacroType : u16

gen_c_library/c_library.refactor

@@ -535,7 +535,7 @@ namespace Lexer_, gen_Lexer_
 word LexContext, gen_LexContext
 word lex,        gen_lex
 
-word StackNode,    gen_StackNode
+word ParseStackNode,    gen_ParseStackNode
 word ParseContext, gen_ParseContext
 
 // namespace parse_, gen_parse_