Added space stripping during for content of various ASTs

* Typedef/Typename
* Function Names
* Pragmas
* Attributes

project/components/ast.cpp

@@ -11,9 +11,9 @@ char const* AST::debug_str()
 		String
 		result = String::make_reserve( GlobalAllocator, kilobytes(1) );
 		result.append_fmt(
-				"\nType    : %s"
-				"\nParent  : %s %s"
-				"\nName    : %s"
+				"\n\tType    : %s"
+				"\n\tParent  : %s %s"
+				"\n\tName    : %s"
 			, type_str()
 			, Parent->type_str()
 			, Parent->Name, Name ? Name : ""
@@ -25,8 +25,8 @@ char const* AST::debug_str()
 	String
 	result = String::make_reserve( GlobalAllocator, kilobytes(1) );
 	result.append_fmt(
-			"\nType    : %s"
-			"\nName    : %s"
+			"\n\tType    : %s"
+			"\n\tName    : %s"
 		, type_str()
 		, Name ? Name : ""
 	);
@@ -1066,7 +1066,11 @@ bool AST::is_equal( AST* other )
 		case Preprocess_EndIf:
 			return true;
 
+
+		// Comments are not validated.
 		case Comment:
+			// return true;
+
 		case Execution:
 		case PlatformAttributes:
 		case Untyped:

project/components/ast.hpp

@@ -4,6 +4,11 @@
 #include "gen/eoperator.hpp"
 #include "gen/especifier.hpp"
 
+namespace Parser
+{
+	struct Token;
+}
+
 struct AST;
 struct AST_Body;
 struct AST_Attributes;
@@ -220,6 +225,7 @@ struct AST
 			- sizeof(CodeT)
 			- sizeof(ModuleFlag)
 			- sizeof(u32)
+			- sizeof(s32)
 	)
 	/ sizeof(SpecifierT) - 1; // -1 for 4 extra bytes
 
@@ -270,6 +276,7 @@ struct AST
 		AccessSpec    ParentAccess;
 		s32           NumEntries;
 	};
+	s32               Token;       // Handle to the token, stored in the CodeFile (Otherwise unretrivable)
 };
 
 struct AST_POD
@@ -321,6 +328,7 @@ struct AST_POD
 		AccessSpec    ParentAccess;
 		s32           NumEntries;
 	};
+	s32               Token;       // Handle to the token, stored in the CodeFile (Otherwise unretrivable)
 };
 
 // Its intended for the AST to have equivalent size to its POD.

project/components/ast_types.hpp

@@ -18,6 +18,7 @@ struct AST_Body
 	CodeT             Type;
 	char              _PAD_UNUSED_[ sizeof(ModuleFlag) ];
 	s32 			  NumEntries;
+	s32               Token;
 };
 static_assert( sizeof(AST_Body) == sizeof(AST), "ERROR: AST_Filtered is not the same size as AST");
 
@@ -33,6 +34,7 @@ struct AST_Attributes
 	StringCached      Name;
 	CodeT             Type;
 	char              _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Attributes) == sizeof(AST), "ERROR: AST_Attributes is not the same size as AST");
 
@@ -48,6 +50,7 @@ struct AST_Comment
 	StringCached      Name;
 	CodeT             Type;
 	char              _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Comment) == sizeof(AST), "ERROR: AST_Comment is not the same size as AST");
 
@@ -72,6 +75,7 @@ struct AST_Class
 	CodeT                   Type;
 	ModuleFlag              ModuleFlags;
 	AccessSpec              ParentAccess;
+	s32                     Token;
 };
 static_assert( sizeof(AST_Class) == sizeof(AST), "ERROR: AST_Class is not the same size as AST");
 
@@ -95,6 +99,7 @@ struct AST_Constructor
 	char              _PAD_NAME_[ sizeof(StringCached) ];
 	CodeT             Type;
 	char              _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Constructor) == sizeof(AST), "ERROR: AST_Constructor is not the same size as AST");
 
@@ -110,6 +115,7 @@ struct AST_Define
 	StringCached      Name;
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Define) == sizeof(AST), "ERROR: AST_Define is not the same size as AST");
 
@@ -132,6 +138,7 @@ struct AST_Destructor
 	char                   _PAD_NAME_[ sizeof(StringCached) ];
 	CodeT                  Type;
 	char                   _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32                    Token;
 };
 static_assert( sizeof(AST_Destructor) == sizeof(AST), "ERROR: AST_Destructor is not the same size as AST");
 
@@ -156,6 +163,7 @@ struct AST_Enum
 	CodeT                  Type;
 	ModuleFlag             ModuleFlags;
 	char 			       _PAD_UNUSED_[ sizeof(u32) ];
+	s32                    Token;
 };
 static_assert( sizeof(AST_Enum) == sizeof(AST), "ERROR: AST_Enum is not the same size as AST");
 
@@ -171,6 +179,7 @@ struct AST_Exec
 	StringCached      Name;
 	CodeT             Type;
 	char              _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Exec) == sizeof(AST), "ERROR: AST_Exec is not the same size as AST");
 
@@ -190,6 +199,7 @@ struct AST_Extern
 	StringCached      Name;
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Extern) == sizeof(AST), "ERROR: AST_Extern is not the same size as AST");
 
@@ -205,6 +215,7 @@ struct AST_Include
 	StringCached      Name;
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Include) == sizeof(AST), "ERROR: AST_Include is not the same size as AST");
 
@@ -225,6 +236,7 @@ struct AST_Friend
 	StringCached      Name;
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Friend) == sizeof(AST), "ERROR: AST_Friend is not the same size as AST");
 
@@ -249,6 +261,7 @@ struct AST_Fn
 	CodeT                   Type;
 	ModuleFlag              ModuleFlags;
 	char 			        _PAD_UNUSED_[ sizeof(u32) ];
+	s32                     Token;
 };
 static_assert( sizeof(AST_Fn) == sizeof(AST), "ERROR: AST_Fn is not the same size as AST");
 
@@ -262,6 +275,7 @@ struct AST_Module
 	CodeT             Type;
 	ModuleFlag        ModuleFlags;
 	char 			  _PAD_UNUSED_[ sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Module) == sizeof(AST), "ERROR: AST_Module is not the same size as AST");
 
@@ -281,6 +295,7 @@ struct AST_NS
 	CodeT             Type;
 	ModuleFlag        ModuleFlags;
 	char 			  _PAD_UNUSED_[ sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_NS) == sizeof(AST), "ERROR: AST_NS is not the same size as AST");
 
@@ -305,6 +320,7 @@ struct AST_Operator
 	CodeT                   Type;
 	ModuleFlag              ModuleFlags;
 	OperatorT               Op;
+	s32                     Token;
 };
 static_assert( sizeof(AST_Operator) == sizeof(AST), "ERROR: AST_Operator is not the same size as AST");
 
@@ -328,6 +344,7 @@ struct AST_OpCast
 	StringCached      Name;
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_OpCast) == sizeof(AST), "ERROR: AST_OpCast is not the same size as AST");
 
@@ -350,6 +367,7 @@ struct AST_Param
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) ];
 	s32               NumEntries;
+	s32               Token;
 };
 static_assert( sizeof(AST_Param) == sizeof(AST), "ERROR: AST_Param is not the same size as AST");
 
@@ -365,6 +383,7 @@ struct AST_Pragma
 	StringCached      Name;
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_Pragma) == sizeof(AST), "ERROR: AST_Pragma is not the same size as AST");
 
@@ -380,6 +399,7 @@ struct AST_PreprocessCond
 	StringCached      Name;
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
+	s32               Token;
 };
 static_assert( sizeof(AST_PreprocessCond) == sizeof(AST), "ERROR: AST_PreprocessCond is not the same size as AST");
 
@@ -393,6 +413,7 @@ struct AST_Specifiers
 	CodeT             Type;
 	char 			  _PAD_UNUSED_[ sizeof(ModuleFlag) ];
 	s32               NumEntries;
+	s32               Token;
 };
 	static_assert( sizeof(AST_Specifiers) == sizeof(AST), "ERROR: AST_Specifier is not the same size as AST");
 
@@ -417,6 +438,7 @@ struct AST_Struct
 	CodeT                  Type;
 	ModuleFlag             ModuleFlags;
 	AccessSpec             ParentAccess;
+	s32                    Token;
 };
 static_assert( sizeof(AST_Struct) == sizeof(AST), "ERROR: AST_Struct is not the same size as AST");
 
@@ -438,6 +460,7 @@ struct AST_Template
 	CodeT                  Type;
 	ModuleFlag             ModuleFlags;
 	char 			       _PAD_UNUSED_[ sizeof(u32) ];
+	s32                    Token;
 };
 static_assert( sizeof(AST_Template) == sizeof(AST), "ERROR: AST_Template is not the same size as AST");
 
@@ -461,6 +484,7 @@ struct AST_Type
 	CodeT                  Type;
 	char 			       _PAD_UNUSED_[ sizeof(ModuleFlag) ];
 	b32                    IsParamPack;
+	s32                    Token;
 };
 static_assert( sizeof(AST_Type) == sizeof(AST), "ERROR: AST_Type is not the same size as AST");
 
@@ -483,6 +507,7 @@ struct AST_Typedef
 	CodeT                  Type;
 	ModuleFlag             ModuleFlags;
 	b32                    IsFunction;
+	s32                    Token;
 };
 static_assert( sizeof(AST_Typedef) == sizeof(AST), "ERROR: AST_Typedef is not the same size as AST");
 
@@ -505,6 +530,7 @@ struct AST_Union
 	CodeT                  Type;
 	ModuleFlag             ModuleFlags;
 	char 			       _PAD_UNUSED_[ sizeof(u32) ];
+	s32                    Token;
 };
 static_assert( sizeof(AST_Union) == sizeof(AST), "ERROR: AST_Union is not the same size as AST");
 
@@ -528,6 +554,7 @@ struct AST_Using
 	CodeT                   Type;
 	ModuleFlag              ModuleFlags;
 	char 			        _PAD_UNUSED_[ sizeof(u32) ];
+	s32                     Token;
 };
 static_assert( sizeof(AST_Using) == sizeof(AST), "ERROR: AST_Using is not the same size as AST");
 
@@ -552,6 +579,7 @@ struct AST_Var
 	CodeT                  Type;
 	ModuleFlag             ModuleFlags;
 	char 			       _PAD_UNUSED_[ sizeof(u32) ];
+	s32                    Token;
 };
 static_assert( sizeof(AST_Var) == sizeof(AST), "ERROR: AST_Var is not the same size as AST");
 

project/components/interface.cpp

@@ -424,6 +424,7 @@ Code make_code()
 	result->Type            = ECode::Invalid;
 	result->ModuleFlags     = ModuleFlag::Invalid;
 	result->NumEntries      = 0;
+	result->Token		    = -1;
 
 	return result;
 }

project/components/interface.parsing.cpp

@@ -521,6 +521,8 @@ namespace Parser
 
 					s32 within_string = false;
 					s32 within_char   = false;
+
+					// SkipWhitespace();
 					while ( left )
 					{
 						if ( current == '"' && ! within_char )
@@ -563,13 +565,11 @@ namespace Parser
 						if ( current == '\r' )
 						{
 							move_forward();
-							// content.Length++;
 						}
 
 						if ( current == '\n' )
 						{
 							move_forward();
-							// content.Length++;
 							break;
 						}
 
@@ -1274,6 +1274,18 @@ CodeDefine parse_define()
 		return CodeInvalid;
 	}
 
+	// s32 left = currtok.Length;
+	// char const* scanner = currtok.Text;
+	// while ( left )
+	// {
+	// 	if ( scanner[0] == ' ' )
+	// 	{
+	// 		scanner++;
+	// 		left--;
+	// 		continue;
+	// 	}
+	// }
+
 	define->Content = get_cached_string( currtok );
 	eat( TokType::Preprocess_Content );
 
@@ -1359,7 +1371,12 @@ CodePragma parse_pragma()
 	}
 
 	Context.Scope->Name = currtok;
-	pragma->Content = get_cached_string( currtok );
+
+	String
+	content_stripped = String::make( GlobalAllocator, currtok );
+	content_stripped.strip_space();
+
+	pragma->Content = get_cached_string( content_stripped );
 	eat( TokType::Preprocess_Content );
 
 	Context.pop();
@@ -1398,7 +1415,16 @@ Code parse_static_assert()
 
 	content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text;
 
+
+	String
+	content_stripped = String::make( GlobalAllocator, content );
+	content_stripped.strip_space();
+
 	char const* result = str_fmt_buf( "%.*s\n", content.Length, content.Text );
+	if ( content_stripped )
+	{
+		result = str_fmt_buf( "%S\n", content_stripped );
+	}
 
 	assert->Content = get_cached_string( to_str( result ) );
 	assert->Name	= assert->Content;
@@ -1538,7 +1564,18 @@ CodeAttributes parse_attributes()
 	{
 		StrC attribute_txt = { len, start.Text };
 		Context.pop();
-		return def_attributes( attribute_txt );
+
+		String
+		name_stripped = String::make( GlobalAllocator, attribute_txt );
+		name_stripped.strip_space();
+
+		Code
+		result          = make_code();
+		result->Type    = ECode::PlatformAttributes;
+		result->Name    = get_cached_string( name_stripped );
+		result->Content = result->Name;
+
+		return (CodeAttributes) result;
 	}
 
 	Context.pop();
@@ -1867,9 +1904,15 @@ CodeFn parse_function_after_name(
 
 	using namespace ECode;
 
+
+
+	String
+	name_stripped = String::make( GlobalAllocator, name );
+	name_stripped.strip_space();
+
 	CodeFn
 	result              = (CodeFn) make_code();
-	result->Name        = get_cached_string( name );
+	result->Name        = get_cached_string( name_stripped );
 	result->ModuleFlags = mflags;
 
 	if ( body )
@@ -4530,7 +4573,11 @@ CodeType parse_type( bool* is_function )
 		}
 	}
 
-	result->Name = get_cached_string( name );
+	String
+	name_stripped = String::make( GlobalAllocator, name );
+	name_stripped.strip_space();
+
+	result->Name = get_cached_string( name_stripped );
 
 	if ( attributes )
 		result->Attributes = attributes;

project/dependencies/strings.hpp

@@ -213,6 +213,27 @@ struct String
 	#undef current
 	}
 
+	void strip_space()
+	{
+		char* write_pos = Data;
+		char* read_pos  = Data;
+
+		while ( * read_pos)
+		{
+			if ( ! char_is_space( *read_pos ))
+			{
+				*write_pos = *read_pos;
+				write_pos++;
+			}
+			read_pos++;
+		}
+
+		write_pos[0] = '\0';  // Null-terminate the modified string
+
+		// Update the length if needed
+		get_header().Length = write_pos - Data;
+	}
+
 	void trim( char const* cut_set )
 	{
 		sw len = 0;