Added space stripping during for content of various ASTs

* Typedef/Typename
* Function Names
* Pragmas
* Attributes
This commit is contained in:
Edward R. Gonzalez 2023-08-26 11:55:05 -04:00
parent abf51e4aa9
commit 7249a7317d
6 changed files with 126 additions and 17 deletions

View File

@ -11,9 +11,9 @@ char const* AST::debug_str()
String
result = String::make_reserve( GlobalAllocator, kilobytes(1) );
result.append_fmt(
"\nType : %s"
"\nParent : %s %s"
"\nName : %s"
"\n\tType : %s"
"\n\tParent : %s %s"
"\n\tName : %s"
, type_str()
, Parent->type_str()
, Parent->Name, Name ? Name : ""
@ -25,8 +25,8 @@ char const* AST::debug_str()
String
result = String::make_reserve( GlobalAllocator, kilobytes(1) );
result.append_fmt(
"\nType : %s"
"\nName : %s"
"\n\tType : %s"
"\n\tName : %s"
, type_str()
, Name ? Name : ""
);
@ -1066,7 +1066,11 @@ bool AST::is_equal( AST* other )
case Preprocess_EndIf:
return true;
// Comments are not validated.
case Comment:
// return true;
case Execution:
case PlatformAttributes:
case Untyped:

View File

@ -4,6 +4,11 @@
#include "gen/eoperator.hpp"
#include "gen/especifier.hpp"
namespace Parser
{
struct Token;
}
struct AST;
struct AST_Body;
struct AST_Attributes;
@ -220,6 +225,7 @@ struct AST
- sizeof(CodeT)
- sizeof(ModuleFlag)
- sizeof(u32)
- sizeof(s32)
)
/ sizeof(SpecifierT) - 1; // -1 for 4 extra bytes
@ -270,6 +276,7 @@ struct AST
AccessSpec ParentAccess;
s32 NumEntries;
};
s32 Token; // Handle to the token, stored in the CodeFile (Otherwise unretrivable)
};
struct AST_POD
@ -321,6 +328,7 @@ struct AST_POD
AccessSpec ParentAccess;
s32 NumEntries;
};
s32 Token; // Handle to the token, stored in the CodeFile (Otherwise unretrivable)
};
// Its intended for the AST to have equivalent size to its POD.

View File

@ -18,6 +18,7 @@ struct AST_Body
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) ];
s32 NumEntries;
s32 Token;
};
static_assert( sizeof(AST_Body) == sizeof(AST), "ERROR: AST_Filtered is not the same size as AST");
@ -33,6 +34,7 @@ struct AST_Attributes
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Attributes) == sizeof(AST), "ERROR: AST_Attributes is not the same size as AST");
@ -48,6 +50,7 @@ struct AST_Comment
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Comment) == sizeof(AST), "ERROR: AST_Comment is not the same size as AST");
@ -72,6 +75,7 @@ struct AST_Class
CodeT Type;
ModuleFlag ModuleFlags;
AccessSpec ParentAccess;
s32 Token;
};
static_assert( sizeof(AST_Class) == sizeof(AST), "ERROR: AST_Class is not the same size as AST");
@ -95,6 +99,7 @@ struct AST_Constructor
char _PAD_NAME_[ sizeof(StringCached) ];
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Constructor) == sizeof(AST), "ERROR: AST_Constructor is not the same size as AST");
@ -110,6 +115,7 @@ struct AST_Define
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Define) == sizeof(AST), "ERROR: AST_Define is not the same size as AST");
@ -132,6 +138,7 @@ struct AST_Destructor
char _PAD_NAME_[ sizeof(StringCached) ];
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Destructor) == sizeof(AST), "ERROR: AST_Destructor is not the same size as AST");
@ -156,6 +163,7 @@ struct AST_Enum
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Enum) == sizeof(AST), "ERROR: AST_Enum is not the same size as AST");
@ -171,6 +179,7 @@ struct AST_Exec
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Exec) == sizeof(AST), "ERROR: AST_Exec is not the same size as AST");
@ -190,6 +199,7 @@ struct AST_Extern
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Extern) == sizeof(AST), "ERROR: AST_Extern is not the same size as AST");
@ -205,6 +215,7 @@ struct AST_Include
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Include) == sizeof(AST), "ERROR: AST_Include is not the same size as AST");
@ -225,6 +236,7 @@ struct AST_Friend
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Friend) == sizeof(AST), "ERROR: AST_Friend is not the same size as AST");
@ -249,6 +261,7 @@ struct AST_Fn
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Fn) == sizeof(AST), "ERROR: AST_Fn is not the same size as AST");
@ -262,6 +275,7 @@ struct AST_Module
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Module) == sizeof(AST), "ERROR: AST_Module is not the same size as AST");
@ -281,6 +295,7 @@ struct AST_NS
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_NS) == sizeof(AST), "ERROR: AST_NS is not the same size as AST");
@ -305,6 +320,7 @@ struct AST_Operator
CodeT Type;
ModuleFlag ModuleFlags;
OperatorT Op;
s32 Token;
};
static_assert( sizeof(AST_Operator) == sizeof(AST), "ERROR: AST_Operator is not the same size as AST");
@ -328,6 +344,7 @@ struct AST_OpCast
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_OpCast) == sizeof(AST), "ERROR: AST_OpCast is not the same size as AST");
@ -350,6 +367,7 @@ struct AST_Param
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) ];
s32 NumEntries;
s32 Token;
};
static_assert( sizeof(AST_Param) == sizeof(AST), "ERROR: AST_Param is not the same size as AST");
@ -365,6 +383,7 @@ struct AST_Pragma
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Pragma) == sizeof(AST), "ERROR: AST_Pragma is not the same size as AST");
@ -380,6 +399,7 @@ struct AST_PreprocessCond
StringCached Name;
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) + sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_PreprocessCond) == sizeof(AST), "ERROR: AST_PreprocessCond is not the same size as AST");
@ -393,6 +413,7 @@ struct AST_Specifiers
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) ];
s32 NumEntries;
s32 Token;
};
static_assert( sizeof(AST_Specifiers) == sizeof(AST), "ERROR: AST_Specifier is not the same size as AST");
@ -417,6 +438,7 @@ struct AST_Struct
CodeT Type;
ModuleFlag ModuleFlags;
AccessSpec ParentAccess;
s32 Token;
};
static_assert( sizeof(AST_Struct) == sizeof(AST), "ERROR: AST_Struct is not the same size as AST");
@ -438,6 +460,7 @@ struct AST_Template
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Template) == sizeof(AST), "ERROR: AST_Template is not the same size as AST");
@ -461,6 +484,7 @@ struct AST_Type
CodeT Type;
char _PAD_UNUSED_[ sizeof(ModuleFlag) ];
b32 IsParamPack;
s32 Token;
};
static_assert( sizeof(AST_Type) == sizeof(AST), "ERROR: AST_Type is not the same size as AST");
@ -483,6 +507,7 @@ struct AST_Typedef
CodeT Type;
ModuleFlag ModuleFlags;
b32 IsFunction;
s32 Token;
};
static_assert( sizeof(AST_Typedef) == sizeof(AST), "ERROR: AST_Typedef is not the same size as AST");
@ -505,6 +530,7 @@ struct AST_Union
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Union) == sizeof(AST), "ERROR: AST_Union is not the same size as AST");
@ -528,6 +554,7 @@ struct AST_Using
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Using) == sizeof(AST), "ERROR: AST_Using is not the same size as AST");
@ -552,6 +579,7 @@ struct AST_Var
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 Token;
};
static_assert( sizeof(AST_Var) == sizeof(AST), "ERROR: AST_Var is not the same size as AST");

View File

@ -424,6 +424,7 @@ Code make_code()
result->Type = ECode::Invalid;
result->ModuleFlags = ModuleFlag::Invalid;
result->NumEntries = 0;
result->Token = -1;
return result;
}

View File

@ -521,6 +521,8 @@ namespace Parser
s32 within_string = false;
s32 within_char = false;
// SkipWhitespace();
while ( left )
{
if ( current == '"' && ! within_char )
@ -563,13 +565,11 @@ namespace Parser
if ( current == '\r' )
{
move_forward();
// content.Length++;
}
if ( current == '\n' )
{
move_forward();
// content.Length++;
break;
}
@ -1274,6 +1274,18 @@ CodeDefine parse_define()
return CodeInvalid;
}
// s32 left = currtok.Length;
// char const* scanner = currtok.Text;
// while ( left )
// {
// if ( scanner[0] == ' ' )
// {
// scanner++;
// left--;
// continue;
// }
// }
define->Content = get_cached_string( currtok );
eat( TokType::Preprocess_Content );
@ -1359,7 +1371,12 @@ CodePragma parse_pragma()
}
Context.Scope->Name = currtok;
pragma->Content = get_cached_string( currtok );
String
content_stripped = String::make( GlobalAllocator, currtok );
content_stripped.strip_space();
pragma->Content = get_cached_string( content_stripped );
eat( TokType::Preprocess_Content );
Context.pop();
@ -1398,7 +1415,16 @@ Code parse_static_assert()
content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text;
String
content_stripped = String::make( GlobalAllocator, content );
content_stripped.strip_space();
char const* result = str_fmt_buf( "%.*s\n", content.Length, content.Text );
if ( content_stripped )
{
result = str_fmt_buf( "%S\n", content_stripped );
}
assert->Content = get_cached_string( to_str( result ) );
assert->Name = assert->Content;
@ -1538,7 +1564,18 @@ CodeAttributes parse_attributes()
{
StrC attribute_txt = { len, start.Text };
Context.pop();
return def_attributes( attribute_txt );
String
name_stripped = String::make( GlobalAllocator, attribute_txt );
name_stripped.strip_space();
Code
result = make_code();
result->Type = ECode::PlatformAttributes;
result->Name = get_cached_string( name_stripped );
result->Content = result->Name;
return (CodeAttributes) result;
}
Context.pop();
@ -1867,9 +1904,15 @@ CodeFn parse_function_after_name(
using namespace ECode;
String
name_stripped = String::make( GlobalAllocator, name );
name_stripped.strip_space();
CodeFn
result = (CodeFn) make_code();
result->Name = get_cached_string( name );
result->Name = get_cached_string( name_stripped );
result->ModuleFlags = mflags;
if ( body )
@ -3445,7 +3488,7 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers )
return CodeInvalid;
}
}
CodeComment inline_cmt = NoCode;
if ( check( TokType::BraceCurly_Open ) )
@ -3454,7 +3497,7 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers )
{
Token stmt_end = currtok;
eat( TokType::Statement_End );
if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line )
inline_cmt = parse_comment();
}
@ -3471,7 +3514,7 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers )
}
else
result->Type = ECode::Destructor_Fwd;
if ( inline_cmt )
result->InlineCmt = inline_cmt;
@ -4151,7 +4194,7 @@ CodeOpCast parse_operator_cast( CodeSpecifiers specifiers )
{
Token stmt_end = currtok;
eat( TokType::Statement_End );
if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line )
inline_cmt = parse_comment();
}
@ -4530,7 +4573,11 @@ CodeType parse_type( bool* is_function )
}
}
result->Name = get_cached_string( name );
String
name_stripped = String::make( GlobalAllocator, name );
name_stripped.strip_space();
result->Name = get_cached_string( name_stripped );
if ( attributes )
result->Attributes = attributes;
@ -4941,7 +4988,7 @@ CodeUsing parse_using()
Token stmt_end = currtok;
eat( TokType::Statement_End );
CodeComment inline_cmt = NoCode;
if ( currtok_noskip.Type == TokType::Comment && currtok_noskip.Line == stmt_end.Line )
{
@ -4971,7 +5018,7 @@ CodeUsing parse_using()
if ( attributes )
result->Attributes = attributes;
if ( inline_cmt )
result->InlineCmt = inline_cmt;
}

View File

@ -213,6 +213,27 @@ struct String
#undef current
}
void strip_space()
{
char* write_pos = Data;
char* read_pos = Data;
while ( * read_pos)
{
if ( ! char_is_space( *read_pos ))
{
*write_pos = *read_pos;
write_pos++;
}
read_pos++;
}
write_pos[0] = '\0'; // Null-terminate the modified string
// Update the length if needed
get_header().Length = write_pos - Data;
}
void trim( char const* cut_set )
{
sw len = 0;