ed/gencpp
1
0
Fork 0
mirror of https://github.com/Ed94/gencpp.git synced 2024-12-22 15:54:45 -08:00
Code Issues Packages Projects Releases Wiki Activity

Began to reduce cpp feature usage in lexer and parser

Browse Source
This commit is contained in:
Edward R. Gonzalez 2024-12-02 22:25:39 -05:00
parent 69a9abcd59
commit 2fe708e4be
6 changed files with 288 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
project/components/ast.hpp
View File

@ -152,10 +152,9 @@ Define_Code(Using);
Define_Code(Var); Define_Code(Var);
#undef Define_Code #undef Define_Code
namespace parser GEN_NS_PARSER_BEGIN
{ struct Token;
struct Token; GEN_NS_PARSER_END
}
#if ! GEN_COMPILER_C #if ! GEN_COMPILER_C
template< class Type> forceinline Type tmpl_cast( Code self ) { return * rcast( Type*, & self ); } template< class Type> forceinline Type tmpl_cast( Code self ) { return * rcast( Type*, & self ); }

4
project/components/interface.parsing.cpp
View File

@ -39,9 +39,9 @@ CodeConstructor parse_constructor( StrC def )
SpecifierT specs_found[ 16 ] { ESpecifier::NumSpecifiers }; SpecifierT specs_found[ 16 ] { ESpecifier::NumSpecifiers };
s32 NumSpecifiers = 0; s32 NumSpecifiers = 0;
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
b32 ignore_spec = false; b32 ignore_spec = false;

248
project/components/lexer.cpp
View File

@ -4,7 +4,7 @@
#include "gen/etoktype.cpp" #include "gen/etoktype.cpp"
#endif #endif
namespace parser { GEN_NS_PARSER_BEGIN
enum TokFlags : u32 enum TokFlags : u32
{ {
@ -31,137 +31,132 @@ struct Token
s32 Line; s32 Line;
s32 Column; s32 Column;
u32 Flags; u32 Flags;
operator bool()
{
return Text && Length && Type != TokType::Invalid;
}
operator StrC()
{
return { Length, Text };
}
bool is_access_operator()
{
return bitfield_is_equal( u32, Flags, TF_AccessOperator );
}
bool is_access_specifier()
{
return bitfield_is_equal( u32, Flags, TF_AccessSpecifier );
}
bool is_attribute()
{
return bitfield_is_equal( u32, Flags, TF_Attribute );
}
bool is_operator()
{
return bitfield_is_equal( u32, Flags, TF_Operator );
}
bool is_preprocessor()
{
return bitfield_is_equal( u32, Flags, TF_Preprocess );
}
bool is_preprocess_cond()
{
return bitfield_is_equal( u32, Flags, TF_Preprocess_Cond );
}
bool is_specifier()
{
return bitfield_is_equal( u32, Flags, TF_Specifier );
}
bool is_end_definition()
{
return bitfield_is_equal( u32, Flags, TF_EndDefinition );
}
AccessSpec to_access_specifier()
{
return scast(AccessSpec, Type);
}
String to_string()
{
String result = string_make_reserve( GlobalAllocator, kilobytes(4) );
StrC type_str = ETokType::to_str( Type );
append_fmt( & result, "Line: %d Column: %d, Type: %.*s Content: %.*s"
, Line, Column
, type_str.Len, type_str.Ptr
, Length, Text
);
return result;
}
}; };
constexpr Token NullToken { nullptr, 0, TokType::Invalid, false, 0, TF_Null }; constexpr Token NullToken { nullptr, 0, TokType::Invalid, false, 0, TF_Null };
AccessSpec to_access_specifier(Token tok)
{
return scast(AccessSpec, tok.Type);
}
StrC to_str(Token tok)
{
return { tok.Length, tok.Text };
}
bool is_valid( Token tok )
{
return tok.Text && tok.Length && tok.Type != TokType::Invalid;
}
bool is_access_operator(Token tok)
{
return bitfield_is_equal( u32, tok.Flags, TF_AccessOperator );
}
bool is_access_specifier(Token tok)
{
return bitfield_is_equal( u32, tok.Flags, TF_AccessSpecifier );
}
bool is_attribute(Token tok)
{
return bitfield_is_equal( u32, tok.Flags, TF_Attribute );
}
bool is_operator(Token tok)
{
return bitfield_is_equal( u32, tok.Flags, TF_Operator );
}
bool is_preprocessor(Token tok)
{
return bitfield_is_equal( u32, tok.Flags, TF_Preprocess );
}
bool is_preprocess_cond(Token tok)
{
return bitfield_is_equal( u32, tok.Flags, TF_Preprocess_Cond );
}
bool is_specifier(Token tok)
{
return bitfield_is_equal( u32, tok.Flags, TF_Specifier );
}
bool is_end_definition(Token tok)
{
return bitfield_is_equal( u32, tok.Flags, TF_EndDefinition );
}
String to_string(Token tok)
{
String result = string_make_reserve( GlobalAllocator, kilobytes(4) );
StrC type_str = ETokType::to_str( tok.Type );
append_fmt( & result, "Line: %d Column: %d, Type: %.*s Content: %.*s"
, tok.Line, tok.Column
, type_str.Len, type_str.Ptr
, tok.Length, tok.Text
);
return result;
}
struct TokArray struct TokArray
{ {
Array<Token> Arr; Array(Token) Arr;
s32 Idx; s32 Idx;
bool __eat( TokType type );
Token& current( bool skip_formatting = true )
{
if ( skip_formatting )
{
while ( Arr[Idx].Type == TokType::NewLine || Arr[Idx].Type == TokType::Comment )
Idx++;
}
return Arr[Idx];
}
Token& previous( bool skip_formatting = false )
{
s32 idx = this->Idx;
if ( skip_formatting )
{
while ( Arr[idx].Type == TokType::NewLine )
idx--;
return Arr[idx];
}
return Arr[idx - 1];
}
Token& next( bool skip_formatting = false )
{
s32 idx = this->Idx;
if ( skip_formatting )
{
while ( Arr[idx].Type == TokType::NewLine )
idx++;
return Arr[idx + 1];
}
return Arr[idx + 1];
}
Token& operator []( s32 idx )
{
return Arr[idx];
}
}; };
bool __eat( TokType type );
Token* current(TokArray* self, bool skip_formatting )
{
if ( skip_formatting )
{
while ( self->Arr[self->Idx].Type == TokType::NewLine || self->Arr[self->Idx].Type == TokType::Comment )
self->Idx++;
}
return & self->Arr[self->Idx];
}
Token* previous(TokArray self, bool skip_formatting)
{
s32 idx = self.Idx;
if ( skip_formatting )
{
while ( self.Arr[idx].Type == TokType::NewLine )
idx --;
return & self.Arr[idx];
}
return & self.Arr[idx - 1];
}
Token* next(TokArray self, bool skip_formatting)
{
s32 idx = self.Idx;
if ( skip_formatting )
{
while ( self.Arr[idx].Type == TokType::NewLine )
idx++;
return & self.Arr[idx + 1];
}
return & self.Arr[idx + 1];
}
global Arena_256KB defines_map_arena; global Arena_256KB defines_map_arena;
global HashTable<StrC> defines; global HashTable(StrC) defines;
global Array<Token> Tokens; global Array(Token) Tokens;
#define current ( * scanner ) #define current ( * scanner )
@ -234,7 +229,7 @@ s32 lex_preprocessor_directive(
token.Length++; token.Length++;
} }
token.Type = ETokType::to_type( token ); token.Type = ETokType::to_type( to_str(token) );
bool is_preprocessor = token.Type >= TokType::Preprocess_Define && token.Type <= TokType::Preprocess_Pragma; bool is_preprocessor = token.Type >= TokType::Preprocess_Define && token.Type <= TokType::Preprocess_Pragma;
if ( ! is_preprocessor ) if ( ! is_preprocessor )
@ -341,7 +336,7 @@ s32 lex_preprocessor_directive(
append( & Tokens, name ); append( & Tokens, name );
u64 key = crc32( name.Text, name.Length ); u64 key = crc32( name.Text, name.Length );
set<StrC>(& defines, key, name ); set(& defines, key, to_str(name) );
} }
Token preprocess_content = { scanner, 0, TokType::Preprocess_Content, line, column, TF_Preprocess }; Token preprocess_content = { scanner, 0, TokType::Preprocess_Content, line, column, TF_Preprocess };
@ -465,7 +460,7 @@ void lex_found_token( StrC& content
return; return;
} }
TokType type = ETokType::to_type( token ); TokType type = ETokType::to_type( to_str(token) );
if (type <= TokType::Access_Public && type >= TokType::Access_Private ) if (type <= TokType::Access_Public && type >= TokType::Access_Private )
{ {
@ -1267,5 +1262,4 @@ TokArray lex( StrC content )
#undef move_forward #undef move_forward
#undef SkipWhitespace #undef SkipWhitespace
// namespace parser GEN_NS_PARSER_END
}

297
project/components/parser.cpp
View File

@ -5,11 +5,12 @@
#include "lexer.cpp" #include "lexer.cpp"
#endif #endif
namespace parser { GEN_NS_PARSER_BEGIN
// TODO(Ed) : Rename ETokType::Capture_Start, ETokType::Capture_End to Open_Parenthesis adn Close_Parenthesis // TODO(Ed) : Rename ETokType::Capture_Start, ETokType::Capture_End to Open_Parenthesis adn Close_Parenthesis
constexpr bool dont_skip_formatting = false; constexpr bool dont_skip_formatting = false;
constexpr bool skip_formatting = true;
struct StackNode struct StackNode
{ {
@ -48,7 +49,7 @@ struct ParseContext
String result = string_make_reserve( GlobalAllocator, kilobytes(4) ); String result = string_make_reserve( GlobalAllocator, kilobytes(4) );
Token scope_start = Scope->Start; Token scope_start = Scope->Start;
Token last_valid = Tokens.Idx >= num(Tokens.Arr) ? Tokens.Arr[num(Tokens.Arr) -1] : Tokens.current(); Token last_valid = Tokens.Idx >= num(Tokens.Arr) ? Tokens.Arr[num(Tokens.Arr) -1] : (* current(& Tokens, true));
sptr length = scope_start.Length; sptr length = scope_start.Length;
char const* current = scope_start.Text + length; char const* current = scope_start.Text + length;
@ -75,7 +76,7 @@ struct ParseContext
s32 level = 0; s32 level = 0;
do do
{ {
if ( curr_scope->Name ) if ( is_valid(curr_scope->Name) )
{ {
append_fmt(& result, "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Length, curr_scope->Name.Text ); append_fmt(& result, "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Length, curr_scope->Name.Text );
} }
@ -94,27 +95,30 @@ struct ParseContext
global ParseContext Context; global ParseContext Context;
bool TokArray::__eat( TokType type ) bool __eat(TokArray* self, TokType type )
{ {
if ( num(Arr) - Idx <= 0 ) if ( num(self->Arr) - self->Idx <= 0 )
{ {
log_failure( "No tokens left.\n%s", Context.to_string() ); log_failure( "No tokens left.\n%s", Context.to_string() );
return false; return false;
} }
if ( ( Arr[ Idx ].Type == TokType::NewLine && type != TokType::NewLine ) Token at_idx = self->Arr[ self->Idx ];
|| ( Arr[ Idx ].Type == TokType::Comment && type != TokType::Comment ) )
if ( ( at_idx.Type == TokType::NewLine && type != TokType::NewLine )
|| ( at_idx.Type == TokType::Comment && type != TokType::Comment ) )
{ {
Idx++; self->Idx ++;
} }
if ( Arr[Idx].Type != type ) if ( at_idx.Type != type )
{ {
Token tok = * current( self, skip_formatting );
log_failure( "Parse Error, TokArray::eat, Expected: ' %s ' not ' %.*s ' (%d, %d)`\n%s" log_failure( "Parse Error, TokArray::eat, Expected: ' %s ' not ' %.*s ' (%d, %d)`\n%s"
, ETokType::to_str(type).Ptr , ETokType::to_str(type).Ptr
, Arr[Idx].Length, Arr[Idx].Text , at_idx.Length, at_idx.Text
, current().Line , tok.Line
, current().Column , tok.Column
, Context.to_string() , Context.to_string()
); );
@ -122,10 +126,10 @@ bool TokArray::__eat( TokType type )
} }
#if 0 && Build_Debug #if 0 && Build_Debug
log_fmt("Ate: %S\n", Arr[Idx].to_string() ); log_fmt("Ate: %S\n", self->Arr[Idx].to_string() );
#endif #endif
Idx++; self->Idx ++;
return true; return true;
} }
@ -162,11 +166,11 @@ if ( def.Ptr == nullptr ) \
return InvalidCode; \ return InvalidCode; \
} }
# define currtok_noskip Context.Tokens.current( dont_skip_formatting ) # define currtok_noskip (* current( & Context.Tokens, dont_skip_formatting ))
# define currtok Context.Tokens.current() # define currtok (* current( & Context.Tokens, skip_formatting ))
# define prevtok Context.Tokens.previous() # define prevtok (* previous( Context.Tokens, dont_skip_formatting))
# define nexttok Context.Tokens.next() # define nexttok (* next( Context.Tokens, skip_formatting ))
# define eat( Type_ ) Context.Tokens.__eat( Type_ ) # define eat( Type_ ) __eat( & Context.Tokens, Type_ )
# define left ( num(Context.Tokens.Arr) - Context.Tokens.Idx ) # define left ( num(Context.Tokens.Arr) - Context.Tokens.Idx )
#ifdef check #ifdef check
@ -488,7 +492,7 @@ Code parse_array_decl()
if ( check( TokType::Operator ) && currtok.Text[0] == '[' && currtok.Text[1] == ']' ) if ( check( TokType::Operator ) && currtok.Text[0] == '[' && currtok.Text[1] == ']' )
{ {
Code array_expr = untyped_str( currtok ); Code array_expr = untyped_str( to_str(currtok) );
eat( TokType::Operator ); eat( TokType::Operator );
// [] // []
@ -524,7 +528,7 @@ Code parse_array_decl()
untyped_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)untyped_tok.Text; untyped_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)untyped_tok.Text;
Code array_expr = untyped_str( untyped_tok ); Code array_expr = untyped_str( to_str(untyped_tok) );
// [ <Content> // [ <Content>
if ( left == 0 ) if ( left == 0 )
@ -571,7 +575,7 @@ CodeAttributes parse_attributes()
// There can be more than one attribute. If there is flatten them to a single string. // There can be more than one attribute. If there is flatten them to a single string.
// TODO(Ed): Support keeping an linked list of attributes similar to parameters // TODO(Ed): Support keeping an linked list of attributes similar to parameters
while ( left && currtok.is_attribute() ) while ( left && is_attribute(currtok) )
{ {
if ( check( TokType::Attribute_Open ) ) if ( check( TokType::Attribute_Open ) )
{ {
@ -625,7 +629,7 @@ CodeAttributes parse_attributes()
len = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )start.Text; len = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )start.Text;
} }
else if ( currtok.is_attribute() ) else if ( is_attribute(currtok) )
{ {
eat( currtok.Type ); eat( currtok.Type );
// <Attribute> // <Attribute>
@ -724,15 +728,15 @@ Code parse_class_struct( TokType which, bool inplace_def = false )
eat( TokType::Assign_Classifer ); eat( TokType::Assign_Classifer );
// <ModuleFlags> <class/struct> <Attributes> <Name> : // <ModuleFlags> <class/struct> <Attributes> <Name> :
if ( currtok.is_access_specifier() ) if ( is_access_specifier(currtok) )
{ {
access = currtok.to_access_specifier(); access = to_access_specifier(currtok);
// <ModuleFlags> <class/struct> <Attributes> <Name> : <Access Specifier> // <ModuleFlags> <class/struct> <Attributes> <Name> : <Access Specifier>
eat( currtok.Type ); eat( currtok.Type );
} }
Token parent_tok = parse_identifier(); Token parent_tok = parse_identifier();
parent = def_type( parent_tok ); parent = def_type( to_str(parent_tok) );
// <ModuleFlags> <class/struct> <Attributes> <Name> : <Access Specifier> <Parent/Interface Name> // <ModuleFlags> <class/struct> <Attributes> <Name> : <Access Specifier> <Parent/Interface Name>
while ( check(TokType::Comma) ) while ( check(TokType::Comma) )
@ -740,13 +744,13 @@ Code parse_class_struct( TokType which, bool inplace_def = false )
eat( TokType::Comma ); eat( TokType::Comma );
// <ModuleFlags> <class/struct> <Attributes> <Name> : <Access Specifier> <Name>, // <ModuleFlags> <class/struct> <Attributes> <Name> : <Access Specifier> <Name>,
if ( currtok.is_access_specifier() ) if ( is_access_specifier(currtok) )
{ {
eat(currtok.Type); eat(currtok.Type);
} }
Token interface_tok = parse_identifier(); Token interface_tok = parse_identifier();
append( & interfaces, def_type( interface_tok ) ); append( & interfaces, def_type( to_str(interface_tok) ) );
// <ModuleFlags> <class/struct> <Attributes> <Name> : <Access Specifier> <Name>, ... // <ModuleFlags> <class/struct> <Attributes> <Name> : <Access Specifier> <Name>, ...
} }
} }
@ -770,10 +774,10 @@ Code parse_class_struct( TokType which, bool inplace_def = false )
} }
if ( which == TokType::Decl_Class ) if ( which == TokType::Decl_Class )
result = def_class( name, body, parent, access, attributes, mflags ); result = def_class( to_str(name), body, parent, access, attributes, mflags );
else else
result = def_struct( name, body, (CodeType)parent, access, attributes, mflags ); result = def_struct( to_str(name), body, (CodeType)parent, access, attributes, mflags );
if ( inline_cmt ) if ( inline_cmt )
result->InlineCmt = inline_cmt; result->InlineCmt = inline_cmt;
@ -818,7 +822,7 @@ CodeBody parse_class_struct_body( TokType which, Token name )
case TokType::Statement_End: case TokType::Statement_End:
{ {
// TODO(Ed): Convert this to a general warning procedure // TODO(Ed): Convert this to a general warning procedure
log_fmt("Dangling end statement found %S\n", currtok_noskip.to_string()); log_fmt("Dangling end statement found %S\n", to_string(currtok_noskip));
eat( TokType::Statement_End ); eat( TokType::Statement_End );
continue; continue;
} }
@ -984,9 +988,9 @@ CodeBody parse_class_struct_body( TokType which, Token name )
SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; SpecifierT specs_found[16] { ESpecifier::NumSpecifiers };
s32 NumSpecifiers = 0; s32 NumSpecifiers = 0;
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
b32 ignore_spec = false; b32 ignore_spec = false;
@ -1033,7 +1037,7 @@ CodeBody parse_class_struct_body( TokType which, Token name )
} }
// <Attributes> <Specifiers> // <Attributes> <Specifiers>
if ( currtok.is_attribute() ) if ( is_attribute(currtok) )
{ {
// Unfortuantely Unreal has code where there is attirbutes before specifiers // Unfortuantely Unreal has code where there is attirbutes before specifiers
CodeAttributes more_attributes = parse_attributes(); CodeAttributes more_attributes = parse_attributes();
@ -1103,7 +1107,7 @@ CodeBody parse_class_struct_body( TokType which, Token name )
eat( currtok.Type ); eat( currtok.Type );
} }
member = untyped_str( untyped_tok ); member = untyped_str( to_str(untyped_tok) );
// Something unknown // Something unknown
break; break;
} }
@ -1133,7 +1137,7 @@ CodeComment parse_comment()
CodeComment CodeComment
result = (CodeComment) make_code(); result = (CodeComment) make_code();
result->Type = ECode::Comment; result->Type = ECode::Comment;
result->Content = get_cached_string( currtok_noskip ); result->Content = get_cached_string( to_str(currtok_noskip) );
result->Name = result->Content; result->Name = result->Content;
// result->Token = currtok_noskip; // result->Token = currtok_noskip;
eat( TokType::Comment ); eat( TokType::Comment );
@ -1155,13 +1159,13 @@ Code parse_complicated_definition( TokType which )
s32 level = 0; s32 level = 0;
for ( ; idx < num(tokens.Arr); idx++ ) for ( ; idx < num(tokens.Arr); idx++ )
{ {
if ( tokens[ idx ].Type == TokType::BraceCurly_Open ) if ( tokens.Arr[ idx ].Type == TokType::BraceCurly_Open )
level++; level++;
if ( tokens[ idx ].Type == TokType::BraceCurly_Close ) if ( tokens.Arr[ idx ].Type == TokType::BraceCurly_Close )
level--; level--;
if ( level == 0 && tokens[ idx ].Type == TokType::Statement_End ) if ( level == 0 && tokens.Arr[ idx ].Type == TokType::Statement_End )
break; break;
} }
@ -1174,23 +1178,23 @@ Code parse_complicated_definition( TokType which )
return result; return result;
} }
Token tok = tokens[ idx - 1 ]; Token tok = tokens.Arr[ idx - 1 ];
if ( tok.is_specifier() && is_trailing( ESpecifier::to_type(tok)) ) if ( is_specifier(tok) && is_trailing( ESpecifier::to_type( to_str(tok))) )
{ {
// <which> <type_identifier>(...) <specifier> ...; // <which> <type_identifier>(...) <specifier> ...;
s32 spec_idx = idx - 1; s32 spec_idx = idx - 1;
Token spec = tokens[spec_idx]; Token spec = tokens.Arr[spec_idx];
while ( spec.is_specifier() && is_trailing( ESpecifier::to_type(spec)) ) while ( is_specifier(spec) && is_trailing( ESpecifier::to_type( to_str(spec))) )
{ {
-- spec_idx; -- spec_idx;
spec = tokens[spec_idx]; spec = tokens.Arr[spec_idx];
} }
if ( tokens[spec_idx].Type == TokType::Capture_End ) if ( tokens.Arr[spec_idx].Type == TokType::Capture_End )
{ {
// Forward declaration with trailing specifiers for a procedure // Forward declaration with trailing specifiers for a procedure
tok = tokens[spec_idx]; tok = tokens.Arr[spec_idx];
Code result = parse_operator_function_or_variable( false, { nullptr }, { nullptr } ); Code result = parse_operator_function_or_variable( false, { nullptr }, { nullptr } );
// <Attributes> <Specifiers> <ReturnType/ValueType> <operator <Op>, or Name> ... // <Attributes> <Specifiers> <ReturnType/ValueType> <operator <Op>, or Name> ...
@ -1204,7 +1208,7 @@ Code parse_complicated_definition( TokType which )
} }
if ( tok.Type == TokType::Identifier ) if ( tok.Type == TokType::Identifier )
{ {
tok = tokens[ idx - 2 ]; tok = tokens.Arr[ idx - 2 ];
bool is_indirection = tok.Type == TokType::Ampersand || tok.Type == TokType::Star; bool is_indirection = tok.Type == TokType::Ampersand || tok.Type == TokType::Star;
bool ok_to_parse = false; bool ok_to_parse = false;
@ -1215,15 +1219,15 @@ Code parse_complicated_definition( TokType which )
ok_to_parse = true; ok_to_parse = true;
is_inplace = true; is_inplace = true;
} }
else if ( tok.Type == TokType::Identifier && tokens[ idx - 3 ].Type == which ) else if ( tok.Type == TokType::Identifier && tokens.Arr[ idx - 3 ].Type == which )
{ {
// Its a variable with type ID using <which> namespace. // Its a variable with type ID using <which> namespace.
// <which> <type_identifier> <identifier>; // <which> <type_identifier> <identifier>;
ok_to_parse = true; ok_to_parse = true;
} }
else if ( tok.Type == TokType::Assign_Classifer else if ( tok.Type == TokType::Assign_Classifer
&& ( ( tokens[idx - 5].Type == which && tokens[idx - 4].Type == TokType::Decl_Class ) && ( ( tokens.Arr[idx - 5].Type == which && tokens.Arr[idx - 4].Type == TokType::Decl_Class )
|| ( tokens[idx - 4].Type == which)) || ( tokens.Arr[idx - 4].Type == which))
) )
{ {
// Its a forward declaration of an enum // Its a forward declaration of an enum
@ -1255,11 +1259,11 @@ Code parse_complicated_definition( TokType which )
} }
else if ( tok.Type >= TokType::Type_Unsigned && tok.Type <= TokType::Type_MS_W64 ) else if ( tok.Type >= TokType::Type_Unsigned && tok.Type <= TokType::Type_MS_W64 )
{ {
tok = tokens[ idx - 2 ]; tok = tokens.Arr[ idx - 2 ];
if ( tok.Type != TokType::Assign_Classifer if ( tok.Type != TokType::Assign_Classifer
|| ( ( tokens[idx - 5].Type != which && tokens[idx - 4].Type != TokType::Decl_Class ) || ( ( tokens.Arr[idx - 5].Type != which && tokens.Arr[idx - 4].Type != TokType::Decl_Class )
&& ( tokens[idx - 4].Type != which)) && ( tokens.Arr[idx - 4].Type != which))
) )
{ {
log_failure( "Unsupported or bad member definition after %s declaration\n%s", to_str(which), Context.to_string() ); log_failure( "Unsupported or bad member definition after %s declaration\n%s", to_str(which), Context.to_string() );
@ -1317,7 +1321,7 @@ CodeDefine parse_define()
} }
Context.Scope->Name = currtok; Context.Scope->Name = currtok;
define->Name = get_cached_string( currtok ); define->Name = get_cached_string( to_str(currtok) );
eat( TokType::Identifier ); eat( TokType::Identifier );
// #define <Name> // #define <Name>
@ -1330,7 +1334,7 @@ CodeDefine parse_define()
if ( currtok.Length == 0 ) if ( currtok.Length == 0 )
{ {
define->Content = get_cached_string( currtok ); define->Content = get_cached_string( to_str(currtok) );
eat( TokType::Preprocess_Content ); eat( TokType::Preprocess_Content );
// #define <Name> <Content> // #define <Name> <Content>
@ -1338,7 +1342,7 @@ CodeDefine parse_define()
return define; return define;
} }
define->Content = get_cached_string( strip_formatting( currtok, strip_formatting_dont_preserve_newlines ) ); define->Content = get_cached_string( strip_formatting( to_str(currtok), strip_formatting_dont_preserve_newlines ) );
eat( TokType::Preprocess_Content ); eat( TokType::Preprocess_Content );
// #define <Name> <Content> // #define <Name> <Content>
@ -1379,7 +1383,7 @@ Code parse_assignment_expression()
} }
expr_tok.Length = ( ( sptr )currtok.Text + currtok.Length ) - ( sptr )expr_tok.Text - 1; expr_tok.Length = ( ( sptr )currtok.Text + currtok.Length ) - ( sptr )expr_tok.Text - 1;
expr = untyped_str( expr_tok ); expr = untyped_str( to_str(expr_tok) );
// = <Expression> // = <Expression>
return expr; return expr;
} }
@ -1431,16 +1435,16 @@ CodeFn parse_function_after_name(
// <Attributes> <Specifiers> <ReturnType> <Name> ( <Parameters> ) // <Attributes> <Specifiers> <ReturnType> <Name> ( <Parameters> )
// TODO(Ed), Review old comment : These have to be kept separate from the return type's specifiers. // TODO(Ed), Review old comment : These have to be kept separate from the return type's specifiers.
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
if ( specifiers.ast == nullptr ) if ( specifiers.ast == nullptr )
{ {
specifiers = def_specifier( ESpecifier::to_type(currtok) ); specifiers = def_specifier( ESpecifier::to_type( to_str(currtok)) );
eat( currtok.Type ); eat( currtok.Type );
continue; continue;
} }
append(specifiers, ESpecifier::to_type(currtok) ); append(specifiers, ESpecifier::to_type( to_str(currtok)) );
eat( currtok.Type ); eat( currtok.Type );
} }
// <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers> // <Attributes> <Specifiers> <ReturnType> <Name> ( <Paraemters> ) <Specifiers>
@ -1485,7 +1489,7 @@ CodeFn parse_function_after_name(
using namespace ECode; using namespace ECode;
String String
name_stripped = string_make( GlobalAllocator, name ); name_stripped = string_make( GlobalAllocator, to_str(name) );
strip_space(name_stripped); strip_space(name_stripped);
CodeFn CodeFn
@ -1562,7 +1566,7 @@ Code parse_function_body()
eat( currtok_noskip.Type ); eat( currtok_noskip.Type );
} }
Token previous = prevtok; Token past = prevtok;
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
@ -1613,7 +1617,7 @@ CodeBody parse_global_nspace( CodeT which )
case TokType::Statement_End: case TokType::Statement_End:
{ {
// TODO(Ed): Convert this to a general warning procedure // TODO(Ed): Convert this to a general warning procedure
log_fmt("Dangling end statement found %S\n", currtok_noskip.to_string()); log_fmt("Dangling end statement found %S\n", to_string(currtok_noskip));
eat( TokType::Statement_End ); eat( TokType::Statement_End );
continue; continue;
} }
@ -1764,9 +1768,9 @@ CodeBody parse_global_nspace( CodeT which )
SpecifierT specs_found[16] { ESpecifier::NumSpecifiers }; SpecifierT specs_found[16] { ESpecifier::NumSpecifiers };
s32 NumSpecifiers = 0; s32 NumSpecifiers = 0;
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
bool ignore_spec = false; bool ignore_spec = false;
@ -1840,12 +1844,12 @@ CodeBody parse_global_nspace( CodeT which )
for ( ; idx < num(Context.Tokens.Arr); idx++ ) for ( ; idx < num(Context.Tokens.Arr); idx++ )
{ {
Token tok = Context.Tokens[ idx ]; Token tok = Context.Tokens.Arr[ idx ];
if ( tok.Type == TokType::Identifier ) if ( tok.Type == TokType::Identifier )
{ {
idx++; idx++;
tok = Context.Tokens[ idx ]; tok = Context.Tokens.Arr[ idx ];
if ( tok.Type == TokType::Access_StaticSymbol ) if ( tok.Type == TokType::Access_StaticSymbol )
continue; continue;
@ -1909,15 +1913,15 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers )
TokArray tokens = Context.Tokens; TokArray tokens = Context.Tokens;
s32 idx = tokens.Idx; s32 idx = tokens.Idx;
Token nav = tokens[ idx ]; Token nav = tokens.Arr[ idx ];
for ( ; idx < num(tokens.Arr); idx++, nav = tokens[ idx ] ) for ( ; idx < num(tokens.Arr); idx++, nav = tokens.Arr[ idx ] )
{ {
if ( nav.Text[0] == '<' ) if ( nav.Text[0] == '<' )
{ {
// Skip templated expressions as they mey have expressions with the () operators // Skip templated expressions as they mey have expressions with the () operators
s32 capture_level = 0; s32 capture_level = 0;
s32 template_level = 0; s32 template_level = 0;
for ( ; idx < num(tokens.Arr); idx++, nav = tokens[idx] ) for ( ; idx < num(tokens.Arr); idx++, nav = tokens.Arr[idx] )
{ {
if (nav.Text[ 0 ] == '<') if (nav.Text[ 0 ] == '<')
++ template_level; ++ template_level;
@ -1945,7 +1949,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers )
} }
-- idx; -- idx;
Token tok_right = tokens[idx]; Token tok_right = tokens.Arr[idx];
Token tok_left = NullToken; Token tok_left = NullToken;
if (tok_right.Type != TokType::Identifier) if (tok_right.Type != TokType::Identifier)
@ -1955,7 +1959,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers )
} }
-- idx; -- idx;
tok_left = tokens[idx]; tok_left = tokens.Arr[idx];
// <Attributes> <Specifiers> ... <Identifier> // <Attributes> <Specifiers> ... <Identifier>
bool possible_destructor = false; bool possible_destructor = false;
@ -1963,14 +1967,14 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers )
{ {
possible_destructor = true; possible_destructor = true;
-- idx; -- idx;
tok_left = tokens[idx]; tok_left = tokens.Arr[idx];
} }
if ( tok_left.Type != TokType::Access_StaticSymbol ) if ( tok_left.Type != TokType::Access_StaticSymbol )
return result; return result;
-- idx; -- idx;
tok_left = tokens[idx]; tok_left = tokens.Arr[idx];
// <Attributes> <Specifiers> ... :: <Identifier> // <Attributes> <Specifiers> ... :: <Identifier>
// We search toward the left until we find the next valid identifier // We search toward the left until we find the next valid identifier
@ -1996,7 +2000,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers )
break; break;
-- idx; -- idx;
tok_left = tokens[idx]; tok_left = tokens.Arr[idx];
} }
bool is_same = str_compare( tok_right.Text, tok_left.Text, tok_right.Length ) == 0; bool is_same = str_compare( tok_right.Text, tok_left.Text, tok_right.Length ) == 0;
@ -2112,7 +2116,7 @@ CodeInclude parse_include()
} }
Context.Scope->Name = currtok; Context.Scope->Name = currtok;
include->Content = get_cached_string( currtok ); include->Content = get_cached_string( to_str(currtok) );
eat( TokType::String ); eat( TokType::String );
// #include <Path> or "Path" // #include <Path> or "Path"
@ -2364,10 +2368,10 @@ CodeOperator parse_operator_after_ret_type(
s32 idx = Context.Tokens.Idx + 1; s32 idx = Context.Tokens.Idx + 1;
{ {
while ( Context.Tokens[ idx ].Type == TokType::NewLine ) while ( Context.Tokens.Arr[ idx ].Type == TokType::NewLine )
idx++; idx++;
} }
Token next = Context.Tokens[idx]; Token next = Context.Tokens.Arr[idx];
if ( currtok.Type == TokType::Operator && str_compare(currtok.Text, "[]", 2) == 0) if ( currtok.Type == TokType::Operator && str_compare(currtok.Text, "[]", 2) == 0)
{ {
eat(ETokType::Operator); eat(ETokType::Operator);
@ -2388,10 +2392,10 @@ CodeOperator parse_operator_after_ret_type(
s32 idx = Context.Tokens.Idx + 1; s32 idx = Context.Tokens.Idx + 1;
{ {
while ( Context.Tokens[ idx ].Type == TokType::NewLine ) while ( Context.Tokens.Arr[ idx ].Type == TokType::NewLine )
idx++; idx++;
} }
Token next = Context.Tokens[idx]; Token next = Context.Tokens.Arr[idx];
if ( currtok.Type == TokType::Operator && str_compare(currtok.Text, "[]", 2) == 0) if ( currtok.Type == TokType::Operator && str_compare(currtok.Text, "[]", 2) == 0)
{ {
eat(ETokType::Operator); eat(ETokType::Operator);
@ -2434,16 +2438,16 @@ CodeOperator parse_operator_after_ret_type(
if ( params.ast == nullptr && op == EOperator::Multiply ) if ( params.ast == nullptr && op == EOperator::Multiply )
op = MemberOfPointer; op = MemberOfPointer;
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
if ( specifiers.ast == nullptr ) if ( specifiers.ast == nullptr )
{ {
specifiers = def_specifier( ESpecifier::to_type(currtok) ); specifiers = def_specifier( ESpecifier::to_type( to_str(currtok)) );
eat( currtok.Type ); eat( currtok.Type );
continue; continue;
} }
append(specifiers, ESpecifier::to_type(currtok) ); append(specifiers, ESpecifier::to_type( to_str(currtok)) );
eat( currtok.Type ); eat( currtok.Type );
} }
// <ExportFlag> <Attributes> <Specifiers> <ReturnType> <Qualifier::...> operator <Op> ( <Parameters> ) <Specifiers> // <ExportFlag> <Attributes> <Specifiers> <ReturnType> <Qualifier::...> operator <Op> ( <Parameters> ) <Specifiers>
@ -2473,7 +2477,7 @@ CodeOperator parse_operator_after_ret_type(
} }
// OpValidateResult check_result = operator__validate( op, params, ret_type, specifiers ); // OpValidateResult check_result = operator__validate( op, params, ret_type, specifiers );
CodeOperator result = def_operator( op, nspace, params, ret_type, body, specifiers, attributes, mflags ); CodeOperator result = def_operator( op, to_str(nspace), params, ret_type, body, specifiers, attributes, mflags );
if ( inline_cmt ) if ( inline_cmt )
result->InlineCmt = inline_cmt; result->InlineCmt = inline_cmt;
@ -2514,12 +2518,12 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes
for ( ; idx < num(Context.Tokens.Arr); idx++ ) for ( ; idx < num(Context.Tokens.Arr); idx++ )
{ {
Token tok = Context.Tokens[ idx ]; Token tok = Context.Tokens.Arr[ idx ];
if ( tok.Type == TokType::Identifier ) if ( tok.Type == TokType::Identifier )
{ {
idx++; idx++;
tok = Context.Tokens[ idx ]; tok = Context.Tokens.Arr[ idx ];
if ( tok.Type == TokType::Access_StaticSymbol ) if ( tok.Type == TokType::Access_StaticSymbol )
continue; continue;
@ -2566,7 +2570,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes
} }
// Dealing with a variable // Dealing with a variable
result = parse_variable_after_name( ModuleFlag_None, attributes, specifiers, type, name ); result = parse_variable_after_name( ModuleFlag_None, attributes, specifiers, type, to_str(name) );
// <Attributes> <Specifiers> <ValueType> <Name> ... // <Attributes> <Specifiers> <ValueType> <Name> ...
} }
} }
@ -2595,7 +2599,7 @@ CodePragma parse_pragma()
Context.Scope->Name = currtok; Context.Scope->Name = currtok;
pragma->Content = get_cached_string( currtok ); pragma->Content = get_cached_string( to_str(currtok) );
eat( TokType::Preprocess_Content ); eat( TokType::Preprocess_Content );
// #pragma <Content> // #pragma <Content>
@ -2729,7 +2733,7 @@ CodeParam parse_params( bool use_template_capture )
eat( currtok.Type ); eat( currtok.Type );
} }
value = untyped_str( strip_formatting( value_tok, strip_formatting_dont_preserve_newlines ) ); value = untyped_str( strip_formatting( to_str(value_tok), strip_formatting_dont_preserve_newlines ) );
// ( <Macro> <ValueType> <Name> = <Expression> // ( <Macro> <ValueType> <Name> = <Expression>
} }
} }
@ -2740,7 +2744,7 @@ CodeParam parse_params( bool use_template_capture )
result->Macro = macro; result->Macro = macro;
if ( name.Length > 0 ) if ( name.Length > 0 )
result->Name = get_cached_string( name ); result->Name = get_cached_string( to_str(name) );
result->ValueType = type; result->ValueType = type;
@ -2844,7 +2848,7 @@ CodeParam parse_params( bool use_template_capture )
eat( currtok.Type ); eat( currtok.Type );
} }
value = untyped_str( strip_formatting( value_tok, strip_formatting_dont_preserve_newlines ) ); value = untyped_str( strip_formatting( to_str(value_tok), strip_formatting_dont_preserve_newlines ) );
// ( <Macro> <ValueType> <Name> = <Expression>, <Macro> <ValueType> <Name> = <Expression> // ( <Macro> <ValueType> <Name> = <Expression>, <Macro> <ValueType> <Name> = <Expression>
} }
// ( <Macro> <ValueType> <Name> = <Expression>, <Macro> <ValueType> <Name> = <Expression>, .. // ( <Macro> <ValueType> <Name> = <Expression>, <Macro> <ValueType> <Name> = <Expression>, ..
@ -2856,7 +2860,7 @@ CodeParam parse_params( bool use_template_capture )
param->Macro = macro; param->Macro = macro;
if ( name.Length > 0 ) if ( name.Length > 0 )
param->Name = get_cached_string( name ); param->Name = get_cached_string( to_str(name) );
param->PostNameMacro = post_name_macro; param->PostNameMacro = post_name_macro;
param->ValueType = type; param->ValueType = type;
@ -2893,7 +2897,7 @@ CodePreprocessCond parse_preprocess_cond()
{ {
push_scope(); push_scope();
if ( ! currtok.is_preprocess_cond() ) if ( ! is_preprocess_cond(currtok) )
{ {
log_failure( "Error, expected preprocess conditional\n%s", Context.to_string() ); log_failure( "Error, expected preprocess conditional\n%s", Context.to_string() );
Context.pop(); Context.pop();
@ -2914,7 +2918,7 @@ CodePreprocessCond parse_preprocess_cond()
} }
Context.Scope->Name = currtok; Context.Scope->Name = currtok;
cond->Content = get_cached_string( currtok ); cond->Content = get_cached_string( to_str(currtok) );
eat( TokType::Preprocess_Content ); eat( TokType::Preprocess_Content );
// #<Conditiona> <Content> // #<Conditiona> <Content>
@ -2993,7 +2997,7 @@ Code parse_simple_preprocess( TokType which )
char const* content = str_fmt_buf( "%.*s ", tok.Length, tok.Text ); char const* content = str_fmt_buf( "%.*s ", tok.Length, tok.Text );
Code result = untyped_str( to_str( content ) ); Code result = untyped_str( GEN_NS to_str(content) );
Context.Scope->Name = tok; Context.Scope->Name = tok;
Context.pop(); Context.pop();
@ -3126,7 +3130,7 @@ CodeVar parse_variable_after_name(
eat( TokType::BraceCurly_Close ); eat( TokType::BraceCurly_Close );
expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text; expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text;
expr = untyped_str( expr_tok ); expr = untyped_str( to_str(expr_tok) );
// <Attributes> <Specifiers> <ValueType> <Name> = { <Expression> } // <Attributes> <Specifiers> <ValueType> <Name> = { <Expression> }
} }
@ -3152,7 +3156,7 @@ CodeVar parse_variable_after_name(
} }
expr_token.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_token.Text; expr_token.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_token.Text;
expr = untyped_str( expr_token ); expr = untyped_str( to_str(expr_token) );
eat( TokType::Capture_End ); eat( TokType::Capture_End );
// <Attributes> <Specifiers> <ValueType> <Name> ( <Expression> ) // <Attributes> <Specifiers> <ValueType> <Name> ( <Expression> )
} }
@ -3177,7 +3181,7 @@ CodeVar parse_variable_after_name(
} }
expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text; expr_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_tok.Text;
bitfield_expr = untyped_str( expr_tok ); bitfield_expr = untyped_str( to_str(expr_tok) );
// <Attributes> <Specifiers> <ValueType> <Name> : <Expression> // <Attributes> <Specifiers> <ValueType> <Name> : <Expression>
} }
@ -3275,9 +3279,9 @@ CodeVar parse_variable_declaration_list()
CodeSpecifiers specifiers = NullCode; CodeSpecifiers specifiers = NullCode;
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
switch ( spec ) switch ( spec )
{ {
@ -3316,7 +3320,7 @@ CodeVar parse_variable_declaration_list()
} }
// , <Specifiers> // , <Specifiers>
StrC name = currtok; StrC name = to_str(currtok);
eat( TokType::Identifier ); eat( TokType::Identifier );
// , <Specifiers> <Name> // , <Specifiers> <Name>
@ -3385,7 +3389,7 @@ CodeConstructor parse_constructor( CodeSpecifiers specifiers )
initializer_list_tok.Length = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )initializer_list_tok.Text; initializer_list_tok.Length = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )initializer_list_tok.Text;
// <Name> ( <Parameters> ) : <InitializerList> // <Name> ( <Parameters> ) : <InitializerList>
initializer_list = untyped_str( initializer_list_tok ); initializer_list = untyped_str( to_str(initializer_list_tok) );
// TODO(Ed): Constructors can have post-fix specifiers // TODO(Ed): Constructors can have post-fix specifiers
@ -3414,7 +3418,7 @@ CodeConstructor parse_constructor( CodeSpecifiers specifiers )
CodeConstructor result = ( CodeConstructor )make_code(); CodeConstructor result = ( CodeConstructor )make_code();
result->Name = get_cached_string(identifier); result->Name = get_cached_string( to_str(identifier));
result->Specs = specifiers; result->Specs = specifiers;
@ -3486,8 +3490,8 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers )
// <Virtual Specifier> ~<Name>() = // <Virtual Specifier> ~<Name>() =
bool skip_formatting = true; bool skip_formatting = true;
Token next = Context.Tokens.next(skip_formatting); Token upcoming = nexttok;
if ( left && next.Text[ 0 ] == '0' ) if ( left && upcoming.Text[ 0 ] == '0' )
{ {
eat( TokType::Operator ); eat( TokType::Operator );
eat( TokType::Number ); eat( TokType::Number );
@ -3495,7 +3499,7 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers )
append(specifiers, ESpecifier::Pure ); append(specifiers, ESpecifier::Pure );
} }
else if ( left && str_compare( next.Text, "default", sizeof("default") - 1 ) == 0) else if ( left && str_compare( upcoming.Text, "default", sizeof("default") - 1 ) == 0)
{ {
body = parse_assignment_expression(); body = parse_assignment_expression();
// <Virtual Specifier> ~< // <Virtual Specifier> ~<
@ -3525,10 +3529,10 @@ CodeDestructor parse_destructor( CodeSpecifiers specifiers )
CodeDestructor result = ( CodeDestructor )make_code(); CodeDestructor result = ( CodeDestructor )make_code();
if ( prefix_identifier ) if ( is_valid(prefix_identifier) )
{ {
prefix_identifier.Length += 1 + identifier.Length; prefix_identifier.Length += 1 + identifier.Length;
result->Name = get_cached_string( prefix_identifier ); result->Name = get_cached_string( to_str(prefix_identifier) );
} }
if ( specifiers ) if ( specifiers )
@ -3646,7 +3650,7 @@ CodeEnum parse_enum( bool inplace_def )
switch ( currtok_noskip.Type ) switch ( currtok_noskip.Type )
{ {
case TokType::NewLine: case TokType::NewLine:
member = untyped_str( currtok_noskip ); member = untyped_str( to_str(currtok_noskip) );
eat( TokType::NewLine ); eat( TokType::NewLine );
break; break;
@ -3730,9 +3734,11 @@ CodeEnum parse_enum( bool inplace_def )
// <Name> = <Expression> <Macro>, // <Inline Comment> // <Name> = <Expression> <Macro>, // <Inline Comment>
// } // }
//Token prev = * previous(Context.Tokens, dont_skip_formatting);
entry.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)entry.Text; entry.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)entry.Text;
member = untyped_str( entry ); member = untyped_str( to_str(entry) );
break; break;
} }
@ -3778,7 +3784,7 @@ CodeEnum parse_enum( bool inplace_def )
result->Type = is_enum_class ? Enum_Class_Fwd : Enum_Fwd; result->Type = is_enum_class ? Enum_Class_Fwd : Enum_Fwd;
} }
result->Name = get_cached_string( name ); result->Name = get_cached_string( to_str(name) );
if ( attributes ) if ( attributes )
result->Attributes = attributes; result->Attributes = attributes;
@ -3836,7 +3842,7 @@ CodeExtern parse_extern_link()
CodeExtern CodeExtern
result = (CodeExtern) make_code(); result = (CodeExtern) make_code();
result->Type = ECode::Extern_Linkage; result->Type = ECode::Extern_Linkage;
result->Name = get_cached_string( name ); result->Name = get_cached_string( to_str(name) );
Code entry = parse_extern_link_body(); Code entry = parse_extern_link_body();
if ( entry == Code_Invalid ) if ( entry == Code_Invalid )
@ -3947,9 +3953,9 @@ CodeFn parse_function()
attributes = parse_attributes(); attributes = parse_attributes();
// <export> <Attributes> // <export> <Attributes>
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
switch ( spec ) switch ( spec )
{ {
@ -3993,7 +3999,7 @@ CodeFn parse_function()
Token name = parse_identifier(); Token name = parse_identifier();
Context.Scope->Name = name; Context.Scope->Name = name;
if ( ! name ) if ( ! is_valid(name) )
{ {
Context.pop(); Context.pop();
return InvalidCode; return InvalidCode;
@ -4030,7 +4036,7 @@ CodeNS parse_namespace()
CodeNS CodeNS
result = (CodeNS) make_code(); result = (CodeNS) make_code();
result->Type = ECode::Namespace; result->Type = ECode::Namespace;
result->Name = get_cached_string( name ); result->Name = get_cached_string( to_str(name) );
result->Body = body; result->Body = body;
@ -4060,9 +4066,9 @@ CodeOperator parse_operator()
attributes = parse_attributes(); attributes = parse_attributes();
// <export> <Attributes> // <export> <Attributes>
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
switch ( spec ) switch ( spec )
{ {
@ -4180,7 +4186,7 @@ CodeOpCast parse_operator_cast( CodeSpecifiers specifiers )
eat( TokType::BraceCurly_Close ); eat( TokType::BraceCurly_Close );
// <Specifiers> <Qualifier> :: ... operator <UnderlyingType>() <const> { <Body> } // <Specifiers> <Qualifier> :: ... operator <UnderlyingType>() <const> { <Body> }
body = untyped_str( body_str ); body = untyped_str( to_str(body_str) );
} }
else else
{ {
@ -4195,8 +4201,8 @@ CodeOpCast parse_operator_cast( CodeSpecifiers specifiers )
CodeOpCast result = (CodeOpCast) make_code(); CodeOpCast result = (CodeOpCast) make_code();
if ( name ) if ( is_valid(name) )
result->Name = get_cached_string( name ); result->Name = get_cached_string( to_str(name) );
if (body) if (body)
{ {
@ -4301,9 +4307,9 @@ CodeTemplate parse_template()
// Specifiers Parsing // Specifiers Parsing
{ {
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
switch ( spec ) switch ( spec )
{ {
@ -4371,12 +4377,12 @@ CodeTemplate parse_template()
for ( ; idx < num(Context.Tokens.Arr); idx++ ) for ( ; idx < num(Context.Tokens.Arr); idx++ )
{ {
Token tok = Context.Tokens[ idx ]; Token tok = Context.Tokens.Arr[ idx ];
if ( tok.Type == TokType::Identifier ) if ( tok.Type == TokType::Identifier )
{ {
idx++; idx++;
tok = Context.Tokens[ idx ]; tok = Context.Tokens.Arr[ idx ];
if ( tok.Type == TokType::Access_StaticSymbol ) if ( tok.Type == TokType::Access_StaticSymbol )
continue; continue;
@ -4443,9 +4449,9 @@ CodeType parse_type( bool from_template, bool* typedef_is_function )
// <Attributes> // <Attributes>
// Prefix specifiers // Prefix specifiers
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
if ( spec != ESpecifier::Const ) if ( spec != ESpecifier::Const )
{ {
@ -4544,7 +4550,7 @@ else if ( currtok.Type == TokType::DeclType )
{ {
name = parse_identifier(); name = parse_identifier();
Context.Scope->Name = name; Context.Scope->Name = name;
if ( ! name ) if ( ! is_valid(name) )
{ {
log_failure( "Error, failed to type signature\n%s", Context.to_string() ); log_failure( "Error, failed to type signature\n%s", Context.to_string() );
Context.pop(); Context.pop();
@ -4558,7 +4564,7 @@ else if ( currtok.Type == TokType::DeclType )
{ {
name = parse_identifier(); name = parse_identifier();
Context.Scope->Name = name; Context.Scope->Name = name;
if ( ! name ) if ( ! is_valid(name) )
{ {
log_failure( "Error, failed to type signature\n%s", Context.to_string() ); log_failure( "Error, failed to type signature\n%s", Context.to_string() );
Context.pop(); Context.pop();
@ -4569,9 +4575,9 @@ else if ( currtok.Type == TokType::DeclType )
} }
// Suffix specifiers for typename. // Suffix specifiers for typename.
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
if ( spec != ESpecifier::Const && spec != ESpecifier::Ptr && spec != ESpecifier::Ref && spec != ESpecifier::RValue ) if ( spec != ESpecifier::Const && spec != ESpecifier::Ptr && spec != ESpecifier::Ref && spec != ESpecifier::RValue )
{ {
@ -4657,7 +4663,7 @@ else if ( currtok.Type == TokType::DeclType )
// String // String
// name_stripped = String::make( GlobalAllocator, name ); // name_stripped = String::make( GlobalAllocator, name );
// name_stripped.strip_space(); // name_stripped.strip_space();
return_type->Name = get_cached_string( name ); return_type->Name = get_cached_string( to_str(name) );
#ifdef GEN_USE_NEW_TYPENAME_PARSING #ifdef GEN_USE_NEW_TYPENAME_PARSING
if ( specifiers ) if ( specifiers )
@ -4769,9 +4775,9 @@ else if ( currtok.Type == TokType::DeclType )
// <Attributes> <ReturnType> <All Kinds of nonsense> ( <Parameters> ) // <Attributes> <ReturnType> <All Kinds of nonsense> ( <Parameters> )
// Look for suffix specifiers for the function // Look for suffix specifiers for the function
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
if ( spec != ESpecifier::Const if ( spec != ESpecifier::Const
// TODO : Add support for NoExcept, l-value, volatile, l-value, etc // TODO : Add support for NoExcept, l-value, volatile, l-value, etc
@ -4814,7 +4820,7 @@ else if ( currtok.Type == TokType::DeclType )
// result->Token = Context.Scope->Start; // result->Token = Context.Scope->Start;
// Need to wait until were using the new parsing method to do this. // Need to wait until were using the new parsing method to do this.
String name_stripped = strip_formatting( name, strip_formatting_dont_preserve_newlines ); String name_stripped = strip_formatting( to_str(name), strip_formatting_dont_preserve_newlines );
// name_stripped.strip_space(); // name_stripped.strip_space();
@ -4922,13 +4928,13 @@ CodeTypedef parse_typedef()
s32 level = 0; s32 level = 0;
for ( ; idx < num(tokens.Arr); idx ++ ) for ( ; idx < num(tokens.Arr); idx ++ )
{ {
if ( tokens[idx].Type == TokType::BraceCurly_Open ) if ( tokens.Arr[idx].Type == TokType::BraceCurly_Open )
level++; level++;
if ( tokens[idx].Type == TokType::BraceCurly_Close ) if ( tokens.Arr[idx].Type == TokType::BraceCurly_Close )
level--; level--;
if ( level == 0 && tokens[idx].Type == TokType::Statement_End ) if ( level == 0 && tokens.Arr[idx].Type == TokType::Statement_End )
break; break;
} }
@ -5055,7 +5061,7 @@ CodeTypedef parse_typedef()
} }
else else
{ {
result->Name = get_cached_string( name ); result->Name = get_cached_string( to_str(name) );
result->IsFunction = false; result->IsFunction = false;
} }
@ -5099,7 +5105,7 @@ CodeUnion parse_union( bool inplace_def )
StrC name = { 0, nullptr }; StrC name = { 0, nullptr };
if ( check( TokType::Identifier ) ) if ( check( TokType::Identifier ) )
{ {
name = currtok; name = to_str(currtok);
Context.Scope->Name = currtok; Context.Scope->Name = currtok;
eat( TokType::Identifier ); eat( TokType::Identifier );
} }
@ -5288,7 +5294,7 @@ CodeUsing parse_using()
CodeUsing CodeUsing
result = (CodeUsing) make_code(); result = (CodeUsing) make_code();
result->Name = get_cached_string( name ); result->Name = get_cached_string( to_str(name) );
result->ModuleFlags = mflags; result->ModuleFlags = mflags;
if ( is_namespace) if ( is_namespace)
@ -5338,9 +5344,9 @@ CodeVar parse_variable()
attributes = parse_attributes(); attributes = parse_attributes();
// <ModuleFlags> <Attributes> // <ModuleFlags> <Attributes>
while ( left && currtok.is_specifier() ) while ( left && is_specifier(currtok) )
{ {
SpecifierT spec = ESpecifier::to_type( currtok ); SpecifierT spec = ESpecifier::to_type( to_str(currtok) );
switch ( spec ) switch ( spec )
{ {
@ -5387,7 +5393,7 @@ CodeVar parse_variable()
Context.Scope->Name = parse_identifier(); Context.Scope->Name = parse_identifier();
// <ModuleFlags> <Attributes> <Specifiers> <ValueType> <Name> // <ModuleFlags> <Attributes> <Specifiers> <ValueType> <Name>
CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, Context.Scope->Name ); CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, to_str(Context.Scope->Name) );
// Regular : <ModuleFlags> <Attributes> <Specifiers> <ValueType> <Name> = <Value>; <InlineCmt> // Regular : <ModuleFlags> <Attributes> <Specifiers> <ValueType> <Name> = <Value>; <InlineCmt>
// Bitfield : <ModuleFlags> <Attributes> <Specifiers> <ValueType> <Name> : <BitfieldSize> = <Value>; <InlineCmt> // Bitfield : <ModuleFlags> <Attributes> <Specifiers> <ValueType> <Name> : <BitfieldSize> = <Value>; <InlineCmt>
@ -5395,8 +5401,7 @@ CodeVar parse_variable()
return result; return result;
} }
// namespace parser GEN_NS_PARSER_END
}
#ifdef CHECK_WAS_DEFINED #ifdef CHECK_WAS_DEFINED
#pragma pop_macro("check") #pragma pop_macro("check")

2
project/dependencies/containers.hpp
View File

@ -385,6 +385,8 @@ bool set_capacity(Array<Type>* array, usize new_capacity)
// TODO(Ed) : This thing needs ALOT of work. // TODO(Ed) : This thing needs ALOT of work.
#pragma region HashTable #pragma region HashTable
#define HashTable(Type) HashTable<Type>
template<class Type> struct HashTable; template<class Type> struct HashTable;
struct HashTableFindResult { struct HashTableFindResult {

6
project/dependencies/platform.hpp
View File

@ -133,12 +133,16 @@
#if GEN_DONT_USE_NAMESPACE || GEN_COMPILER_C #if GEN_DONT_USE_NAMESPACE || GEN_COMPILER_C
# if GEN_COMPILER_C # if GEN_COMPILER_C
# define GEN_NS_PARSER_BEGIN
# define GEN_NS_PARSER_END
# define GEN_NS_ENUM_BEGIN # define GEN_NS_ENUM_BEGIN
# define GEN_NS_ENUM_END # define GEN_NS_ENUM_END
# define GEN_NS # define GEN_NS
# define GEN_NS_BEGIN # define GEN_NS_BEGIN
# define GEN_NS_END # define GEN_NS_END
# else # else
# define GEN_NS_PARSER_BEGIN namespace parser {
# define GEN_NS_PARSER_END }
# define GEN_NS_ENUM_BEGIN namespace gen_internal_enums { # define GEN_NS_ENUM_BEGIN namespace gen_internal_enums {
# define GEN_NS_ENUM_END } # define GEN_NS_ENUM_END }
# define GEN_NS :: # define GEN_NS ::
@ -146,6 +150,8 @@
# define GEN_NS_END # define GEN_NS_END
# endif # endif
#else #else
# define GEN_NS_PARSER_BEGIN namespace parser {
# define GEN_NS_PARSER_END }
# define GEN_NS_ENUM_BEGIN namespace gen_internal_enums { # define GEN_NS_ENUM_BEGIN namespace gen_internal_enums {
# define GEN_NS_ENUM_END } # define GEN_NS_ENUM_END }
# define GEN_NS gen:: # define GEN_NS gen::