ed/gencpp
1
0
Fork 0
mirror of https://github.com/Ed94/gencpp.git synced 2025-06-15 03:01:47 -07:00
Code Issues Packages Projects Releases Wiki Activity

WIP : Fixes and other changes

Browse Source 
* Number literals weren't getting properly lexed
* Fixes for compiler errors with Unreal Engine configuration.
* Support for "post-name" macros in parameters
* Support for variables initializing directly using constructor syntax.
* Explicitly added inline keyword to header inlines for compiling compile library in multiple translation units.
This commit is contained in:
Edward R. Gonzalez
2024-10-25 01:04:17 -04:00
parent e1592ba410
commit b8e1aa6eb7
20 changed files with 248 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
project/components/ast.hpp
View File

@ -339,13 +339,14 @@ struct AST
};
union {
AST* ArrExpr; // Typename
AST* Body; // Class, Constructr, Destructor, Enum, Friend, Function, Namespace, Struct, Union
AST* Body; // Class, Constructor, Destructor, Enum, Friend, Function, Namespace, Struct, Union
AST* Declaration; // Friend, Template
AST* Value; // Parameter, Variable
};
union {
AST* NextVar; // Variable; Possible way to handle comma separated variables declarations. ( , NextVar->Specs NextVar->Name NextVar->ArrExpr = NextVar->Value )
AST* SuffixSpecs; // Only used with typenames, to store the function suffix if typename is function signature. ( May not be needed )
AST* PostNameMacro; // Only used with parameters for specifically UE_REQUIRES (Thanks Unreal)
};
};
StringCached Content; // Attributes, Comment, Execution, Include
@ -375,6 +376,7 @@ struct AST
OperatorT Op;
AccessSpec ParentAccess;
s32 NumEntries;
s32 VarConstructorInit; // Used by variables to know that initialization is using a constructor expression instead of an assignment expression.
};
};
@ -407,6 +409,7 @@ struct AST_POD
union {
AST* NextVar; // Variable; Possible way to handle comma separated variables declarations. ( , NextVar->Specs NextVar->Name NextVar->ArrExpr = NextVar->Value )
AST* SuffixSpecs; // Only used with typenames, to store the function suffix if typename is function signature. ( May not be needed )
AST* PostNameMacro; // Only used with parameters for specifically UE_REQUIRES (Thanks Unreal)
};
};
StringCached Content; // Attributes, Comment, Execution, Include
@ -436,6 +439,7 @@ struct AST_POD
OperatorT Op;
AccessSpec ParentAccess;
s32 NumEntries;
s32 VarConstructorInit; // Used by variables to know that initialization is using a constructor expression instead of an assignment expression.
};
};

5
project/components/ast_types.hpp
View File

@ -644,7 +644,8 @@ struct AST_Param
CodeType ValueType;
Code Macro;
Code Value;
char _PAD_PROPERTIES_3_[ sizeof(AST*) ];
Code PostNameMacro; // Thanks Unreal
// char _PAD_PROPERTIES_3_[sizeof( AST* )];
};
};
CodeParam Last;
@ -1115,7 +1116,7 @@ struct AST_Var
StringCached Name;
CodeT Type;
ModuleFlag ModuleFlags;
char _PAD_UNUSED_[ sizeof(u32) ];
s32 VarConstructorInit;
};
static_assert( sizeof(AST_Var) == sizeof(AST), "ERROR: AST_Var is not the same size as AST");

41
project/components/code_serialization.cpp
View File

@ -480,7 +480,7 @@ void CodeFn::to_string_def( String& result )
if ( ast->Attributes )
result.append_fmt( " %S ", ast->Attributes.to_string() );
b32 prefix_specs = false;
bool prefix_specs = false;
if ( ast->Specs )
{
for ( SpecifierT spec : ast->Specs )
@ -847,6 +847,11 @@ void CodeParam::to_string( String& result )
else if ( ast->ValueType )
result.append_fmt( " %S", ast->ValueType.to_string() );
if ( ast->PostNameMacro )
{
result.append_fmt(" %S", ast->PostNameMacro.to_string() );
}
if ( ast->Value )
result.append_fmt( " = %S", ast->Value.to_string() );
@ -1096,7 +1101,7 @@ String CodeType::to_string()
void CodeType::to_string( String& result )
{
#if GEN_USE_NEW_TYPENAME_PARSING
#if defined(GEN_USE_NEW_TYPENAME_PARSING)
if ( ast->ReturnType && ast->Params )
{
if ( ast->Attributes )
@ -1253,7 +1258,7 @@ void CodeVar::to_string( String& result )
result.append( ast->Name );
if ( ast->ValueType && ast->ValueType->ArrExpr )
if ( ast->ValueType->ArrExpr )
{
result.append_fmt( "[ %S ]", ast->ValueType->ArrExpr.to_string() );
@ -1266,12 +1271,20 @@ void CodeVar::to_string( String& result )
}
if ( ast->Value )
result.append_fmt( " = %S", ast->Value.to_string() );
{
if ( ast->VarConstructorInit )
result.append_fmt( "( %S ", ast->Value.to_string() );
else
result.append_fmt( " = %S", ast->Value.to_string() );
}
// Keep the chain going...
if ( ast->NextVar )
result.append_fmt( ", %S", ast->NextVar.to_string() );
if ( ast->VarConstructorInit )
result.append( " )");
return;
}
@ -1304,11 +1317,19 @@ void CodeVar::to_string( String& result )
result.append_fmt( " : %S", ast->BitfieldSize.to_string() );
if ( ast->Value )
result.append_fmt( " = %S", ast->Value.to_string() );
{
if ( ast->VarConstructorInit )
result.append_fmt( "( %S ", ast->Value.to_string() );
else
result.append_fmt( " = %S", ast->Value.to_string() );
}
if ( ast->NextVar )
result.append_fmt( ", %S", ast->NextVar.to_string() );
if ( ast->VarConstructorInit )
result.append( " )");
if ( ast->InlineCmt )
result.append_fmt("; %S", ast->InlineCmt->Content);
else
@ -1336,11 +1357,19 @@ void CodeVar::to_string( String& result )
result.append_fmt( "%S %S", ast->ValueType.to_string(), ast->Name );
if ( ast->Value )
result.append_fmt( " = %S", ast->Value.to_string() );
{
if ( ast->VarConstructorInit )
result.append_fmt( "( %S ", ast->Value.to_string() );
else
result.append_fmt( " = %S", ast->Value.to_string() );
}
if ( ast->NextVar )
result.append_fmt( ", %S", ast->NextVar.to_string() );
if ( ast->VarConstructorInit )
result.append( " )");
result.append( ";" );
if ( ast->InlineCmt )

1
project/components/header_end.hpp
View File

@ -63,7 +63,6 @@ extern CodeAttributes attrib_api_import;
extern Code module_global_fragment;
extern Code module_private_fragment;
// Exposed, but this is really used for parsing.
extern Code fmt_newline;
extern CodePragma pragma_once;

16
project/components/inlines.hpp
View File

@ -3,6 +3,7 @@
#include "interface.hpp"
#endif
inline
void AST::append( AST* other )
{
if ( other->Parent )
@ -27,6 +28,7 @@ void AST::append( AST* other )
NumEntries++;
}
inline
Code& AST::entry( u32 idx )
{
AST** current = & Front;
@ -42,21 +44,25 @@ Code& AST::entry( u32 idx )
return * rcast( Code*, current);
}
inline
bool AST::has_entries()
{
return NumEntries;
return NumEntries > 0;
}
inline
char const* AST::type_str()
{
return ECode::to_str( Type );
}
inline
AST::operator Code()
{
return { this };
}
inline
Code& Code::operator ++()
{
if ( ast )
@ -65,6 +71,7 @@ Code& Code::operator ++()
return *this;
}
inline
void CodeClass::add_interface( CodeType type )
{
CodeType possible_slot = ast->ParentType;
@ -84,6 +91,7 @@ void CodeClass::add_interface( CodeType type )
possible_slot.ast = type.ast;
}
inline
void CodeParam::append( CodeParam other )
{
AST* self = (AST*) ast;
@ -107,6 +115,7 @@ void CodeParam::append( CodeParam other )
self->NumEntries++;
}
inline
CodeParam CodeParam::get( s32 idx )
{
CodeParam param = *this;
@ -122,17 +131,20 @@ CodeParam CodeParam::get( s32 idx )
return { nullptr };
}
inline
bool CodeParam::has_entries()
{
return ast->NumEntries > 0;
}
inline
CodeParam& CodeParam::operator ++()
{
ast = ast->Next.ast;
return * this;
}
inline
void CodeStruct::add_interface( CodeType type )
{
CodeType possible_slot = ast->ParentType;
@ -152,6 +164,7 @@ void CodeStruct::add_interface( CodeType type )
possible_slot.ast = type.ast;
}
inline
CodeBody def_body( CodeT type )
{
switch ( type )
@ -179,6 +192,7 @@ CodeBody def_body( CodeT type )
return (CodeBody)result;
}
inline
StrC token_fmt_impl( sw num, ... )
{
local_persist thread_local

2
project/components/interface.cpp
View File

@ -374,7 +374,7 @@ AllocatorInfo get_string_allocator( s32 str_length )
uw size_req = str_length + sizeof(String::Header) + sizeof(char*);
if ( last->TotalUsed + size_req > last->TotalSize )
if ( last->TotalUsed + sw(size_req) > last->TotalSize )
{
Arena new_arena = Arena::init_from_allocator( Allocator_StringArena, SizePer_StringArena );

22
project/components/lexer.cpp
View File

@ -586,7 +586,7 @@ TokArray lex( StrC content )
{
s32 length = 0;
char const* scanner = entry.Data;
while ( entry.length() > length && (char_is_alphanumeric( *scanner ) || *scanner == '_') )
while ( entry.length() > length && char_is_alphanumeric( *scanner ) || *scanner == '_' )
{
scanner++;
length ++;
@ -1202,6 +1202,26 @@ TokArray lex( StrC content )
move_forward();
token.Length++;
}
// Handle number literal suffixes in a botched way
if (left && (
current == 'l' || current == 'L' || // long/long long
current == 'u' || current == 'U' || // unsigned
current == 'f' || current == 'F' || // float
current == 'i' || current == 'I' || // imaginary
current == 'z' || current == 'Z')) // complex
{
char prev = current;
move_forward();
token.Length++;
// Handle 'll'/'LL' as a special case when we just processed an 'l'/'L'
if (left && (prev == 'l' || prev == 'L') && (current == 'l' || current == 'L'))
{
move_forward();
token.Length++;
}
}
}
goto FoundToken;

180
project/components/parser.cpp
View File

@ -481,7 +481,7 @@ Code parse_array_decl()
if ( check( TokType::Operator ) && currtok.Text[0] == '[' && currtok.Text[1] == ']' )
{
Code array_expr = untyped_str( get_cached_string(txt(" ")) );
Code array_expr = untyped_str( currtok );
eat( TokType::Operator );
// []
@ -554,44 +554,6 @@ Code parse_array_decl()
return { nullptr };
}
internal inline
Code parse_assignment_expression()
{
Code expr = { nullptr };
eat( TokType::Operator );
// <Attributes> <Specifiers> <ValueType> <Name> =
Token expr_tok = currtok;
if ( currtok.Type == TokType::Statement_End && currtok.Type != TokType::Comma )
{
log_failure( "Expected expression after assignment operator\n%s", Context.to_string() );
Context.pop();
return CodeInvalid;
}
s32 level = 0;
while ( left && currtok.Type != TokType::Statement_End && (currtok.Type != TokType::Comma || level > 0) )
{
if (currtok.Type == TokType::BraceCurly_Open )
level++;
if (currtok.Type == TokType::BraceCurly_Close )
level--;
if (currtok.Type == TokType::Capture_Start)
level++;
else if (currtok.Type == TokType::Capture_End)
level--;
eat( currtok.Type );
}
expr_tok.Length = ( ( sptr )currtok.Text + currtok.Length ) - ( sptr )expr_tok.Text - 1;
expr = untyped_str( expr_tok );
// = <Expression>
return expr;
}
internal inline
CodeAttributes parse_attributes()
{
@ -1349,17 +1311,14 @@ CodeDefine parse_define()
eat( TokType::Identifier );
// #define <Name>
// Defines don't necessarily need content.
#if 0
if ( ! check( TokType::Preprocess_Content ))
{
log_failure( "Error, expected content after #define %s\n%s", define->Name, Context.to_string() );
Context.pop();
return CodeInvalid;
}
#endif
if ( check(TokType::Preprocess_Content) && currtok.Length != 0 )
if ( currtok.Length == 0 )
{
define->Content = get_cached_string( currtok );
eat( TokType::Preprocess_Content );
@ -1377,6 +1336,44 @@ CodeDefine parse_define()
return define;
}
internal inline
Code parse_assignment_expression()
{
Code expr = { nullptr };
eat( TokType::Operator );
// <Attributes> <Specifiers> <ValueType> <Name> =
Token expr_tok = currtok;
if ( currtok.Type == TokType::Statement_End && currtok.Type != TokType::Comma )
{
log_failure( "Expected expression after assignment operator\n%s", Context.to_string() );
Context.pop();
return CodeInvalid;
}
s32 level = 0;
while ( left && currtok.Type != TokType::Statement_End && (currtok.Type != TokType::Comma || level > 0) )
{
if (currtok.Type == TokType::BraceCurly_Open )
level++;
if (currtok.Type == TokType::BraceCurly_Close )
level--;
if (currtok.Type == TokType::Capture_Start)
level++;
else if (currtok.Type == TokType::Capture_End)
level--;
eat( currtok.Type );
}
expr_tok.Length = ( ( sptr )currtok.Text + currtok.Length ) - ( sptr )expr_tok.Text - 1;
expr = untyped_str( expr_tok );
// = <Expression>
return expr;
}
internal inline
Code parse_forward_or_definition( TokType which, bool is_inplace )
{
@ -1407,8 +1404,6 @@ Code parse_forward_or_definition( TokType which, bool is_inplace )
return CodeInvalid;
}
return CodeInvalid;
}
// Function parsing is handled in multiple places because its initial signature is shared with variable parsing
@ -1755,7 +1750,6 @@ CodeBody parse_global_nspace( CodeT which )
case TokType::Spec_Internal_Linkage:
case TokType::Spec_NeverInline:
case TokType::Spec_Static:
case TokType::Spec_ThreadLocal:
{
SpecifierT specs_found[16] { ESpecifier::NumSpecifiers };
s32 NumSpecifiers = 0;
@ -1779,7 +1773,6 @@ CodeBody parse_global_nspace( CodeT which )
case ESpecifier::NeverInline:
case ESpecifier::Static:
case ESpecifier::Volatile:
case ESpecifier::Thread_Local:
break;
case ESpecifier::Consteval:
@ -2300,7 +2293,7 @@ CodeOperator parse_operator_after_ret_type(
case '<':
{
if ( currtok.Text[1] == '=' )
op = LesserEqual;
op = LEqual;
else if ( currtok.Text[1] == '<' )
{
@ -2540,7 +2533,14 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes
Token name = parse_identifier();
Context.Scope->Name = name;
if ( check( TokType::Capture_Start) )
bool detected_capture = check( TokType::Capture_Start );
// Check three tokens ahead to make sure that were not dealing with a constructor initialization...
// ( 350.0f , <--- Could be the scenario
// Example : <Capture_Start> <Value> <Comma>
// idx +1 +2
bool detected_comma = Context.Tokens.Arr[ Context.Tokens.Idx + 2 ].Type == TokType::Comma;
if ( detected_capture && ! detected_comma )
{
// Dealing with a function
result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, type, name );
@ -2625,10 +2625,11 @@ CodeParam parse_params( bool use_template_capture )
return { nullptr };
}
Code macro = { nullptr };
CodeType type = { nullptr };
Code value = { nullptr };
Token name = NullToken;
Code macro = { nullptr };
CodeType type = { nullptr };
Code value = { nullptr };
Token name = NullToken;
Code post_name_macro = { nullptr };
if ( check( TokType::Varadic_Argument ) )
{
@ -2670,6 +2671,15 @@ CodeParam parse_params( bool use_template_capture )
// ( <Macro> <ValueType> <Name>
}
// Unreal has yet another type of macro:
// template<class T UE_REQUIRES(TPointerIsConvertibleFromTo<T, UInterface>::Value)>
// class T ... and then ^this^ UE_REQUIRES shows up
// So we need to consume that.
if ( check( TokType::Preprocess_Macro ))
{
post_name_macro = parse_simple_preprocess( ETokType::Preprocess_Macro );
}
// In template captures you can have a typename have direct assignment without a name
// typename = typename ...
// Which would result in a static value type from a struct expansion (traditionally)
@ -2689,7 +2699,7 @@ CodeParam parse_params( bool use_template_capture )
s32 capture_level = 0;
s32 template_level = 0;
while ( left && (currtok.Type != TokType::Comma) && template_level >= 0 && (CheckEndParams() || capture_level > 0 || template_level > 0) )
while ( left && ( currtok.Type != TokType::Comma ) && template_level >= 0 && CheckEndParams() || capture_level > 0 || template_level > 0 )
{
if (currtok.Text[ 0 ] == '<')
++ template_level;
@ -2773,6 +2783,15 @@ CodeParam parse_params( bool use_template_capture )
// ( <Macro> <ValueType> <Name> = <Expression>, <Macro> <ValueType> <Name>
}
// Unreal has yet another type of macro:
// template<class T UE_REQUIRES(TPointerIsConvertibleFromTo<T, UInterface>::Value)>
// class T ... and then ^this^ UE_REQUIRES shows up
// So we need to consume that.
if ( check( TokType::Preprocess_Macro ))
{
post_name_macro = parse_simple_preprocess( ETokType::Preprocess_Macro );
}
// In template captures you can have a typename have direct assignment without a name
// typename = typename ...
// Which would result in a static value type from a struct expansion (traditionally)
@ -2795,7 +2814,7 @@ CodeParam parse_params( bool use_template_capture )
while ( left
&& currtok.Type != TokType::Comma
&& template_level >= 0
&& (CheckEndParams() || capture_level > 0 || template_level > 0) )
&& CheckEndParams() || capture_level > 0 || template_level > 0 )
{
if (currtok.Text[ 0 ] == '<')
++ template_level;
@ -2829,7 +2848,8 @@ CodeParam parse_params( bool use_template_capture )
if ( name.Length > 0 )
param->Name = get_cached_string( name );
param->ValueType = type;
param->PostNameMacro = post_name_macro;
param->ValueType = type;
if ( value )
param->Value = value;
@ -2872,7 +2892,7 @@ CodePreprocessCond parse_preprocess_cond()
CodePreprocessCond
cond = (CodePreprocessCond) make_code();
cond->Type = scast(CodeT, currtok.Type - (s32(ETokType::Preprocess_If) - s32(ECode::Preprocess_If)) );
cond->Type = scast(CodeT, currtok.Type - ( TokType::Preprocess_If - ECode::Preprocess_If ) );
eat( currtok.Type );
// #<Conditional>
@ -3067,6 +3087,8 @@ CodeVar parse_variable_after_name(
Code expr = { nullptr };
Code bitfield_expr = { nullptr };
b32 using_constructor_initializer = false;
if ( bitfield_is_equal( u32, currtok.Flags, TF_Assign ) )
{
// <Attributes> <Specifiers> <ValueType> <Name> = <Expression>
@ -3098,6 +3120,33 @@ CodeVar parse_variable_after_name(
// <Attributes> <Specifiers> <ValueType> <Name> = { <Expression> }
}
if ( currtok.Type == TokType::Capture_Start )
{
eat( TokType:: Capture_Start);
// <Attributes> <Specifiers> <ValueType> <Name> (
Token expr_token = currtok;
using_constructor_initializer = true;
s32 level = 0;
while ( left && ( currtok.Type != TokType::Capture_End || level > 0 ) )
{
if ( currtok.Type == TokType::Capture_Start )
level++;
else if ( currtok.Type == TokType::Capture_End && level > 0 )
level--;
eat( currtok.Type );
}
expr_token.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)expr_token.Text;
expr = untyped_str( expr_token );
eat( TokType::Capture_End );
// <Attributes> <Specifiers> <ValueType> <Name> ( <Expression> )
}
if ( currtok.Type == TokType::Assign_Classifer )
{
eat( TokType::Assign_Classifer );
@ -3192,6 +3241,8 @@ CodeVar parse_variable_after_name(
result->NextVar->Parent = result;
}
result->VarConstructorInit = using_constructor_initializer;
Context.pop();
return result;
}
@ -3246,7 +3297,7 @@ CodeVar parse_variable_declaration_list()
break;
}
eat(currtok.Type);
// eat(currtok.Type);
if ( specifiers )
specifiers.append( spec );
@ -3650,11 +3701,11 @@ CodeEnum parse_enum( bool inplace_def )
}
// Consume inline comments
if ( currtok.Type == TokType::Comment && prevtok.Line == currtok.Line )
{
eat( TokType::Comment );
// if ( currtok.Type == TokType::Comment && prevtok.Line == currtok.Line )
// {
// eat( TokType::Comment );
// <Name> = <Expression> <Macro>, // <Inline Comment>
}
// }
entry.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)entry.Text;
@ -4399,13 +4450,10 @@ CodeType parse_type( bool from_template, bool* typedef_is_function )
else if ( currtok.Type == TokType::Decl_Class || currtok.Type == TokType::Decl_Enum || currtok.Type == TokType::Decl_Struct
|| currtok.Type == TokType::Decl_Union )
{
Token fwd_key = currtok;
eat( currtok.Type );
// <Attributes> <Specifiers> <class, enum, struct, union>
name = parse_identifier();
fwd_key.Length = sptr(name.Text + name.Length) - sptr(fwd_key.Text);
name = fwd_key;
name = parse_identifier();
// name.Length = ( ( sptr )currtok.Text + currtok.Length ) - ( sptr )name.Text;
// eat( TokType::Identifier );

2
project/components/types.hpp
View File

@ -71,6 +71,7 @@ enum class ModuleFlag : u32
Invalid,
};
inline
StrC to_str( ModuleFlag flag )
{
local_persist
@ -86,6 +87,7 @@ StrC to_str( ModuleFlag flag )
return lookup[ (u32)flag ];
}
inline
ModuleFlag operator|( ModuleFlag A, ModuleFlag B)
{
return (ModuleFlag)( (u32)A | (u32)B );