ed/gencpp
1
0
Fork 0
mirror of https://github.com/Ed94/gencpp.git synced 2025-07-15 18:08:59 -07:00
Code Issues Packages Projects Releases Wiki Activity

Support for module and attribute parsing ( untested )

Browse Source
This commit is contained in:
Edward R. Gonzalez
2023-07-23 22:14:48 -04:00
parent b1de5b1ac7
commit e7374ec328
8 changed files with 400 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

418
project/gen.cpp
View File

@ -17,7 +17,7 @@
namespace gen {
#pragma region StaticData
// TODO : Convert global allocation strategy to use the dual-scratch allocator for a contextual scope.
// TODO : Convert global allocation strategy to use a slab allocation strategy.
global AllocatorInfo GlobalAllocator;
global Array<Arena> Global_AllocatorBuckets;
@ -849,11 +849,6 @@ String AST::to_string()
result.append( "typedef ");
if ( Attributes )
{
result.append_fmt( "%s ", Attributes->to_string() );
}
result.append_fmt( "%s %s", UnderlyingType->to_string(), Name );
if ( UnderlyingType->Type == Typename && UnderlyingType->ArrExpr )
@ -2567,9 +2562,6 @@ CodeTypedef def_typedef( StrC name, Code type, CodeAttributes attributes, Module
result->UnderlyingType = type;
if ( attributes )
result->Attributes = attributes;
return result;
}
@ -3307,77 +3299,83 @@ namespace Parser
It will not be capable of lexing C++ code with unsupported features.
For the sake of scanning files, it can scan preprocessor directives
Attributes_Start is only used to indicate the start of the user_defined attribute list.
*/
# define Define_TokType \
Entry( Access_Private, "private" ) \
Entry( Access_Protected, "protected" ) \
Entry( Access_Public, "public" ) \
Entry( Access_MemberSymbol, "." ) \
Entry( Access_StaticSymbol, "::") \
Entry( Ampersand, "&" ) \
Entry( Ampersand_DBL, "&&" ) \
Entry( Assign_Classifer, ":" ) \
Entry( BraceCurly_Open, "{" ) \
Entry( BraceCurly_Close, "}" ) \
Entry( BraceSquare_Open, "[" ) \
Entry( BraceSquare_Close, "]" ) \
Entry( Capture_Start, "(" ) \
Entry( Capture_End, ")" ) \
Entry( Comment, "__comment__" ) \
Entry( Char, "__char__" ) \
Entry( Comma, "," ) \
Entry( Decl_Class, "class" ) \
Entry( Decl_Enum, "enum" ) \
Entry( Decl_Extern_Linkage, "extern" ) \
Entry( Decl_Friend, "friend" ) \
Entry( Decl_Module, "module" ) \
Entry( Decl_Namespace, "namespace" ) \
Entry( Decl_Operator, "operator" ) \
Entry( Decl_Struct, "struct" ) \
Entry( Decl_Template, "template" ) \
Entry( Decl_Typedef, "typedef" ) \
Entry( Decl_Using, "using" ) \
Entry( Decl_Union, "union" ) \
Entry( Identifier, "__identifier__" ) \
Entry( Module_Import, "import" ) \
Entry( Module_Export, "export" ) \
Entry( Number, "number" ) \
Entry( Operator, "operator" ) \
Entry( Preprocessor_Directive, "#") \
Entry( Preprocessor_Include, "include" ) \
Entry( Spec_Alignas, "alignas" ) \
Entry( Spec_Const, "const" ) \
Entry( Spec_Consteval, "consteval" ) \
Entry( Spec_Constexpr, "constexpr" ) \
Entry( Spec_Constinit, "constinit" ) \
Entry( Spec_Extern, "extern" ) \
Entry( Spec_Global, "global" ) \
Entry( Spec_Inline, "inline" ) \
Entry( Spec_Internal_Linkage, "internal" ) \
Entry( Spec_LocalPersist, "local_persist" ) \
Entry( Spec_Mutable, "mutable" ) \
Entry( Spec_Static, "static" ) \
Entry( Spec_ThreadLocal, "thread_local" ) \
Entry( Spec_Volatile, "volatile") \
Entry( Star, "*" ) \
Entry( Statement_End, ";" ) \
Entry( String, "__string__" ) \
Entry( Type_Unsigned, "unsigned" ) \
Entry( Type_Signed, "signed" ) \
Entry( Type_Short, "short" ) \
Entry( Type_Long, "long" ) \
Entry( Type_char, "char" ) \
Entry( Type_int, "int" ) \
Entry( Type_double, "double" )
Entry( Access_Private, "private" ) \
Entry( Access_Protected, "protected" ) \
Entry( Access_Public, "public" ) \
Entry( Access_MemberSymbol, "." ) \
Entry( Access_StaticSymbol, "::") \
Entry( Ampersand, "&" ) \
Entry( Ampersand_DBL, "&&" ) \
Entry( Assign_Classifer, ":" ) \
Entry( Attribute_Open, "[[" ) \
Entry( Attribute_Close, "]]" ) \
Entry( BraceCurly_Open, "{" ) \
Entry( BraceCurly_Close, "}" ) \
Entry( BraceSquare_Open, "[" ) \
Entry( BraceSquare_Close, "]" ) \
Entry( Capture_Start, "(" ) \
Entry( Capture_End, ")" ) \
Entry( Comment, "__comment__" ) \
Entry( Char, "__char__" ) \
Entry( Comma, "," ) \
Entry( Decl_Class, "class" ) \
Entry( Decl_GNU_Attribute, "__attribute__" ) \
Entry( Decl_MSVC_Attribute, "__declspec" ) \
Entry( Decl_Enum, "enum" ) \
Entry( Decl_Extern_Linkage, "extern" ) \
Entry( Decl_Friend, "friend" ) \
Entry( Decl_Module, "module" ) \
Entry( Decl_Namespace, "namespace" ) \
Entry( Decl_Operator, "operator" ) \
Entry( Decl_Struct, "struct" ) \
Entry( Decl_Template, "template" ) \
Entry( Decl_Typedef, "typedef" ) \
Entry( Decl_Using, "using" ) \
Entry( Decl_Union, "union" ) \
Entry( Identifier, "__identifier__" ) \
Entry( Module_Import, "import" ) \
Entry( Module_Export, "export" ) \
Entry( Number, "number" ) \
Entry( Operator, "operator" ) \
Entry( Preprocessor_Directive, "#") \
Entry( Preprocessor_Include, "include" ) \
Entry( Spec_Alignas, "alignas" ) \
Entry( Spec_Const, "const" ) \
Entry( Spec_Consteval, "consteval" ) \
Entry( Spec_Constexpr, "constexpr" ) \
Entry( Spec_Constinit, "constinit" ) \
Entry( Spec_Extern, "extern" ) \
Entry( Spec_Global, "global" ) \
Entry( Spec_Inline, "inline" ) \
Entry( Spec_Internal_Linkage, "internal" ) \
Entry( Spec_LocalPersist, "local_persist" ) \
Entry( Spec_Mutable, "mutable" ) \
Entry( Spec_Static, "static" ) \
Entry( Spec_ThreadLocal, "thread_local" ) \
Entry( Spec_Volatile, "volatile") \
Entry( Star, "*" ) \
Entry( Statement_End, ";" ) \
Entry( String, "__string__" ) \
Entry( Type_Unsigned, "unsigned" ) \
Entry( Type_Signed, "signed" ) \
Entry( Type_Short, "short" ) \
Entry( Type_Long, "long" ) \
Entry( Type_char, "char" ) \
Entry( Type_int, "int" ) \
Entry( Type_double, "double" ) \
Entry( Attributes_Start, "__attrib_start__" )
enum class TokType : u32
{
# define Entry( Name_, Str_ ) Name_,
Define_TokType
GEN_Define_Attribute_Tokens
# undef Entry
Attr_Keyword,
Num,
Invalid
};
@ -3408,9 +3406,8 @@ namespace Parser
{
# define Entry( Name_, Str_ ) { sizeof(Str_), Str_ },
Define_TokType
GEN_Define_Attribute_Tokens
# undef Entry
{ sizeof(Attribute::Keyword), Attribute::Keyword },
};
for ( u32 index = 0; index < (u32)TokType::Num; index++ )
@ -3436,9 +3433,8 @@ namespace Parser
{
# define Entry( Name_, Str_ ) Str_,
Define_TokType
GEN_Define_Attribute_Tokens
# undef Entry
Attribute::Keyword,
};
return lookup[(u32)type];
@ -3467,6 +3463,11 @@ namespace Parser
return scast(AccessSpec, tok.Type);
}
internal inline
bool tok_is_attribute( Token const& tok )
{
return tok.Type > TokType::Attributes_Start;
}
struct TokArray
{
@ -4108,6 +4109,68 @@ Code parse_array_decl( Parser::TokArray& toks, char const* context )
return { nullptr };
}
internal inline
CodeAttributes parse_attributes( Parser::TokArray& toks, char const* context )
{
using namespace Parser;
Token start;
if ( check(TokType::Attribute_Open) )
{
eat( TokType::Attribute_Open);
while ( left && currtok.Type != TokType::Attribute_Close )
{
eat( currtok.Type );
}
eat( TokType::Attribute_Close );
}
else if ( check(TokType::Decl_GNU_Attribute) )
{
eat(TokType::BraceCurly_Open);
eat(TokType::BraceCurly_Open);
while ( left && currtok.Type != TokType::BraceCurly_Close )
{
eat(currtok.Type);
}
eat(TokType::BraceCurly_Close);
eat(TokType::BraceCurly_Close);
}
else if ( check(TokType::Decl_MSVC_Attribute) )
{
eat( TokType::Decl_MSVC_Attribute );
eat( TokType::BraceCurly_Open);
while ( left && currtok.Type != TokType::BraceCurly_Close )
{
eat(currtok.Type);
}
eat(TokType::BraceCurly_Close);
}
else if ( tok_is_attribute( currtok ) )
{
eat(currtok.Type);
}
s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text;
if ( len > 0 )
{
StrC attribute_txt = { len, start.Text };
return def_attributes( attribute_txt );
}
return { nullptr };
}
internal inline
Parser::Token parse_identifier( Parser::TokArray& toks, char const* context )
{
@ -4288,7 +4351,7 @@ CodeParam parse_params( Parser::TokArray& toks, char const* context, bool use_te
// Function parsing is handled in multiple places because its initial signature is shared with variable parsing
internal inline
CodeFn parse_function_after_name(
ModuleFlag mflags
ModuleFlag mflags
, CodeAttributes attributes
, CodeSpecifier specifiers
, CodeType ret_type
@ -4711,7 +4774,7 @@ Code parse_operator_function_or_variable( bool expects_function, CodeAttributes
{
if ( expects_function )
{
log_failure( "gen::parse_template: expected function declaration (consteval was used)" );
log_failure( "gen::parse_operator_function_or_variable: expected function declaration (consteval was used)" );
return Code::Invalid;
}
@ -4809,36 +4872,14 @@ CodeBody parse_class_struct_body( Parser::TokType which, Parser::TokArray& toks,
member = parse_using( toks, context );
break;
case TokType::BraceSquare_Open:
case TokType::Attr_Keyword:
case TokType::Attribute_Open:
case TokType::Decl_GNU_Attribute:
case TokType::Decl_MSVC_Attribute:
#define Entry( attribute, str ) case TokType::attribute:
GEN_Define_Attribute_Tokens
#undef Entry
{
// Standard attribute
if ( currtok.Type == TokType::BraceSquare_Open )
{
eat( TokType::BraceSquare_Open );
if ( currtok.Type != TokType::BraceSquare_Open )
{
log_failure( "%s: Error, expected attribute name", context );
return result;
}
while ( left && currtok.Type != TokType::BraceSquare_Close )
{
// TODO : Parse attributes
}
eat( TokType::BraceSquare_Close );
eat( TokType::BraceSquare_Close );
}
// Platform Specific attribute
eat( TokType::Attr_Keyword );
eat( TokType::Capture_Start );
// TODO : Parse attributes
eat( TokType::Capture_End );
attributes = parse_attributes( toks, context );
}
//! Fallthrough intended
case TokType::Spec_Consteval:
@ -4943,11 +4984,15 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con
CodeClass result = CodeInvalid;
// TODO : Parse module specifiers
if ( check(TokType::Module_Export) )
{
mflags = ModuleFlag::Export;
eat( TokType::Module_Export );
}
eat( which );
// TODO : Parse attributes
attributes = parse_attributes( toks, context );
if ( check( TokType::Identifier ) )
name = parse_identifier( toks, context );
@ -4979,16 +5024,14 @@ Code parse_class_struct( Parser::TokType which, Parser::TokArray& toks, char con
if ( which == TokType::Decl_Class )
result = def_class( name, body, parent, access
// TODO : Set these up later
, NoCode // Attributes
, ModuleFlag::None
, attributes
, mflags
);
else
result = def_struct( name, body, (CodeType)parent, access
// TODO : Set these up later
, NoCode // Attributes
, ModuleFlag::None
, attributes
, mflags
);
return result;
@ -5115,38 +5158,13 @@ CodeBody parse_global_nspace( CodeT which, Parser::TokArray& toks, char const* c
not_implemented( context );
}
//! Fallthrough intentional
case TokType::BraceSquare_Open:
case TokType::Attr_Keyword:
case TokType::Decl_GNU_Attribute:
case TokType::Decl_MSVC_Attribute:
#define Entry( attribute, str ) case TokType::attribute:
GEN_Define_Attribute_Tokens
#undef Entry
{
not_implemented( context );
// Standard attribute
if ( currtok.Type == TokType::BraceSquare_Open )
{
eat( TokType::BraceSquare_Open );
if ( currtok.Type != TokType::BraceSquare_Open )
{
log_failure( "%s: Error, expected attribute name", context );
return result;
}
while ( left && currtok.Type != TokType::BraceSquare_Close )
{
// TODO : Parse attributes
}
eat( TokType::BraceSquare_Close );
eat( TokType::BraceSquare_Close );
}
// Platform Specific attribute
eat( TokType::Attr_Keyword );
eat( TokType::Capture_Start );
// TODO : Parse attributes
eat( TokType::Capture_End );
attributes = parse_attributes( toks, context );
}
//! Fallthrough intentional
case TokType::Spec_Consteval:
@ -5494,10 +5512,15 @@ CodeFn parse_functon( Parser::TokArray& toks, char const* context )
CodeAttributes attributes = { nullptr };
CodeSpecifier specifiers = { nullptr };
ModuleFlag mflags = ModuleFlag::None;
// TODO : Parse module specifiers
if ( check(TokType::Module_Export) )
{
mflags = ModuleFlag::Export;
eat( TokType::Module_Export );
}
// TODO : Parse attributes
attributes = parse_attributes( toks, context );
while ( left && tok_is_specifier( currtok ) )
{
@ -5536,7 +5559,7 @@ CodeFn parse_functon( Parser::TokArray& toks, char const* context )
if ( ! name )
return CodeInvalid;
CodeFn result = parse_function_after_name( ModuleFlag::None, attributes, specifiers, ret_type, name, toks, context );
CodeFn result = parse_function_after_name( mflags, attributes, specifiers, ret_type, name, toks, context );
return result;
}
@ -5603,16 +5626,23 @@ CodeNamespace parse_namespace( StrC def )
internal
CodeOperator parse_operator( Parser::TokArray& toks, char const* context )
{
// TODO : Parse Module specifier
using namespace Parser;
// TODO : Parse Attributes
CodeAttributes attributes = { nullptr };
CodeSpecifier specifiers = { nullptr };
CodeSpecifier specifiers = { nullptr };
ModuleFlag mflags = ModuleFlag::None;
SpecifierT specs_found[16] { ESpecifier::Num_Specifiers };
s32 num_specifiers = 0;
if ( check(TokType::Module_Export) )
{
mflags = ModuleFlag::Export;
eat( TokType::Module_Export );
}
attributes = parse_attributes( toks, context );
while ( left && tok_is_specifier( currtok ) )
{
SpecifierT spec = ESpecifier::to_type( currtok );
@ -5643,7 +5673,7 @@ CodeOperator parse_operator( Parser::TokArray& toks, char const* context )
// Parse Return Type
CodeType ret_type = parse_type( toks, stringize(parse_operator) );
CodeOperator result = parse_operator_after_ret_type( ModuleFlag::None, attributes, specifiers, ret_type, toks, context );
CodeOperator result = parse_operator_after_ret_type( mflags, attributes, specifiers, ret_type, toks, context );
return result;
}
@ -5751,7 +5781,13 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context )
using namespace Parser;
// TODO : Parse Module specifier
ModuleFlag mflags = ModuleFlag::None;
if ( check(TokType::Module_Export) )
{
mflags = ModuleFlag::Export;
eat( TokType::Module_Export );
}
eat( TokType::Decl_Template );
@ -5784,17 +5820,16 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context )
// Its either a function or a variable
Token name = { nullptr, 0, TokType::Invalid };
CodeAttributes attributes = { nullptr };
CodeSpecifier specifiers = { nullptr };
// TODO : Parse attributes
CodeAttributes attributes = { nullptr };
CodeSpecifier specifiers = { nullptr };
bool expects_function = false;
SpecifierT specs_found[16] { ESpecifier::Num_Specifiers };
s32 num_specifiers = 0;
attributes = parse_attributes( toks, stringize(parse_template) );
while ( left && tok_is_specifier( currtok ) )
{
SpecifierT spec = ESpecifier::to_type( currtok );
@ -5846,6 +5881,7 @@ CodeTemplate parse_template( Parser::TokArray& toks, char const* context )
result->Type = ECode::Template;
result->Params = params;
result->Declaration = definition;
result->ModuleFlags = mflags;
return result;
# undef UseTemplateCapture
@ -5876,6 +5912,8 @@ CodeType parse_type( Parser::TokArray& toks, char const* context )
Token name = { nullptr, 0, TokType::Invalid };
Token brute_sig = { currtok.Text, 0, TokType::Invalid };
CodeAttributes attributes = parse_attributes( toks, context );
while ( left && tok_is_specifier( currtok ) )
{
SpecifierT spec = ESpecifier::to_type( currtok );
@ -6030,6 +6068,9 @@ CodeType parse_type( Parser::TokArray& toks, char const* context )
result->Name = get_cached_string( name );
if ( attributes )
result->Attributes = attributes;
return result;
}
@ -6056,6 +6097,14 @@ CodeTypedef parse_typedef( Parser::TokArray& toks, char const* context )
Code array_expr = { nullptr };
Code type = { nullptr };
ModuleFlag mflags = ModuleFlag::None;
if ( check(TokType::Module_Export) )
{
mflags = ModuleFlag::Export;
eat( TokType::Module_Export );
}
eat( TokType::Decl_Typedef );
if ( check( TokType::Decl_Enum ) )
@ -6086,9 +6135,10 @@ CodeTypedef parse_typedef( Parser::TokArray& toks, char const* context )
using namespace ECode;
CodeTypedef
result = (CodeTypedef) make_code();
result->Type = Typedef;
result->Name = get_cached_string( name );
result = (CodeTypedef) make_code();
result->Type = Typedef;
result->Name = get_cached_string( name );
result->ModuleFlags = mflags;
result->UnderlyingType = type;
@ -6115,12 +6165,17 @@ CodeUnion parse_union( Parser::TokArray& toks, char const* context )
{
using namespace Parser;
// TODO : Parse module spec
ModuleFlag mflags = ModuleFlag::None;
if ( check(TokType::Module_Export) )
{
mflags = ModuleFlag::Export;
eat( TokType::Module_Export );
}
eat( TokType::Decl_Union );
// TODO : Parse attributes
CodeAttributes attributes = { nullptr };
CodeAttributes attributes = parse_attributes( toks, context );
StrC name = { 0, nullptr };
@ -6150,7 +6205,8 @@ CodeUnion parse_union( Parser::TokArray& toks, char const* context )
CodeUnion
result = (CodeUnion) make_code();
result->Type = ECode::Union;
result->Type = ECode::Union;
result->ModuleFlags = mflags;
if ( name )
result->Name = get_cached_string( name );
@ -6190,7 +6246,14 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context )
bool is_namespace = false;
// TODO : Parse module specs
ModuleFlag mflags = ModuleFlag::None;
CodeAttributes attributes = { nullptr };
if ( check(TokType::Module_Export) )
{
mflags = ModuleFlag::Export;
eat( TokType::Module_Export );
}
eat( TokType::Decl_Using );
@ -6205,7 +6268,7 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context )
if ( currtok.IsAssign )
{
// TODO : Parse Attributes (using type-alias)
attributes = parse_attributes( toks, context );
eat( TokType::Operator );
@ -6219,8 +6282,9 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context )
using namespace ECode;
CodeUsing
result = (CodeUsing) make_code();
result->Name = get_cached_string( name );
result = (CodeUsing) make_code();
result->Name = get_cached_string( name );
result->ModuleFlags = mflags;
if ( is_namespace)
{
@ -6235,6 +6299,9 @@ CodeUsing parse_using( Parser::TokArray& toks, char const* context )
if ( array_expr )
type->ArrExpr = array_expr;
if ( attributes )
result->Attributes = attributes;
}
return result;
}
@ -6261,12 +6328,17 @@ CodeVar parse_variable( Parser::TokArray& toks, char const* context )
SpecifierT specs_found[16] { ESpecifier::Num_Specifiers };
s32 num_specifiers = 0;
ModuleFlag mflags = ModuleFlag::None;
CodeAttributes attributes = { nullptr };
CodeSpecifier specifiers = { nullptr };
// TODO : Parse module specifiers
if ( check(TokType::Module_Export) )
{
mflags = ModuleFlag::Export;
eat( TokType::Module_Export );
}
// TODO : Parse attributes
attributes = parse_attributes( toks, context );
while ( left && tok_is_specifier( currtok ) )
{
@ -6314,7 +6386,7 @@ CodeVar parse_variable( Parser::TokArray& toks, char const* context )
name = currtok;
eat( TokType::Identifier );
CodeVar result = parse_variable_after_name( ModuleFlag::None, attributes, specifiers, type, name, toks, context );
CodeVar result = parse_variable_after_name( mflags, attributes, specifiers, type, name, toks, context );
return result;
}