Heavy refactor..

Isolating large macros to their own directory (components/temp).
- Plan is to remove them soon with proper generation.

Added additional component files, separating the old data_structures header for a set of ast headers.
Header_end also had its inlines extracted out.
Necessary to complete the macro isolation.

ZPL parser dependencies were removed from the core library along with builder, its now generated in bootstrap as pare of making a gen_builder set of files.

Singleheader will be changed in next few commits to reflect this as well (By making builder deps and components a conditional option).

Tests are most likely all broken for now.
This commit is contained in:
2023-08-03 11:01:43 -04:00
parent 114f678f1b
commit 5d7dfaf666
63 changed files with 3341 additions and 1382 deletions

View File

@@ -0,0 +1,8 @@
# Temporary Code
These are heavy macro code used throughout the library thats intended to be replaced with codegen done with the library itself.
The reason for this is to minimize macro generation to only trivial cases.
This makes the library more verbose but makes it easier to debug which is of higher priority.
Any sort of verbosity cost will be mitigated with better docs and heavy usage of pragma regions.

View File

@@ -0,0 +1,179 @@
// This is the non-bootstraped version of the Common AST Implementation. This will be obsolete once bootstrap is stress tested.
#pragma region AST Common
#define Define_CodeImpl( Typename ) \
char const* Typename::debug_str() \
{ \
if ( ast == nullptr ) \
return "Code::debug_str: AST is null!"; \
\
return rcast(AST*, ast)->debug_str(); \
} \
Code Typename::duplicate() \
{ \
if ( ast == nullptr ) \
{ \
log_failure("Code::duplicate: Cannot duplicate code, AST is null!"); \
return Code::Invalid; \
} \
\
return { rcast(AST*, ast)->duplicate() }; \
} \
bool Typename::is_equal( Code other ) \
{ \
if ( ast == nullptr || other.ast == nullptr ) \
{ \
log_failure("Code::is_equal: Cannot compare code, AST is null!"); \
return false; \
} \
\
return rcast(AST*, ast)->is_equal( other.ast ); \
} \
bool Typename::is_valid() \
{ \
return (AST*) ast != nullptr && rcast( AST*, ast)->Type != CodeT::Invalid; \
} \
void Typename::set_global() \
{ \
if ( ast == nullptr ) \
{ \
log_failure("Code::set_global: Cannot set code as global, AST is null!"); \
return; \
} \
\
rcast(AST*, ast)->Parent = Code::Global.ast; \
} \
String Typename::to_string() \
{ \
if ( ast == nullptr ) \
{ \
log_failure("Code::to_string: Cannot convert code to string, AST is null!"); \
return { nullptr }; \
} \
\
return rcast(AST*, ast)->to_string(); \
} \
Typename& Typename::operator =( Code other ) \
{ \
if ( other.ast && other->Parent ) \
{ \
ast = rcast( decltype(ast), other.ast->duplicate() ); \
rcast( AST*, ast)->Parent = nullptr; \
} \
\
ast = rcast( decltype(ast), other.ast ); \
return *this; \
} \
bool Typename::operator ==( Code other ) \
{ \
return (AST*) ast == other.ast; \
} \
bool Typename::operator !=( Code other ) \
{ \
return (AST*) ast != other.ast; \
} \
Typename::operator bool() \
{ \
return ast != nullptr; \
}
Define_CodeImpl( Code );
Define_CodeImpl( CodeBody );
Define_CodeImpl( CodeAttributes );
Define_CodeImpl( CodeComment );
Define_CodeImpl( CodeClass );
Define_CodeImpl( CodeDefine );
Define_CodeImpl( CodeEnum );
Define_CodeImpl( CodeExec );
Define_CodeImpl( CodeExtern );
Define_CodeImpl( CodeInclude );
Define_CodeImpl( CodeFriend );
Define_CodeImpl( CodeFn );
Define_CodeImpl( CodeModule );
Define_CodeImpl( CodeNamespace );
Define_CodeImpl( CodeOperator );
Define_CodeImpl( CodeOpCast );
Define_CodeImpl( CodeParam );
Define_CodeImpl( CodePragma );
Define_CodeImpl( CodePreprocessCond );
Define_CodeImpl( CodeSpecifiers );
Define_CodeImpl( CodeStruct );
Define_CodeImpl( CodeTemplate );
Define_CodeImpl( CodeType );
Define_CodeImpl( CodeTypedef );
Define_CodeImpl( CodeUnion );
Define_CodeImpl( CodeUsing );
Define_CodeImpl( CodeVar );
#undef Define_CodeImpl
#define Define_AST_Cast( typename ) \
AST::operator Code ## typename() \
{ \
return { rcast( AST_ ## typename*, this ) }; \
}
Define_AST_Cast( Body );
Define_AST_Cast( Attributes );
Define_AST_Cast( Comment );
Define_AST_Cast( Class );
Define_AST_Cast( Define );
Define_AST_Cast( Enum );
Define_AST_Cast( Exec );
Define_AST_Cast( Extern );
Define_AST_Cast( Include );
Define_AST_Cast( Friend );
Define_AST_Cast( Fn );
Define_AST_Cast( Module );
Define_AST_Cast( Namespace );
Define_AST_Cast( Operator );
Define_AST_Cast( OpCast );
Define_AST_Cast( Param );
Define_AST_Cast( Pragma );
Define_AST_Cast( PreprocessCond );
Define_AST_Cast( Struct );
Define_AST_Cast( Specifiers );
Define_AST_Cast( Template );
Define_AST_Cast( Type );
Define_AST_Cast( Typedef );
Define_AST_Cast( Union );
Define_AST_Cast( Using );
Define_AST_Cast( Var );
#undef Define_AST_Cast
#define Define_CodeCast( type ) \
Code::operator Code ## type() const \
{ \
return { (AST_ ## type*) ast }; \
}
Define_CodeCast( Attributes );
Define_CodeCast( Comment );
Define_CodeCast( Class );
Define_CodeCast( Define );
Define_CodeCast( Exec );
Define_CodeCast( Enum );
Define_CodeCast( Extern );
Define_CodeCast( Include );
Define_CodeCast( Friend );
Define_CodeCast( Fn );
Define_CodeCast( Module );
Define_CodeCast( Namespace );
Define_CodeCast( Operator );
Define_CodeCast( OpCast );
Define_CodeCast( Param );
Define_CodeCast( Pragma );
Define_CodeCast( PreprocessCond );
Define_CodeCast( Specifiers );
Define_CodeCast( Struct );
Define_CodeCast( Template );
Define_CodeCast( Type );
Define_CodeCast( Typedef );
Define_CodeCast( Union );
Define_CodeCast( Using );
Define_CodeCast( Var );
Define_CodeCast( Body);
#undef Define_CodeCast
#pragma endregion AST Common

View File

@@ -0,0 +1,87 @@
// This is the non-bootstraped version of the ECode. This will be obsolete once bootstrap is stress tested.
namespace ECode
{
# define Define_Types \
Entry( Invalid ) \
Entry( Untyped ) \
Entry( Comment ) \
Entry( Access_Private ) \
Entry( Access_Protected ) \
Entry( Access_Public ) \
Entry( PlatformAttributes ) \
Entry( Class ) \
Entry( Class_Fwd ) \
Entry( Class_Body ) \
Entry( Enum ) \
Entry( Enum_Fwd ) \
Entry( Enum_Body ) \
Entry( Enum_Class ) \
Entry( Enum_Class_Fwd ) \
Entry( Execution ) \
Entry( Export_Body ) \
Entry( Extern_Linkage ) \
Entry( Extern_Linkage_Body ) \
Entry( Friend ) \
Entry( Function ) \
Entry( Function_Fwd ) \
Entry( Function_Body ) \
Entry( Global_Body ) \
Entry( Module ) \
Entry( Namespace ) \
Entry( Namespace_Body ) \
Entry( Operator ) \
Entry( Operator_Fwd ) \
Entry( Operator_Member ) \
Entry( Operator_Member_Fwd ) \
Entry( Operator_Cast ) \
Entry( Operator_Cast_Fwd ) \
Entry( Parameters ) \
Entry( Preprocess_Define ) \
Entry( Preprocess_If ) \
Entry( Preprocess_IfDef ) \
Entry( Preprocess_IfNotDef ) \
Entry( Preprocess_ElIf ) \
Entry( Preprocess_Else ) \
Entry( Preprocess_EndIf ) \
Entry( Preprocess_Include ) \
Entry( Preprocess_Pragma ) \
Entry( Specifiers ) \
Entry( Struct ) \
Entry( Struct_Fwd ) \
Entry( Struct_Body ) \
Entry( Template ) \
Entry( Typedef ) \
Entry( Typename ) \
Entry( Union ) \
Entry( Union_Body) \
Entry( Using ) \
Entry( Using_Namespace ) \
Entry( Variable )
enum Type : u32
{
# define Entry( Type ) Type,
Define_Types
# undef Entry
Num_Types
};
inline
StrC to_str( Type type )
{
static
StrC lookup[Num_Types] = {
# define Entry( Type ) { sizeof(stringize(Type)), stringize(Type) },
Define_Types
# undef Entry
};
return lookup[ type ];
}
# undef Define_Types
}
using CodeT = ECode::Type;

View File

@@ -0,0 +1,76 @@
// This is the non-bootstraped version of the EOperator. This will be obsolete once bootstrap is stress tested.
namespace EOperator
{
# define Define_Operators \
Entry( Invalid, INVALID ) \
Entry( Assign, = ) \
Entry( Assign_Add, += ) \
Entry( Assign_Subtract, -= ) \
Entry( Assign_Multiply, *= ) \
Entry( Assign_Divide, /= ) \
Entry( Assign_Modulo, %= ) \
Entry( Assign_BAnd, &= ) \
Entry( Assign_BOr, |= ) \
Entry( Assign_BXOr, ^= ) \
Entry( Assign_LShift, <<= ) \
Entry( Assign_RShift, >>= ) \
Entry( Increment, ++ ) \
Entry( Decrement, -- ) \
Entry( Unary_Plus, + ) \
Entry( Unary_Minus, - ) \
Entry( UnaryNot, ! ) \
Entry( Add, + ) \
Entry( Subtract, - ) \
Entry( Multiply, * ) \
Entry( Divide, / ) \
Entry( Modulo, % ) \
Entry( BNot, ~ ) \
Entry( BAnd, & ) \
Entry( BOr, | ) \
Entry( BXOr, ^ ) \
Entry( LShift, << ) \
Entry( RShift, >> ) \
Entry( LAnd, && ) \
Entry( LOr, || ) \
Entry( LEqual, == ) \
Entry( LNot, != ) \
Entry( Lesser, < ) \
Entry( Greater, > ) \
Entry( LesserEqual, <= ) \
Entry( GreaterEqual, >= ) \
Entry( Subscript, [] ) \
Entry( Indirection, * ) \
Entry( AddressOf, & ) \
Entry( MemberOfPointer, -> ) \
Entry( PtrToMemOfPtr, ->* ) \
Entry( FunctionCall, () )
enum Type : u32
{
# define Entry( Type_, Token_ ) Type_,
Define_Operators
# undef Entry
Comma,
Num_Ops,
};
inline
char const* to_str( Type op )
{
local_persist
char const* lookup[ Num_Ops ] = {
# define Entry( Type_, Token_ ) stringize(Token_),
Define_Operators
# undef Entry
","
};
return lookup[ op ];
}
# undef Define_Operators
}
using OperatorT = EOperator::Type;

View File

@@ -0,0 +1,109 @@
// This is the non-bootstraped version of the ESpecifier. This will be obsolete once bootstrap is stress tested.
namespace ESpecifier
{
/*
Note: The following are handled separately:
attributes
alignas
*/
# define Define_Specifiers \
Entry( Invalid, INVALID ) \
Entry( Consteval, consteval ) \
Entry( Constexpr, constexpr ) \
Entry( Constinit, constinit ) \
Entry( Explicit, explicit ) \
Entry( External_Linkage, extern ) \
Entry( Global, global ) \
Entry( Inline, inline ) \
Entry( Internal_Linkage, internal ) \
Entry( Local_Persist, local_persist ) \
Entry( Mutable, mutable ) \
Entry( NeverInline, neverinline ) \
Entry( Ptr, * ) \
Entry( Ref, & ) \
Entry( Register, register ) \
Entry( RValue, && ) \
Entry( Static, static ) \
Entry( Thread_Local, thread_local ) \
Entry( Volatile, volatile ) \
Entry( Virtual, virtual ) \
Entry( Const, const ) \
Entry( Final, final ) \
Entry( Override, override )
enum Type : u32
{
# define Entry( Specifier, Code ) Specifier,
Define_Specifiers
# undef Entry
NumSpecifiers,
};
inline
bool is_trailing( Type specifier )
{
return specifier > Virtual;
}
// Specifier to string
inline
StrC to_str( Type specifier )
{
local_persist
StrC lookup[ NumSpecifiers ] = {
# pragma push_macro( "global" )
# pragma push_macro( "internal" )
# pragma push_macro( "local_persist" )
# pragma push_macro( "neverinline" )
# undef global
# undef internal
# undef local_persist
# undef neverinline
# define Entry( Spec_, Code_ ) { sizeof(stringize(Code_)), stringize(Code_) },
Define_Specifiers
# undef Entry
# pragma pop_macro( "global" )
# pragma pop_macro( "internal" )
# pragma pop_macro( "local_persist" )
# pragma pop_macro( "neverinline" )
};
return lookup[ specifier ];
}
inline
Type to_type( StrC str )
{
local_persist
u32 keymap[ NumSpecifiers ];
do_once_start
for ( u32 index = 0; index < NumSpecifiers; index++ )
{
StrC enum_str = to_str( (Type)index );
// We subtract 1 to remove the null terminator
// This is because the tokens lexed are not null terminated.
keymap[index] = crc32( enum_str.Ptr, enum_str.Len - 1);
}
do_once_end
u32 hash = crc32( str.Ptr, str.Len );
for ( u32 index = 0; index < NumSpecifiers; index++ )
{
if ( keymap[index] == hash )
return (Type)index;
}
return Invalid;
}
# undef Define_Specifiers
}
using SpecifierT = ESpecifier::Type;

View File

@@ -0,0 +1,165 @@
namespace Parser
{
/*
This is a simple lexer that focuses on tokenizing only tokens relevant to the library.
It will not be capable of lexing C++ code with unsupported features.
For the sake of scanning files, it can scan preprocessor directives
__Attributes_Start is only used to indicate the start of the user_defined attribute list.
*/
#ifndef GEN_DEFINE_ATTRIBUTE_TOKENS
# define GEN_DEFINE_ATTRIBUTE_TOKENS \
Entry( API_Export, "GEN_API_Export_Code" ) \
Entry( API_Import, "GEN_API_Import_Code" )
#endif
# define Define_TokType \
Entry( Invalid, "INVALID" ) \
Entry( Access_Private, "private" ) \
Entry( Access_Protected, "protected" ) \
Entry( Access_Public, "public" ) \
Entry( Access_MemberSymbol, "." ) \
Entry( Access_StaticSymbol, "::") \
Entry( Ampersand, "&" ) \
Entry( Ampersand_DBL, "&&" ) \
Entry( Assign_Classifer, ":" ) \
Entry( Attribute_Open, "[[" ) \
Entry( Attribute_Close, "]]" ) \
Entry( BraceCurly_Open, "{" ) \
Entry( BraceCurly_Close, "}" ) \
Entry( BraceSquare_Open, "[" ) \
Entry( BraceSquare_Close, "]" ) \
Entry( Capture_Start, "(" ) \
Entry( Capture_End, ")" ) \
Entry( Comment, "__comment__" ) \
Entry( Char, "__character__" ) \
Entry( Comma, "," ) \
Entry( Decl_Class, "class" ) \
Entry( Decl_GNU_Attribute, "__attribute__" ) \
Entry( Decl_MSVC_Attribute, "__declspec" ) \
Entry( Decl_Enum, "enum" ) \
Entry( Decl_Extern_Linkage, "extern" ) \
Entry( Decl_Friend, "friend" ) \
Entry( Decl_Module, "module" ) \
Entry( Decl_Namespace, "namespace" ) \
Entry( Decl_Operator, "operator" ) \
Entry( Decl_Struct, "struct" ) \
Entry( Decl_Template, "template" ) \
Entry( Decl_Typedef, "typedef" ) \
Entry( Decl_Using, "using" ) \
Entry( Decl_Union, "union" ) \
Entry( Identifier, "__identifier__" ) \
Entry( Module_Import, "import" ) \
Entry( Module_Export, "export" ) \
Entry( Number, "__number__" ) \
Entry( Operator, "__operator__" ) \
Entry( Preprocess_Define, "define") \
Entry( Preprocess_If, "if") \
Entry( Preprocess_IfDef, "ifdef") \
Entry( Preprocess_IfNotDef, "ifndef") \
Entry( Preprocess_ElIf, "elif") \
Entry( Preprocess_Else, "else") \
Entry( Preprocess_EndIf, "endif") \
Entry( Preprocess_Include, "include" ) \
Entry( Preprocess_Pragma, "pragma") \
Entry( Preprocess_Content, "__macro_content__") \
Entry( Preprocess_Macro, "__macro__") \
Entry( Preprocess_Unsupported, "__unsupported__" ) \
Entry( Spec_Alignas, "alignas" ) \
Entry( Spec_Const, "const" ) \
Entry( Spec_Consteval, "consteval" ) \
Entry( Spec_Constexpr, "constexpr" ) \
Entry( Spec_Constinit, "constinit" ) \
Entry( Spec_Explicit, "explicit" ) \
Entry( Spec_Extern, "extern" ) \
Entry( Spec_Final, "final" ) \
Entry( Spec_Global, "global" ) \
Entry( Spec_Inline, "inline" ) \
Entry( Spec_Internal_Linkage, "internal" ) \
Entry( Spec_LocalPersist, "local_persist" ) \
Entry( Spec_Mutable, "mutable" ) \
Entry( Spec_NeverInline, "neverinline" ) \
Entry( Spec_Override, "override" ) \
Entry( Spec_Static, "static" ) \
Entry( Spec_ThreadLocal, "thread_local" ) \
Entry( Spec_Volatile, "volatile") \
Entry( Star, "*" ) \
Entry( Statement_End, ";" ) \
Entry( StaticAssert, "static_assert" ) \
Entry( String, "__string__" ) \
Entry( Type_Unsigned, "unsigned" ) \
Entry( Type_Signed, "signed" ) \
Entry( Type_Short, "short" ) \
Entry( Type_Long, "long" ) \
Entry( Type_char, "char" ) \
Entry( Type_int, "int" ) \
Entry( Type_double, "double" ) \
Entry( Type_MS_int8, "__int8" ) \
Entry( Type_MS_int16, "__int16" ) \
Entry( Type_MS_int32, "__int32" ) \
Entry( Type_MS_int64, "__int64" ) \
Entry( Type_MS_W64, "_W64" ) \
Entry( Varadic_Argument, "..." ) \
Entry( __Attributes_Start, "__attrib_start__" )
namespace ETokType
{
enum Type : u32
{
# define Entry( Name_, Str_ ) Name_,
Define_TokType
GEN_DEFINE_ATTRIBUTE_TOKENS
# undef Entry
NumTokens,
};
internal inline
Type to_type( StrC str_tok )
{
local_persist
StrC lookup[(u32)NumTokens] =
{
# define Entry( Name_, Str_ ) { sizeof(Str_), Str_ },
Define_TokType
GEN_DEFINE_ATTRIBUTE_TOKENS
# undef Entry
};
for ( u32 index = 0; index < (u32)NumTokens; index++ )
{
s32 lookup_len = lookup[index].Len - 1;
char const* lookup_str = lookup[index].Ptr;
if ( lookup_len != str_tok.Len )
continue;
if ( str_compare( str_tok.Ptr, lookup_str, lookup_len ) == 0 )
return scast(Type, index);
}
return Invalid;
}
internal inline
char const* to_str( Type type )
{
local_persist
char const* lookup[(u32)NumTokens] =
{
# define Entry( Name_, Str_ ) Str_,
Define_TokType
GEN_DEFINE_ATTRIBUTE_TOKENS
# undef Entry
};
return lookup[(u32)type];
}
# undef Define_TokType
};
using TokType = ETokType::Type;
} // Parser