ed/gencpp
1
0
Fork 0
mirror of https://github.com/Ed94/gencpp.git synced 2024-12-22 07:44:45 -08:00
Code Issues Packages Projects Releases Wiki Activity

WIP: prepped for dealing with the compiler...

Browse Source
This commit is contained in:
Edward R. Gonzalez 2024-12-14 20:33:14 -05:00
parent c8cf55403b
commit 1b4f9a2e77
5 changed files with 332 additions and 401 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
base/components/lexer.cpp
View File

@ -427,30 +427,24 @@ s32 lex_preprocessor_directive( LexContext* ctx )
forceinline
void lex_found_token( LexContext* ctx )
{
if ( ctx->token.Type != Tok_Invalid )
{
if ( ctx->token.Type != Tok_Invalid ) {
array_append( _ctx->Lexer_Tokens, ctx->token );
return;
}
TokType type = str_to_toktype( tok_to_str(ctx->token) );
if (type <= Tok_Access_Public && type >= Tok_Access_Private )
{
if (type <= Tok_Access_Public && type >= Tok_Access_Private ) {
ctx->token.Flags |= TF_AccessSpecifier;
}
if ( type > Tok___Attributes_Start )
{
if ( type > Tok___Attributes_Start ) {
ctx->token.Flags |= TF_Attribute;
}
if ( type == Tok_Decl_Extern_Linkage )
{
skip_whitespace();
if ( (* ctx->scanner) != '"' )
{
if ( (* ctx->scanner) != '"' ) {
type = Tok_Spec_Extern;
ctx->token.Flags |= TF_Specifier;
}
@ -459,7 +453,6 @@ void lex_found_token( LexContext* ctx )
array_append( _ctx->Lexer_Tokens, ctx->token );
return;
}
if ( ( type <= Tok_Star && type >= Tok_Spec_Alignas)
|| type == Tok_Ampersand
|| type == Tok_Ampersand_DBL )
@ -469,8 +462,6 @@ void lex_found_token( LexContext* ctx )
array_append( _ctx->Lexer_Tokens, ctx->token );
return;
}
if ( type != Tok_Invalid )
{
ctx->token.Type = type;
@ -478,21 +469,17 @@ void lex_found_token( LexContext* ctx )
return;
}
u64 key = 0;
// if ( (* ctx->scanner) == '(')
// key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len + 1 );
// else
key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len );
Str* define = hashtable_get(_ctx->PreprocessMacros, key );
if ( define )
PreprocessorMacro* macro = lookup_preprocess_macro( ctx->token.Text );
if ( macro )
{
// TODO(Ed): Needs updating (Macros)
ctx->token.Type = Tok_Preprocess_Macro;
ctx->token.Type = macrotype_to_toktype(macro->Type);
// TODO(Ed): When we introduce a macro AST (and expression support), we'll properly lex this section.
// Want to ignore any arguments the define may have as they can be execution expressions.
if ( ctx->left && (* ctx->scanner) == '(' )
{
ctx->token.Flags |= TF_Macro_Functional;
move_forward();
ctx->token.Text.Len++;
@ -512,7 +499,6 @@ void lex_found_token( LexContext* ctx )
move_forward();
ctx->token.Text.Len++;
}
//if ( (* ctx->scanner) == '\r' && ctx->scanner[1] == '\n' )
//{
// move_forward();
@ -1002,7 +988,7 @@ TokArray lex( Str content )
goto FoundToken;
}
// Dash is unfortunatlly a bit more complicated...
// Dash is unfortunately a bit more complicated...
case '-':
{
Str text = { c.scanner, 1 };
@ -1139,8 +1125,7 @@ TokArray lex( Str content )
c.token.Text = text;
move_forward();
while ( c.left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) )
{
while ( c.left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) {
move_forward();
c.token.Text.Len++;
}
@ -1166,8 +1151,7 @@ TokArray lex( Str content )
move_forward();
c.token.Text.Len++;
while ( c.left && char_is_hex_digit((* ctx->scanner)) )
{
while ( c.left && char_is_hex_digit((* ctx->scanner)) ) {
move_forward();
c.token.Text.Len++;
}
@ -1175,8 +1159,7 @@ TokArray lex( Str content )
goto FoundToken;
}
while ( c.left && char_is_digit((* ctx->scanner)) )
{
while ( c.left && char_is_digit((* ctx->scanner)) ) {
move_forward();
c.token.Text.Len++;
}
@ -1186,8 +1169,7 @@ TokArray lex( Str content )
move_forward();
c.token.Text.Len++;
while ( c.left && char_is_digit((* ctx->scanner)) )
{
while ( c.left && char_is_digit((* ctx->scanner)) ) {
move_forward();
c.token.Text.Len++;
}
@ -1205,8 +1187,7 @@ TokArray lex( Str content )
c.token.Text.Len++;
// Handle 'll'/'LL' as a special case when we just processed an 'l'/'L'
if (c.left && (prev == 'l' || prev == 'L') && ((* ctx->scanner) == 'l' || (* ctx->scanner) == 'L'))
{
if (c.left && (prev == 'l' || prev == 'L') && ((* ctx->scanner) == 'l' || (* ctx->scanner) == 'L')) {
move_forward();
c.token.Text.Len++;
}
@ -1232,8 +1213,7 @@ TokArray lex( Str content )
log_failure( "Failed to lex token '%c' (%d, %d)\n%s", (* ctx->scanner), c.line, c.column, context_str );
// Skip to next whitespace since we can't know if anything else is valid until then.
while ( c.left && ! char_is_space( (* ctx->scanner) ) )
{
while ( c.left && ! char_is_space( (* ctx->scanner) ) ) {
move_forward();
}
}
@ -1242,17 +1222,14 @@ TokArray lex( Str content )
{
lex_found_token( ctx );
TokType last_type = array_back(_ctx->Lexer_Tokens)->Type;
// TODO(Ed): Change this to just detect if its a MACRO THAT SHOULD ACCEPT NEWLINES
if ( last_type == Tok_Preprocess_Macro )
if ( last_type == Tok_Preprocess_Macro_Stmt )
{
Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null };
c.token = thanks_c;
if ( (* ctx->scanner) == '\r')
{
if ( (* ctx->scanner) == '\r') {
move_forward();
c.token.Text.Len = 1;
}
if ( (* ctx->scanner) == '\n' )
{
c.token.Type = Tok_NewLine;
@ -1266,13 +1243,10 @@ TokArray lex( Str content )
}
}
if ( array_num(_ctx->Lexer_Tokens) == 0 )
{
if ( array_num(_ctx->Lexer_Tokens) == 0 ) {
log_failure( "Failed to lex any tokens" );
{
TokArray tok_array = {};
return tok_array;
}
TokArray tok_array = {};
return tok_array;
}
TokArray result = { _ctx->Lexer_Tokens, 0 };

514
base/components/parser.cpp
View File

File diff suppressed because it is too large Load Diff

13
base/components/parser_types.hpp
View File

@ -138,7 +138,18 @@ enum MacroType : u16
MF_UnderlyingType = GEN_U16_Max,
};
Str macro_type_to_str( MacroType type )
forceinline
TokenType macrotype_to_toktype( MacroType type ) {
switch ( type ) {
case MT_Statement : return Tok_Preprocess_Macro_Stmt;
case MT_Expression : return Tok_Preprocess_Macro_Expr;
case MT_Typename : return Tok_Preprocess_Macro_Typename;
}
// All others unsupported for now.
return Tok_Invalid;
}
Str macrotype_to_str( MacroType type )
{
local_persist
Str lookup[ (u32)Num_ModuleFlags ] = {

4
base/enums/ETokType.csv
View File

@ -51,7 +51,7 @@ Preprocess_Else, "else"
Preprocess_EndIf, "endif"
Preprocess_Include, "include"
Preprocess_Pragma, "pragma"
Preprocess_Content, "__macro_content__"
Preprocess_Content, "__macro_content__"
Preprocess_Macro_Expr, "__macro_expression__"
Preprocess_Macro_Stmt, "__macro_statment__"
Preprocess_Macro_Typename, "__macro_typename__"
@ -64,7 +64,7 @@ Spec_Constinit, "constinit"
Spec_Explicit, "explicit"
Spec_Extern, "extern"
Spec_Final, "final"
Spec_ForceInline, "forceinline"
Spec_ForceInline, "forceinline"
Spec_Global, "global"
Spec_Inline, "inline"
Spec_Internal_Linkage, "internal"

1 Invalid __invalid__
51 Preprocess_EndIf endif
52 Preprocess_Include include
53 Preprocess_Pragma pragma
54 Preprocess_Content __macro_content__
55 Preprocess_Macro_Expr __macro_expression__
56 Preprocess_Macro_Stmt __macro_statment__
57 Preprocess_Macro_Typename __macro_typename__
64 Spec_Explicit explicit
65 Spec_Extern extern
66 Spec_Final final
67 Spec_ForceInline forceinline
68 Spec_Global global
69 Spec_Inline inline
70 Spec_Internal_Linkage internal

118
base/helpers/base_codegen.hpp
View File

@ -499,35 +499,36 @@ CodeBody gen_ast_inlines()
#pragma pop_macro("GEN_NS")
#pragma pop_macro("CodeInvalid")
CodeBody impl_code = parse_global_body( token_fmt( "typename", Str name(Code), code_impl_tmpl ));
CodeBody impl_code_body = parse_global_body( token_fmt( "typename", Str name(CodeBody), code_impl_tmpl ));
CodeBody impl_code_attr = parse_global_body( token_fmt( "typename", Str name(CodeAttributes), code_impl_tmpl ));
CodeBody impl_code_cmt = parse_global_body( token_fmt( "typename", Str name(CodeComment), code_impl_tmpl ));
CodeBody impl_code_constr = parse_global_body( token_fmt( "typename", Str name(CodeConstructor), code_impl_tmpl ));
CodeBody impl_code_class = parse_global_body( token_fmt( "typename", Str name(CodeClass), code_impl_tmpl ));
CodeBody impl_code_define = parse_global_body( token_fmt( "typename", Str name(CodeDefine), code_impl_tmpl ));
CodeBody impl_code_destruct = parse_global_body( token_fmt( "typename", Str name(CodeDestructor), code_impl_tmpl ));
CodeBody impl_code_enum = parse_global_body( token_fmt( "typename", Str name(CodeEnum), code_impl_tmpl ));
CodeBody impl_code_exec = parse_global_body( token_fmt( "typename", Str name(CodeExec), code_impl_tmpl ));
CodeBody impl_code_extern = parse_global_body( token_fmt( "typename", Str name(CodeExtern), code_impl_tmpl ));
CodeBody impl_code_include = parse_global_body( token_fmt( "typename", Str name(CodeInclude), code_impl_tmpl ));
CodeBody impl_code_friend = parse_global_body( token_fmt( "typename", Str name(CodeFriend), code_impl_tmpl ));
CodeBody impl_code_fn = parse_global_body( token_fmt( "typename", Str name(CodeFn), code_impl_tmpl ));
CodeBody impl_code_module = parse_global_body( token_fmt( "typename", Str name(CodeModule), code_impl_tmpl ));
CodeBody impl_code_ns = parse_global_body( token_fmt( "typename", Str name(CodeNS), code_impl_tmpl ));
CodeBody impl_code_op = parse_global_body( token_fmt( "typename", Str name(CodeOperator), code_impl_tmpl ));
CodeBody impl_code_opcast = parse_global_body( token_fmt( "typename", Str name(CodeOpCast), code_impl_tmpl ));
CodeBody impl_code_params = parse_global_body( token_fmt( "typename", Str name(CodeParams), code_impl_tmpl ));
CodeBody impl_code_pragma = parse_global_body( token_fmt( "typename", Str name(CodePragma), code_impl_tmpl ));
CodeBody impl_code_precond = parse_global_body( token_fmt( "typename", Str name(CodePreprocessCond), code_impl_tmpl ));
CodeBody impl_code_specs = parse_global_body( token_fmt( "typename", Str name(CodeSpecifiers), code_impl_tmpl ));
CodeBody impl_code_struct = parse_global_body( token_fmt( "typename", Str name(CodeStruct), code_impl_tmpl ));
CodeBody impl_code_tmpl = parse_global_body( token_fmt( "typename", Str name(CodeTemplate), code_impl_tmpl ));
CodeBody impl_code_type = parse_global_body( token_fmt( "typename", Str name(CodeTypename), code_impl_tmpl ));
CodeBody impl_code_typedef = parse_global_body( token_fmt( "typename", Str name(CodeTypedef), code_impl_tmpl ));
CodeBody impl_code_union = parse_global_body( token_fmt( "typename", Str name(CodeUnion), code_impl_tmpl ));
CodeBody impl_code_using = parse_global_body( token_fmt( "typename", Str name(CodeUsing), code_impl_tmpl ));
CodeBody impl_code_var = parse_global_body( token_fmt( "typename", Str name(CodeVar), code_impl_tmpl ));
CodeBody impl_code = parse_global_body( token_fmt( "typename", Str name(Code), code_impl_tmpl ));
CodeBody impl_code_body = parse_global_body( token_fmt( "typename", Str name(CodeBody), code_impl_tmpl ));
CodeBody impl_code_attr = parse_global_body( token_fmt( "typename", Str name(CodeAttributes), code_impl_tmpl ));
CodeBody impl_code_cmt = parse_global_body( token_fmt( "typename", Str name(CodeComment), code_impl_tmpl ));
CodeBody impl_code_constr = parse_global_body( token_fmt( "typename", Str name(CodeConstructor), code_impl_tmpl ));
CodeBody impl_code_class = parse_global_body( token_fmt( "typename", Str name(CodeClass), code_impl_tmpl ));
CodeBody impl_code_define = parse_global_body( token_fmt( "typename", Str name(CodeDefine), code_impl_tmpl ));
CodeBody impl_code_define_params = parse_global_body( token_fmt( "typename", Str name(CodeDefineParams), code_impl_tmpl ));
CodeBody impl_code_destruct = parse_global_body( token_fmt( "typename", Str name(CodeDestructor), code_impl_tmpl ));
CodeBody impl_code_enum = parse_global_body( token_fmt( "typename", Str name(CodeEnum), code_impl_tmpl ));
CodeBody impl_code_exec = parse_global_body( token_fmt( "typename", Str name(CodeExec), code_impl_tmpl ));
CodeBody impl_code_extern = parse_global_body( token_fmt( "typename", Str name(CodeExtern), code_impl_tmpl ));
CodeBody impl_code_include = parse_global_body( token_fmt( "typename", Str name(CodeInclude), code_impl_tmpl ));
CodeBody impl_code_friend = parse_global_body( token_fmt( "typename", Str name(CodeFriend), code_impl_tmpl ));
CodeBody impl_code_fn = parse_global_body( token_fmt( "typename", Str name(CodeFn), code_impl_tmpl ));
CodeBody impl_code_module = parse_global_body( token_fmt( "typename", Str name(CodeModule), code_impl_tmpl ));
CodeBody impl_code_ns = parse_global_body( token_fmt( "typename", Str name(CodeNS), code_impl_tmpl ));
CodeBody impl_code_op = parse_global_body( token_fmt( "typename", Str name(CodeOperator), code_impl_tmpl ));
CodeBody impl_code_opcast = parse_global_body( token_fmt( "typename", Str name(CodeOpCast), code_impl_tmpl ));
CodeBody impl_code_params = parse_global_body( token_fmt( "typename", Str name(CodeParams), code_impl_tmpl ));
CodeBody impl_code_pragma = parse_global_body( token_fmt( "typename", Str name(CodePragma), code_impl_tmpl ));
CodeBody impl_code_precond = parse_global_body( token_fmt( "typename", Str name(CodePreprocessCond), code_impl_tmpl ));
CodeBody impl_code_specs = parse_global_body( token_fmt( "typename", Str name(CodeSpecifiers), code_impl_tmpl ));
CodeBody impl_code_struct = parse_global_body( token_fmt( "typename", Str name(CodeStruct), code_impl_tmpl ));
CodeBody impl_code_tmpl = parse_global_body( token_fmt( "typename", Str name(CodeTemplate), code_impl_tmpl ));
CodeBody impl_code_type = parse_global_body( token_fmt( "typename", Str name(CodeTypename), code_impl_tmpl ));
CodeBody impl_code_typedef = parse_global_body( token_fmt( "typename", Str name(CodeTypedef), code_impl_tmpl ));
CodeBody impl_code_union = parse_global_body( token_fmt( "typename", Str name(CodeUnion), code_impl_tmpl ));
CodeBody impl_code_using = parse_global_body( token_fmt( "typename", Str name(CodeUsing), code_impl_tmpl ));
CodeBody impl_code_var = parse_global_body( token_fmt( "typename", Str name(CodeVar), code_impl_tmpl ));
body_append(impl_code_attr, parse_global_body( token_fmt( "typename", Str name(Attributes), codetype_impl_tmpl )));
body_append(impl_code_cmt, parse_global_body( token_fmt( "typename", Str name(Comment), codetype_impl_tmpl )));
@ -563,34 +564,35 @@ CodeBody gen_ast_inlines()
);
#pragma pop_macro("forceinline")
CodeBody impl_cast_body = parse_global_body( token_fmt( "typename", Str name(Body), cast_tmpl ));
CodeBody impl_cast_attribute = parse_global_body( token_fmt( "typename", Str name(Attributes), cast_tmpl ));
CodeBody impl_cast_cmt = parse_global_body( token_fmt( "typename", Str name(Comment), cast_tmpl ));
CodeBody impl_cast_constr = parse_global_body( token_fmt( "typename", Str name(Constructor), cast_tmpl ));
CodeBody impl_cast_class = parse_global_body( token_fmt( "typename", Str name(Class), cast_tmpl ));
CodeBody impl_cast_define = parse_global_body( token_fmt( "typename", Str name(Define), cast_tmpl ));
CodeBody impl_cast_destruct = parse_global_body( token_fmt( "typename", Str name(Destructor), cast_tmpl ));
CodeBody impl_cast_enum = parse_global_body( token_fmt( "typename", Str name(Enum), cast_tmpl ));
CodeBody impl_cast_exec = parse_global_body( token_fmt( "typename", Str name(Exec), cast_tmpl ));
CodeBody impl_cast_extern = parse_global_body( token_fmt( "typename", Str name(Extern), cast_tmpl ));
CodeBody impl_cast_friend = parse_global_body( token_fmt( "typename", Str name(Friend), cast_tmpl ));
CodeBody impl_cast_fn = parse_global_body( token_fmt( "typename", Str name(Fn), cast_tmpl ));
CodeBody impl_cast_include = parse_global_body( token_fmt( "typename", Str name(Include), cast_tmpl ));
CodeBody impl_cast_module = parse_global_body( token_fmt( "typename", Str name(Module), cast_tmpl ));
CodeBody impl_cast_ns = parse_global_body( token_fmt( "typename", Str name(NS), cast_tmpl ));
CodeBody impl_cast_op = parse_global_body( token_fmt( "typename", Str name(Operator), cast_tmpl ));
CodeBody impl_cast_opcast = parse_global_body( token_fmt( "typename", Str name(OpCast), cast_tmpl ));
CodeBody impl_cast_params = parse_global_body( token_fmt( "typename", Str name(Params), cast_tmpl ));
CodeBody impl_cast_pragma = parse_global_body( token_fmt( "typename", Str name(Pragma), cast_tmpl ));
CodeBody impl_cast_precond = parse_global_body( token_fmt( "typename", Str name(PreprocessCond), cast_tmpl ));
CodeBody impl_cast_specs = parse_global_body( token_fmt( "typename", Str name(Specifiers), cast_tmpl ));
CodeBody impl_cast_struct = parse_global_body( token_fmt( "typename", Str name(Struct), cast_tmpl ));
CodeBody impl_cast_tmpl = parse_global_body( token_fmt( "typename", Str name(Template), cast_tmpl ));
CodeBody impl_cast_type = parse_global_body( token_fmt( "typename", Str name(Typename), cast_tmpl ));
CodeBody impl_cast_typedef = parse_global_body( token_fmt( "typename", Str name(Typedef), cast_tmpl ));
CodeBody impl_cast_union = parse_global_body( token_fmt( "typename", Str name(Union), cast_tmpl ));
CodeBody impl_cast_using = parse_global_body( token_fmt( "typename", Str name(Using), cast_tmpl ));
CodeBody impl_cast_var = parse_global_body( token_fmt( "typename", Str name(Var), cast_tmpl ));
CodeBody impl_cast_body = parse_global_body( token_fmt( "typename", Str name(Body), cast_tmpl ));
CodeBody impl_cast_attribute = parse_global_body( token_fmt( "typename", Str name(Attributes), cast_tmpl ));
CodeBody impl_cast_cmt = parse_global_body( token_fmt( "typename", Str name(Comment), cast_tmpl ));
CodeBody impl_cast_constr = parse_global_body( token_fmt( "typename", Str name(Constructor), cast_tmpl ));
CodeBody impl_cast_class = parse_global_body( token_fmt( "typename", Str name(Class), cast_tmpl ));
CodeBody impl_cast_define = parse_global_body( token_fmt( "typename", Str name(Define), cast_tmpl ));
CodeBody impl_cast_define_params = parse_global_body( token_fmt( "typename", Str name(DefineParams), cast_tmpl ));
CodeBody impl_cast_destruct = parse_global_body( token_fmt( "typename", Str name(Destructor), cast_tmpl ));
CodeBody impl_cast_enum = parse_global_body( token_fmt( "typename", Str name(Enum), cast_tmpl ));
CodeBody impl_cast_exec = parse_global_body( token_fmt( "typename", Str name(Exec), cast_tmpl ));
CodeBody impl_cast_extern = parse_global_body( token_fmt( "typename", Str name(Extern), cast_tmpl ));
CodeBody impl_cast_friend = parse_global_body( token_fmt( "typename", Str name(Friend), cast_tmpl ));
CodeBody impl_cast_fn = parse_global_body( token_fmt( "typename", Str name(Fn), cast_tmpl ));
CodeBody impl_cast_include = parse_global_body( token_fmt( "typename", Str name(Include), cast_tmpl ));
CodeBody impl_cast_module = parse_global_body( token_fmt( "typename", Str name(Module), cast_tmpl ));
CodeBody impl_cast_ns = parse_global_body( token_fmt( "typename", Str name(NS), cast_tmpl ));
CodeBody impl_cast_op = parse_global_body( token_fmt( "typename", Str name(Operator), cast_tmpl ));
CodeBody impl_cast_opcast = parse_global_body( token_fmt( "typename", Str name(OpCast), cast_tmpl ));
CodeBody impl_cast_params = parse_global_body( token_fmt( "typename", Str name(Params), cast_tmpl ));
CodeBody impl_cast_pragma = parse_global_body( token_fmt( "typename", Str name(Pragma), cast_tmpl ));
CodeBody impl_cast_precond = parse_global_body( token_fmt( "typename", Str name(PreprocessCond), cast_tmpl ));
CodeBody impl_cast_specs = parse_global_body( token_fmt( "typename", Str name(Specifiers), cast_tmpl ));
CodeBody impl_cast_struct = parse_global_body( token_fmt( "typename", Str name(Struct), cast_tmpl ));
CodeBody impl_cast_tmpl = parse_global_body( token_fmt( "typename", Str name(Template), cast_tmpl ));
CodeBody impl_cast_type = parse_global_body( token_fmt( "typename", Str name(Typename), cast_tmpl ));
CodeBody impl_cast_typedef = parse_global_body( token_fmt( "typename", Str name(Typedef), cast_tmpl ));
CodeBody impl_cast_union = parse_global_body( token_fmt( "typename", Str name(Union), cast_tmpl ));
CodeBody impl_cast_using = parse_global_body( token_fmt( "typename", Str name(Using), cast_tmpl ));
CodeBody impl_cast_var = parse_global_body( token_fmt( "typename", Str name(Var), cast_tmpl ));
CodeBody result = def_global_body( args(
def_pragma( txt("region generated code inline implementation")),
@ -602,6 +604,7 @@ CodeBody gen_ast_inlines()
impl_code_constr,
impl_code_class,
impl_code_define,
impl_code_define_params,
impl_code_destruct,
impl_code_enum,
impl_code_exec,
@ -636,6 +639,7 @@ CodeBody gen_ast_inlines()
impl_cast_constr,
impl_cast_class,
impl_cast_define,
impl_cast_define_params,
impl_cast_destruct,
impl_cast_enum,
impl_cast_exec,