ed/gencpp
1
0
Fork 0
mirror of https://github.com/Ed94/gencpp.git synced 2025-11-03 15:26:12 -08:00
Code Issues Packages Projects Releases Wiki Activity

lexer/parser revamp progress: got c11 ver compiling (synchup before continuation)

Browse Source
This commit is contained in:
Edward R. Gonzalez
2025-03-19 12:09:07 -04:00
parent ad5cb6597b
commit 84f4fc5ae9
6 changed files with 70 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
base/components/interface.hpp
View File

@@ -108,7 +108,7 @@ inline
void logger_fmt(Context* ctx, LogLevel level, char const* fmt, ...)
{
local_persist thread_local
PrintF_Buffer buf = struct_zero(PrintF_Buffer);
PrintF_Buffer buf = struct_zero_init();
va_list va;
va_start(va, fmt);

42
base/components/interface.parsing.cpp
View File

@@ -22,7 +22,7 @@ ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts)
// TODO(Ed): ParseInfo should be set to the parser context.
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
ctx->parser.tokens = lexed.tokens;
ParseStackNode scope = NullScope;
@@ -41,7 +41,7 @@ CodeClass parse_class( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -62,7 +62,7 @@ CodeConstructor parse_constructor(Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -129,7 +129,7 @@ CodeDefine parse_define( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -150,7 +150,7 @@ CodeDestructor parse_destructor( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -171,7 +171,7 @@ CodeEnum parse_enum( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -189,7 +189,7 @@ CodeBody parse_export_body( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -206,7 +206,7 @@ CodeExtern parse_extern_link( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -223,7 +223,7 @@ CodeFriend parse_friend( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -240,7 +240,7 @@ CodeFn parse_function( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -257,7 +257,7 @@ CodeBody parse_global_body( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -278,7 +278,7 @@ CodeNS parse_namespace( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -295,7 +295,7 @@ CodeOperator parse_operator( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -312,7 +312,7 @@ CodeOpCast parse_operator_cast( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -329,7 +329,7 @@ CodeStruct parse_struct( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -350,7 +350,7 @@ CodeTemplate parse_template( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -367,7 +367,7 @@ CodeTypename parse_type( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -384,7 +384,7 @@ CodeTypedef parse_typedef( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -401,7 +401,7 @@ CodeUnion parse_union( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -418,7 +418,7 @@ CodeUsing parse_using( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
@@ -435,7 +435,7 @@ CodeVar parse_variable( Str def )
check_parse_args( def );
ctx->parser = struct_zero();
ctx->parser = struct_zero(ParseContext);
LexedInfo lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;

30
base/components/lexer.cpp
View File

@@ -23,10 +23,10 @@ Token* lex_current(ParseContext* self, bool skip_formatting )
{
if ( skip_formatting )
{
while ( self->tokens[self->token_id].Type == Tok_NewLine || self->tokens[self->token_id].Type == Tok_Comment )
while ( self->tokens.ptr[self->token_id].Type == Tok_NewLine || self->tokens.ptr[self->token_id].Type == Tok_Comment )
self->token_id++;
}
return & self->tokens[self->token_id];
return & self->tokens.ptr[self->token_id];
}
Token* lex_peek(ParseContext const* self, bool skip_formatting)
@@ -34,12 +34,12 @@ Token* lex_peek(ParseContext const* self, bool skip_formatting)
s32 idx = self->token_id;
if ( skip_formatting )
{
while ( self->tokens[idx].Type == Tok_NewLine )
while ( self->tokens.ptr[idx].Type == Tok_NewLine )
idx++;
return & self->tokens[idx];
return & self->tokens.ptr[idx];
}
return & self->tokens[idx];
return & self->tokens.ptr[idx];
}
Token* lex_previous(ParseContext const* self, bool skip_formatting)
@@ -47,12 +47,12 @@ Token* lex_previous(ParseContext const* self, bool skip_formatting)
s32 idx = self->token_id;
if ( skip_formatting )
{
while ( self->tokens[idx].Type == Tok_NewLine )
while ( self->tokens.ptr[idx].Type == Tok_NewLine )
idx --;
return & self->tokens[idx];
return & self->tokens.ptr[idx];
}
return & self->tokens[idx - 1];
return & self->tokens.ptr[idx - 1];
}
Token* lex_next(ParseContext const* self, bool skip_formatting)
@@ -60,12 +60,12 @@ Token* lex_next(ParseContext const* self, bool skip_formatting)
s32 idx = self->token_id;
if ( skip_formatting )
{
while ( self->tokens[idx].Type == Tok_NewLine )
while ( self->tokens.ptr[idx].Type == Tok_NewLine )
idx++;
return & self->tokens[idx + 1];
return & self->tokens.ptr[idx + 1];
}
return & self->tokens[idx + 1];
return & self->tokens.ptr[idx + 1];
}
enum
@@ -137,7 +137,7 @@ s32 lex_preprocessor_define( LexContext* ctx )
);
// GEN_DEBUG_TRAP();
}
array_append( ctx->tokens, name );
array_append(ctx->tokens, name);
if ( ctx->left && (* ctx->scanner) == '(' )
{
@@ -152,7 +152,7 @@ s32 lex_preprocessor_define( LexContext* ctx )
}
Token opening_paren = { { ctx->scanner, 1 }, Tok_Paren_Open, ctx->line, ctx->column, TF_Preprocess };
array_append( ctx->tokens, opening_paren );
array_append(ctx->tokens, opening_paren);
move_forward();
Token last_parameter = {};
@@ -571,9 +571,9 @@ void lex_found_token( LexContext* ctx )
neverinline
LexedInfo lex(Context* lib_ctx, Str content)
{
LexedInfo info = struct_zero();
LexedInfo info = struct_zero(LexedInfo);
LexContext c = struct_zero(); LexContext* ctx = & c;
LexContext c = struct_zero(LexContext); LexContext* ctx = & c;
c.content = content;
c.left = content.Len;
c.scanner = content.Ptr;

49
base/components/parser.cpp
View File

@@ -34,11 +34,11 @@ StrBuilder parser_to_strbuilder(ParseContext const* ctx, AllocatorInfo temp)
StrBuilder result = strbuilder_make_reserve( temp, kilobytes(4) );
Token scope_start = * ctx->Scope->Start;
Token last_valid = (ctx->token_id >= ctx->tokens.num) ? ctx->tokens[ctx->tokens.num -1] : (* lex_peek(ctx, true));
Token last_valid = (ctx->token_id >= ctx->tokens.num) ? ctx->tokens.ptr[ctx->tokens.num -1] : (* lex_peek(ctx, true));
sptr length = scope_start.Text.Len;
char const* current = scope_start.Text.Ptr + length;
while ( current <= ctx->tokens[ctx->tokens.num - 1].Text.Ptr && (* current) != '\n' && length < 74 )
while ( current <= ctx->tokens.ptr[ctx->tokens.num - 1].Text.Ptr && (* current) != '\n' && length < 74 )
{
current++;
length++;
@@ -85,7 +85,7 @@ bool lex__eat(Context* ctx, ParseContext* parser, TokType type)
return false;
}
Token at_idx = parser->tokens[ parser->token_id ];
Token at_idx = parser->tokens.ptr[ parser->token_id ];
if ( ( at_idx.Type == Tok_NewLine && type != Tok_NewLine )
|| ( at_idx.Type == Tok_Comment && type != Tok_Comment ) )
@@ -126,14 +126,11 @@ bool lex__eat(Context* ctx, ParseContext* parser, TokType type)
internal
void parser_init(Context* ctx)
{
// ctx->Lexer_Tokens = array_init_reserve(Token, ctx->Allocator_DyanmicContainers, ctx->InitSize_LexerTokens );
}
internal
void parser_deinit(Context* ctx)
{
// Array(Token) null_array = { nullptr };
// ctx->Lexer_Tokens = null_array;
}
#pragma region Helper Macros
@@ -183,7 +180,7 @@ bool _check_parse_args(ParseContext* parser, Str def, char const* func_name )
#if GEN_COMPILER_CPP
# define NullScope { nullptr, {nullptr, 0}, lex_current( & ctx->parser, lex_dont_skip_formatting ), Str{nullptr, 0}, txt( __func__ ), { nullptr} }
#else
# define NullScope (ParseStackNode){ nullptr, {nullptr, 0}, lex_current( & ctx->parser.Tokens, lex_dont_skip_formatting ), (Str){nullptr, 0}, txt( __func__ ), { nullptr} }
# define NullScope (ParseStackNode){ nullptr, {nullptr, 0}, lex_current( & ctx->parser, lex_dont_skip_formatting ), (Str){nullptr, 0}, txt( __func__ ), { nullptr} }
#endif
#pragma endregion Helper Macros
@@ -1186,17 +1183,17 @@ Code parse_complicated_definition(Context* ctx, TokType which)
b32 had_paren = false;
for ( ; idx < tokens.num; idx++ )
{
if ( tokens[ idx ].Type == Tok_BraceCurly_Open )
if ( tokens.ptr[ idx ].Type == Tok_BraceCurly_Open )
level++;
if ( tokens[ idx ].Type == Tok_BraceCurly_Close ) {
if ( tokens.ptr[ idx ].Type == Tok_BraceCurly_Close ) {
level--;
had_def = level == 0;
}
b32 found_fn_def = had_def && had_paren;
if ( level == 0 && (tokens[ idx ].Type == Tok_Statement_End || found_fn_def) )
if ( level == 0 && (tokens.ptr[ idx ].Type == Tok_Statement_End || found_fn_def) )
break;
}
@@ -1219,23 +1216,23 @@ Code parse_complicated_definition(Context* ctx, TokType which)
return result;
}
Token tok = tokens[ idx - 1 ];
Token tok = tokens.ptr[ idx - 1 ];
if ( tok_is_specifier(tok) && spec_is_trailing( str_to_specifier( tok.Text)) )
{
// <which> <type_identifier>(...) <specifier> ...;
s32 spec_idx = idx - 1;
Token spec = tokens[spec_idx];
Token spec = tokens.ptr[spec_idx];
while ( tok_is_specifier(spec) && spec_is_trailing( str_to_specifier( spec.Text)) )
{
-- spec_idx;
spec = tokens[spec_idx];
spec = tokens.ptr[spec_idx];
}
if ( tokens[spec_idx].Type == Tok_Paren_Close )
if ( tokens.ptr[spec_idx].Type == Tok_Paren_Close )
{
// Forward declaration with trailing specifiers for a procedure
tok = tokens[spec_idx];
tok = tokens.ptr[spec_idx];
Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode);
// <Attributes> <Specifiers> <ReturnType/ValueType> <operator <Op>, or Name> ...
@@ -1249,7 +1246,7 @@ Code parse_complicated_definition(Context* ctx, TokType which)
}
if ( tok.Type == Tok_Identifier )
{
tok = tokens[ idx - 2 ];
tok = tokens.ptr[ idx - 2 ];
bool is_indirection = tok.Type == Tok_Ampersand || tok.Type == Tok_Star;
bool ok_to_parse = false;
@@ -1270,15 +1267,15 @@ Code parse_complicated_definition(Context* ctx, TokType which)
parser_pop(& ctx->parser);
return (Code) result;
}
else if ( tok.Type == Tok_Identifier && tokens[ idx - 3 ].Type == which )
else if ( tok.Type == Tok_Identifier && tokens.ptr[ idx - 3 ].Type == which )
{
// Its a variable with type ID using <which> namespace.
// <which> <type_identifier> <identifier>;
ok_to_parse = true;
}
else if ( tok.Type == Tok_Assign_Classifer
&& ( ( tokens[idx - 5].Type == which && tokens[idx - 4].Type == Tok_Decl_Class )
|| ( tokens[idx - 4].Type == which))
&& ( ( tokens.ptr[idx - 5].Type == which && tokens.ptr[idx - 4].Type == Tok_Decl_Class )
|| ( tokens.ptr[idx - 4].Type == which))
)
{
// Its a forward declaration of an enum
@@ -1310,11 +1307,11 @@ Code parse_complicated_definition(Context* ctx, TokType which)
}
else if ( tok.Type >= Tok_Type_Unsigned && tok.Type <= Tok_Type_MS_W64 )
{
tok = tokens[ idx - 2 ];
tok = tokens.ptr[ idx - 2 ];
if ( tok.Type != Tok_Assign_Classifer
|| ( ( tokens[idx - 5].Type != which && tokens[idx - 4].Type != Tok_Decl_Class )
&& ( tokens[idx - 4].Type != which))
|| ( ( tokens.ptr[idx - 5].Type != which && tokens.ptr[idx - 4].Type != Tok_Decl_Class )
&& ( tokens.ptr[idx - 4].Type != which))
)
{
log_failure( "Unsupported or bad member definition after %s declaration\n%s", toktype_to_str(which), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) );
@@ -1905,12 +1902,12 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
for ( ; idx < ctx->parser.tokens.num; idx++ )
{
Token tok = ctx->parser.tokens[ idx ];
Token tok = ctx->parser.tokens.ptr[ idx ];
if ( tok.Type == Tok_Identifier )
{
idx++;
tok = ctx->parser.tokens[ idx ];
tok = ctx->parser.tokens.ptr[ idx ];
if ( tok.Type == Tok_Access_StaticSymbol )
continue;
@@ -1978,8 +1975,8 @@ Code parse_global_nspace_constructor_destructor(Context* ctx, CodeSpecifiers spe
TokenSlice tokens = ctx->parser.tokens;
s32 idx = ctx->parser.token_id;
Token nav = tokens[ idx ];
for ( ; idx < tokens.num; idx++, nav = tokens[ idx ] )
Token nav = tokens.ptr[ idx ];
for ( ; idx < tokens.num; idx++, nav = tokens.ptr[ idx ] )
{
if ( nav.Text.Ptr[0] == '<' )
{

10
base/dependencies/macros.hpp
View File

@@ -310,7 +310,15 @@
# if GEN_COMPILER_CPP
# define struct_zero(type) {}
# else
# define struct_zero(type) {0}
# define struct_zero(type) (type) {0}
# endif
#endif
#ifndef struct_zero_init
# if GEN_COMPILER_CPP
# define struct_zero_init() {}
# else
# define struct_zero_init() {0}
# endif
#endif

2
base/dependencies/strings.hpp
View File

@@ -320,7 +320,7 @@ inline
StrBuilder strbuilder_fmt_buf(AllocatorInfo allocator, char const* fmt, ...)
{
local_persist thread_local
PrintF_Buffer buf = struct_zero(PrintF_Buffer);
PrintF_Buffer buf = struct_zero_init();
va_list va;
va_start(va, fmt);