Got past parsing, fixing serialization

This commit is contained in:
Edward R. Gonzalez 2023-08-01 14:02:54 -04:00
parent 528ef72a51
commit 0f16d1131e
6 changed files with 252 additions and 80 deletions

View File

@ -508,39 +508,39 @@ String AST::to_string()
break; break;
case Preprocess_Define: case Preprocess_Define:
result.append_fmt( "#define %s %s", Name, Content ); result.append_fmt( "#define %s %s\n", Name, Content );
break; break;
case Preprocess_If: case Preprocess_If:
result.append_fmt( "#if %s", Content ); result.append_fmt( "#if %s\n", Content );
break; break;
case Preprocess_IfDef: case Preprocess_IfDef:
result.append_fmt( "#ifdef %s", Content ); result.append_fmt( "#ifdef %s\n", Content );
break; break;
case Preprocess_IfNotDef: case Preprocess_IfNotDef:
result.append_fmt( "#ifndef %s", Content ); result.append_fmt( "#ifndef %s\n", Content );
break; break;
case Preprocess_Include: case Preprocess_Include:
result.append_fmt( "#include \"%s\"", Content ); result.append_fmt( "#include \"%s\"\n", Content );
break; break;
case Preprocess_ElIf: case Preprocess_ElIf:
result.append_fmt( "#elif %s", Content ); result.append_fmt( "#elif %s\n", Content );
break; break;
case Preprocess_Else: case Preprocess_Else:
result.append_fmt( "#else" ); result.append_fmt( "#else\n" );
break; break;
case Preprocess_EndIf: case Preprocess_EndIf:
result.append_fmt( "#endif" ); result.append_fmt( "#endif\n" );
break; break;
case Preprocess_Pragma: case Preprocess_Pragma:
result.append_fmt( "#pragma %s", Content ); result.append_fmt( "#pragma %s\n", Content );
break; break;
case Specifiers: case Specifiers:
@ -653,11 +653,11 @@ String AST::to_string()
if ( UnderlyingType->Type == Typename && UnderlyingType->ArrExpr ) if ( UnderlyingType->Type == Typename && UnderlyingType->ArrExpr )
{ {
result.append_fmt( "[%s];", UnderlyingType->ArrExpr->to_string() ); result.append_fmt( "[%s];\n", UnderlyingType->ArrExpr->to_string() );
} }
else else
{ {
result.append( ";" ); result.append( ";\n" );
} }
} }
break; break;

View File

@ -26,12 +26,29 @@ void AST::append( AST* other )
char const* AST::debug_str() char const* AST::debug_str()
{ {
if ( Parent )
{
char const* fmt = stringize(
\nType : %s
\nParent : %s %s
\nName : %s
);
// These should be used immediately in a log.
// Thus if its desired to keep the debug str
// for multiple calls to bprintf,
// allocate this to proper string.
return str_fmt_buf( fmt
, type_str()
, Parent->Name
, Parent->type_str()
, Name ? Name : ""
);
}
char const* fmt = stringize( char const* fmt = stringize(
\nCode Debug:
\nType : %s \nType : %s
\nParent : %s
\nName : %s \nName : %s
\nComment : %s
); );
// These should be used immediately in a log. // These should be used immediately in a log.
@ -40,7 +57,6 @@ char const* AST::debug_str()
// allocate this to proper string. // allocate this to proper string.
return str_fmt_buf( fmt return str_fmt_buf( fmt
, type_str() , type_str()
, Parent ? Parent->Name : ""
, Name ? Name : "" , Name ? Name : ""
); );
} }

View File

@ -1,6 +1,3 @@
/*
These constructors are the most implementation intensive other than the editor or scanner.
*/
namespace Parser namespace Parser
{ {
@ -331,6 +328,12 @@ namespace Parser
continue; // Skip found token, its all handled here. continue; // Skip found token, its all handled here.
} }
if ( token.Type == TokType::Preprocess_Else || token.Type == TokType::Preprocess_EndIf )
{
Tokens.append( token );
continue;
}
Tokens.append( token ); Tokens.append( token );
SkipWhitespace(); SkipWhitespace();
@ -355,12 +358,6 @@ namespace Parser
defines.set( key, name ); defines.set( key, name );
} }
if ( token.Type == TokType::Preprocess_Else || token.Type == TokType::Preprocess_EndIf )
{
Tokens.append( token );
continue;
}
Token content = { scanner, 0, TokType::Preprocess_Content, false, line, column }; Token content = { scanner, 0, TokType::Preprocess_Content, false, line, column };
if ( token.Type == TokType::Preprocess_Include ) if ( token.Type == TokType::Preprocess_Include )
@ -1165,8 +1162,10 @@ Code parse_static_assert()
Code Code
assert = make_code(); assert = make_code();
assert->Type = ECode::Untyped; assert->Type = ECode::Untyped;
eat( TokType::StaticAssert );
Token content = currtok;
eat( TokType::StaticAssert );
eat( TokType::Capture_Start ); eat( TokType::Capture_Start );
s32 level = 0; s32 level = 0;
@ -1182,6 +1181,14 @@ Code parse_static_assert()
eat( TokType::Capture_End ); eat( TokType::Capture_End );
eat( TokType::Statement_End ); eat( TokType::Statement_End );
content.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)content.Text;
content.Text = str_fmt_buf( "%.*s\n", content.Length, content.Text );
content.Length++;
assert->Content = get_cached_string( content );
assert->Name = assert->Content;
Context.pop(); Context.pop();
return assert; return assert;
} }
@ -1682,7 +1689,10 @@ CodeOperator parse_operator_after_ret_type(
// Parse Operator // Parse Operator
eat( TokType::Decl_Operator ); eat( TokType::Decl_Operator );
if ( ! check( TokType::Operator ) ) if ( ! left && currtok.Type != TokType::Operator
&& currtok.Type != TokType::Star
&& currtok.Type != TokType::Ampersand
&& currtok.Type != TokType::Ampersand_DBL )
{ {
log_failure( "Expected operator after 'operator' keyword\n%s", Context.to_string() ); log_failure( "Expected operator after 'operator' keyword\n%s", Context.to_string() );
Context.pop(); Context.pop();
@ -1697,12 +1707,26 @@ CodeOperator parse_operator_after_ret_type(
if ( currtok.Text[1] == '=' ) if ( currtok.Text[1] == '=' )
op = Assign_Add; op = Assign_Add;
if ( currtok.Text[1] == '+' )
op = Increment;
else else
op = Add; op = Add;
} }
break; break;
case '-': case '-':
{ {
if ( currtok.Text[1] == '>' )
{
if ( currtok.Text[2] == '*' )
op = MemberOfPointer;
else
op = MemberOfPointer;
break;
}
if ( currtok.Text[1] == '=' ) if ( currtok.Text[1] == '=' )
op = Assign_Subtract; op = Assign_Subtract;
@ -1876,11 +1900,14 @@ CodeOperator parse_operator_after_ret_type(
return CodeInvalid; return CodeInvalid;
} }
eat( TokType::Operator ); eat( currtok.Type );
// Parse Params // Parse Params
CodeParam params = parse_params(); CodeParam params = parse_params();
if ( params.ast == nullptr && op == EOperator::Multiply )
op = MemberOfPointer;
while ( left && currtok.is_specifier() ) while ( left && currtok.is_specifier() )
{ {
if ( specifiers.ast == nullptr ) if ( specifiers.ast == nullptr )
@ -2273,15 +2300,11 @@ CodeBody parse_class_struct_body( Parser::TokType which )
break; break;
case TokType::Decl_Class: case TokType::Decl_Class:
{
member = parse_complicated_definition( TokType::Decl_Class ); member = parse_complicated_definition( TokType::Decl_Class );
}
break; break;
case TokType::Decl_Enum: case TokType::Decl_Enum:
{
member = parse_complicated_definition( TokType::Decl_Enum ); member = parse_complicated_definition( TokType::Decl_Enum );
}
break; break;
case TokType::Decl_Friend: case TokType::Decl_Friend:
@ -2293,9 +2316,7 @@ CodeBody parse_class_struct_body( Parser::TokType which )
break; break;
case TokType::Decl_Struct: case TokType::Decl_Struct:
{
member = parse_complicated_definition( TokType::Decl_Struct ); member = parse_complicated_definition( TokType::Decl_Struct );
}
break; break;
case TokType::Decl_Template: case TokType::Decl_Template:
@ -2307,10 +2328,7 @@ CodeBody parse_class_struct_body( Parser::TokType which )
break; break;
case TokType::Decl_Union: case TokType::Decl_Union:
{
member = parse_complicated_definition( TokType::Decl_Union ); member = parse_complicated_definition( TokType::Decl_Union );
}
break; break;
case TokType::Decl_Using: case TokType::Decl_Using:
@ -2489,6 +2507,7 @@ Code parse_class_struct( Parser::TokType which, bool inplace_def = false )
attributes = parse_attributes(); attributes = parse_attributes();
if ( check( TokType::Identifier ) )
name = parse_identifier(); name = parse_identifier();
local_persist local_persist
@ -2834,7 +2853,6 @@ CodeEnum parse_enum( bool inplace_def )
Token name = { nullptr, 0, TokType::Invalid }; Token name = { nullptr, 0, TokType::Invalid };
Code array_expr = { nullptr }; Code array_expr = { nullptr };
CodeType type = { nullptr }; CodeType type = { nullptr };
Token body = { nullptr, 0, TokType::Invalid };
char entries_code[ kilobytes(128) ] { 0 }; char entries_code[ kilobytes(128) ] { 0 };
s32 entries_length = 0; s32 entries_length = 0;
@ -2851,15 +2869,11 @@ CodeEnum parse_enum( bool inplace_def )
attributes = parse_attributes(); attributes = parse_attributes();
if ( currtok.Type != TokType::Identifier ) if ( check( TokType::Identifier ) )
{ {
log_failure( "Expected identifier for enum name\n%s", Context.to_string() );
Context.pop();
return CodeInvalid;
}
name = currtok; name = currtok;
eat( TokType::Identifier ); eat( TokType::Identifier );
}
if ( currtok.Type == TokType::Assign_Classifer ) if ( currtok.Type == TokType::Assign_Classifer )
{ {
@ -2874,24 +2888,57 @@ CodeEnum parse_enum( bool inplace_def )
} }
} }
CodeBody body = { nullptr };
if ( currtok.Type == TokType::BraceCurly_Open ) if ( currtok.Type == TokType::BraceCurly_Open )
{ {
body = (CodeBody) make_code();
eat( TokType::BraceCurly_Open ); eat( TokType::BraceCurly_Open );
body = currtok; Code member = CodeInvalid;
while ( currtok.Type != TokType::BraceCurly_Close ) while ( currtok.Type != TokType::BraceCurly_Close )
{ {
if ( currtok.Type == TokType::Comment ) switch ( currtok.Type )
{ {
case TokType::Comment:
member = def_comment( currtok );
eat( TokType::Comment ); eat( TokType::Comment );
continue; break;
}
else if ( currtok.Type == TokType::Preprocess_Macro ) case TokType::Preprocess_Define:
{ member = parse_define();
break;
case TokType::Preprocess_If:
case TokType::Preprocess_IfDef:
case TokType::Preprocess_IfNotDef:
case TokType::Preprocess_ElIf:
member = parse_preprocess_cond();
break;
case TokType::Preprocess_Else:
member = preprocess_else;
eat( TokType::Preprocess_Else );
break;
case TokType::Preprocess_EndIf:
member = preprocess_endif;
eat( TokType::Preprocess_EndIf );
break;
case TokType::Preprocess_Macro:
member = untyped_str( currtok );
eat( TokType::Preprocess_Macro ); eat( TokType::Preprocess_Macro );
continue; break;
}
case TokType::Preprocess_Pragma:
member = parse_pragma();
break;
default:
Token entry = currtok;
eat( TokType::Identifier); eat( TokType::Identifier);
@ -2909,9 +2956,22 @@ CodeEnum parse_enum( bool inplace_def )
{ {
eat( TokType::Comma ); eat( TokType::Comma );
} }
entry.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)entry.Text;
member = untyped_str( entry );
break;
} }
body.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)body.Text; if ( member == Code::Invalid )
{
log_failure( "Failed to parse member\n%s", Context.to_string() );
Context.pop();
return CodeInvalid;
}
body.append( member );
}
eat( TokType::BraceCurly_Close ); eat( TokType::BraceCurly_Close );
} }
@ -2924,14 +2984,10 @@ CodeEnum parse_enum( bool inplace_def )
CodeEnum CodeEnum
result = (CodeEnum) make_code(); result = (CodeEnum) make_code();
if ( body.Length ) if ( body.ast )
{ {
// mem_copy( entries_code, body.Text, body.Length );
Code untyped_body = untyped_str( body );
result->Type = is_enum_class ? Enum_Class : Enum; result->Type = is_enum_class ? Enum_Class : Enum;
result->Body = untyped_body; result->Body = body;
} }
else else
{ {
@ -3884,10 +3940,67 @@ CodeUnion parse_union( bool inplace_def )
while ( ! check( TokType::BraceCurly_Close ) ) while ( ! check( TokType::BraceCurly_Close ) )
{ {
Code entry = parse_variable(); Code member = { nullptr };
switch ( currtok.Type )
{
case TokType::Comment:
member = def_comment( currtok );
eat( TokType::Comment );
break;
if ( entry ) case TokType::Decl_Class:
body.append( entry ); member = parse_complicated_definition( TokType::Decl_Class );
break;
case TokType::Decl_Enum:
member = parse_complicated_definition( TokType::Decl_Enum );
break;
case TokType::Decl_Struct:
member = parse_complicated_definition( TokType::Decl_Struct );
break;
case TokType::Decl_Union:
member = parse_complicated_definition( TokType::Decl_Union );
break;
case TokType::Preprocess_Define:
member = parse_define();
break;
case TokType::Preprocess_If:
case TokType::Preprocess_IfDef:
case TokType::Preprocess_IfNotDef:
case TokType::Preprocess_ElIf:
member = parse_preprocess_cond();
break;
case TokType::Preprocess_Else:
member = preprocess_else;
eat( TokType::Preprocess_Else );
break;
case TokType::Preprocess_EndIf:
member = preprocess_endif;
eat( TokType::Preprocess_EndIf );
break;
case TokType::Preprocess_Macro:
member = untyped_str( currtok );
eat( TokType::Preprocess_Macro );
break;
case TokType::Preprocess_Pragma:
member = parse_pragma();
break;
default:
member = parse_variable();
break;
}
if ( member )
body.append( member );
} }
eat( TokType::BraceCurly_Close ); eat( TokType::BraceCurly_Close );

View File

@ -99,17 +99,35 @@ sw token_fmt_va( char* buf, uw buf_size, s32 num_tokens, va_list va )
Code untyped_str( StrC content ) Code untyped_str( StrC content )
{ {
if ( content.Len == 0 )
{
log_failure( "untyped_str: empty string" );
return CodeInvalid;
}
Code Code
result = make_code(); result = make_code();
result->Name = get_cached_string( content ); result->Name = get_cached_string( content );
result->Type = ECode::Untyped; result->Type = ECode::Untyped;
result->Content = result->Name; result->Content = result->Name;
if ( result->Name == nullptr )
{
log_failure( "untyped_str: could not cache string" );
return CodeInvalid;
}
return result; return result;
} }
Code untyped_fmt( char const* fmt, ...) Code untyped_fmt( char const* fmt, ...)
{ {
if ( fmt == nullptr )
{
log_failure( "untyped_fmt: null format string" );
return CodeInvalid;
}
local_persist thread_local local_persist thread_local
char buf[GEN_PRINTF_MAXLEN] = { 0 }; char buf[GEN_PRINTF_MAXLEN] = { 0 };
@ -124,11 +142,23 @@ Code untyped_fmt( char const* fmt, ...)
result->Type = ECode::Untyped; result->Type = ECode::Untyped;
result->Content = get_cached_string( { length, buf } ); result->Content = get_cached_string( { length, buf } );
if ( result->Name == nullptr )
{
log_failure( "untyped_fmt: could not cache string" );
return CodeInvalid;
}
return result; return result;
} }
Code untyped_token_fmt( s32 num_tokens, ... ) Code untyped_token_fmt( s32 num_tokens, ... )
{ {
if ( num_tokens == 0 )
{
log_failure( "untyped_token_fmt: zero tokens" );
return CodeInvalid;
}
local_persist thread_local local_persist thread_local
char buf[GEN_PRINTF_MAXLEN] = { 0 }; char buf[GEN_PRINTF_MAXLEN] = { 0 };
@ -143,5 +173,11 @@ Code untyped_token_fmt( s32 num_tokens, ... )
result->Type = ECode::Untyped; result->Type = ECode::Untyped;
result->Content = result->Name; result->Content = result->Name;
if ( result->Name == nullptr )
{
log_failure( "untyped_fmt: could not cache string" );
return CodeInvalid;
}
return result; return result;
} }

View File

View File

@ -18,11 +18,18 @@ void check_singleheader_ast()
CodeBody ast = parse_global_body( { file.size, (char const*)file.data } ); CodeBody ast = parse_global_body( { file.size, (char const*)file.data } );
log_fmt("generated AST!!!"); log_fmt("generated AST!!!\n");
s32 idx = 0;
for ( Code entry : ast )
{
log_fmt("Entry %d: %s", idx, entry.to_string() );
idx++;
}
Builder builder; Builder builder;
builder.open( "singleheader_copy.hpp" ); builder.open( "singleheader_copy.hpp" );
log_fmt("serializng ast"); log_fmt("serializng ast\n");
builder.print( ast ); builder.print( ast );
builder.write(); builder.write();