ed/gencpp
1
0 Fork 0
mirror of https://github.com/Ed94/gencpp.git synced 2026-11-14 02:03:24 -08:00
Code Issues Packages Projects Releases Wiki Activity

Parser: Started to support proper errors as values, error recovery, and verbose testing of upfront and parsing interface.

Browse Source 
Also fix to c_library generation for Genric_ on code body.
This commit is contained in:
ed committed 2026-10-10 20:23:19 -04:00
1 parent ed755e34f1
commit 8ba02de6ed
9 files changed
+1982 -604
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

base/components/interface.hpp
+44 -26
View File
@@ -371,8 +371,7 @@ struct ParseStackNode
// TODO(Ed): When an error occurs, the parse stack is not released and instead the scope is left dangling.
};
struct ParseOpts
{
struct ParseOpts {
AllocatorInfo backing_msgs;
AllocatorInfo backing_tokens;
AllocatorInfo backing_ast;
@@ -380,34 +379,53 @@ struct ParseOpts
ParseInfo wip_parse_str( LexedInfo lexed, ParseOpts* opts GEN_PARAM_DEFAULT );
GEN_API CodeClass parse_class ( Str class_def );
GEN_API CodeConstructor parse_constructor ( Str constructor_def );
GEN_API CodeDefine parse_define ( Str define_def );
GEN_API CodeDestructor parse_destructor ( Str destructor_def );
GEN_API CodeEnum parse_enum ( Str enum_def );
GEN_API CodeBody parse_export_body ( Str export_def );
GEN_API CodeExtern parse_extern_link ( Str exten_link_def );
GEN_API CodeFriend parse_friend ( Str friend_def );
GEN_API CodeFn parse_function ( Str fn_def );
struct Opts_parse_global_body
{
struct Opts_parse {
Context* ctx;
ParseInfo* info;
};
GEN_API void parse_global_body_base( Context* ctx, Str body_def, ParseInfo* info );
GEN_API CodeBody parse_global_body( Str body_def, Opts_parse_global_body opts GEN_PARAM_DEFAULT );
typedef Opts_parse Opts_parse_global_body;
GEN_API CodeNS parse_namespace ( Str namespace_def );
GEN_API CodeOperator parse_operator ( Str operator_def );
GEN_API CodeOpCast parse_operator_cast( Str operator_def );
GEN_API CodeStruct parse_struct ( Str struct_def );
GEN_API CodeTemplate parse_template ( Str template_def );
GEN_API CodeTypename parse_type ( Str type_def );
GEN_API CodeTypedef parse_typedef ( Str typedef_def );
GEN_API CodeUnion parse_union ( Str union_def );
GEN_API CodeUsing parse_using ( Str using_def );
GEN_API CodeVar parse_variable ( Str var_def );
GEN_API void parse_class_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_constructor_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_define_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_destructor_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_enum_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_export_body_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_extern_link_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_friend_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_function_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_global_body_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_namespace_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_operator_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_operator_cast_base(Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_struct_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_template_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_type_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_typedef_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_union_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_using_base (Context* ctx, Str def, ParseInfo* info);
GEN_API void parse_variable_base (Context* ctx, Str def, ParseInfo* info);
GEN_API CodeClass parse_class (Str class_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeConstructor parse_constructor (Str constructor_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeDefine parse_define (Str define_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeDestructor parse_destructor (Str destructor_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeEnum parse_enum (Str enum_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeBody parse_export_body (Str export_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeExtern parse_extern_link (Str exten_link_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeFriend parse_friend (Str friend_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeFn parse_function (Str fn_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeBody parse_global_body (Str body_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeNS parse_namespace (Str namespace_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeOperator parse_operator (Str operator_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeOpCast parse_operator_cast(Str operator_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeStruct parse_struct (Str struct_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeTemplate parse_template (Str template_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeTypename parse_type (Str type_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeTypedef parse_typedef (Str typedef_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeUnion parse_union (Str union_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeUsing parse_using (Str using_def, Opts_parse opts GEN_PARAM_DEFAULT);
GEN_API CodeVar parse_variable (Str var_def, Opts_parse opts GEN_PARAM_DEFAULT);
#pragma endregion Parsing
base/components/interface.parsing.cpp
+289 -481
View File
@@ -11,8 +11,7 @@
ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts)
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
ParseInfo info = struct_zero(ParseInfo);
Context* ctx = _ctx; ParseInfo info = struct_zero(ParseInfo);
if (lexed.tokens.num == 0 && lexed.tokens.ptr == nullptr) {
ctx->parser = struct_zero(ParseContext);
@@ -26,92 +25,74 @@ ParseInfo wip_parse_str(LexedInfo lexed, ParseOpts* opts)
// TODO(Ed): ParseInfo should be set to the parser context.
ctx->parser = struct_zero(ParseContext);
ctx->parser.tokens = lexed.tokens;
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope);
CodeBody result = parse_global_nspace(ctx,CT_Global_Body);
(void)result;
parser_pop(& ctx->parser);
ctx->parser = struct_zero(ParseContext);
ctx->parser.tokens = lexed.tokens; {
ctx->parser.lexed_messages = lexed.messages;
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope);
CodeBody result = parse_global_nspace(ctx,CT_Global_Body); (void)result;
parser_pop(& ctx->parser);
}
done:
return info;
}
CodeClass parse_class( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeClass result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
internal b32 parse_base_begin(Context* ctx, Str def, ParseInfo* info, char const* function_name) {
if (ctx == nullptr || info == nullptr) GEN_FATAL("parse base: null context or info");
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
info[0] = struct_zero(ParseInfo);
info->result = Code_Invalid;
ctx->parser = struct_zero(ParseContext);
if (_check_parse_args(ctx, def, function_name) == false) {
info->messages = ctx->parser.messages;
return false;
}
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope);
result = (CodeClass) parse_class_struct( ctx, Tok_Decl_Class, parser_not_inplace_def );
parser_pop(& ctx->parser);
info->lexed = lex(ctx, def);
ctx->parser.lexed_messages = info->lexed.messages;
if (info->lexed.tokens.num == 0) {
info->messages = ctx->parser.messages;
return false;
}
done:
return result;
ctx->parser.tokens = info->lexed.tokens;
return true;
}
CodeConstructor parse_constructor(Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeConstructor result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
void parse_class_base(Context* ctx, Str def, ParseInfo* info) {
CodeClass result = InvalidCode;
if (parse_base_begin(ctx, def, info, "parse_class") == false) return;
/*Parse*/ {
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope);
result = (CodeClass) parse_class_struct( ctx, Tok_Decl_Class, parser_not_inplace_def );
parser_pop(& ctx->parser);
}
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope);
CodeClass parse_class(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_class: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_class_base(ctx, def, info); return cast(CodeClass, info->result);
}
// TODO(Ed): Constructors can have prefix attributes
void parse_constructor_base(Context* ctx, Str def, ParseInfo* info)
{
CodeConstructor result = InvalidCode;
if (parse_base_begin(ctx, def, info, "parse_constructor") == false) return;
/*Parse*/ {
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope);
CodeSpecifiers specifiers = NullCode;
Specifier specs_found[ 16 ] = { Spec_NumSpecifiers };
s32 NumSpecifiers = 0;
while ( left && tok_is_specifier(currtok) )
{
Specifier spec = str_to_specifier( currtok.Text );
b32 ignore_spec = false;
switch ( spec )
// TODO(Ed): Constructors can have prefix attributes
CodeSpecifiers specifiers = NullCode;
Specifier specs_found[ 16 ] = { Spec_NumSpecifiers };
s32 NumSpecifiers = 0;
while ( left && tok_is_specifier(currtok) )
{
b32 ignore_spec = false;
Specifier spec = str_to_specifier( currtok.Text );
switch ( spec ) {
case Spec_Constexpr :
case Spec_Explicit:
case Spec_Inline :
@@ -126,227 +107,160 @@ CodeConstructor parse_constructor(Str def )
default :
log_failure( "Invalid specifier %s for variable\n%S", spec_to_str( spec ), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) );
parser_pop(& ctx->parser);
goto done;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
return;
}
// Every specifier after would be considered part of the type type signature
if (ignore_spec) break;
specs_found[ NumSpecifiers ] = spec;
++ NumSpecifiers;
eat( currtok.Type );
}
// Every specifier after would be considered part of the type type signature
if (ignore_spec)
break;
specs_found[ NumSpecifiers ] = spec;
NumSpecifiers++;
eat( currtok.Type );
if ( NumSpecifiers ) {
specifiers = def_specifiers_arr( NumSpecifiers, specs_found );
// <specifiers> ...
}
result = parser_parse_constructor(ctx, specifiers);
parser_pop(& ctx->parser);
}
if ( NumSpecifiers ) {
specifiers = def_specifiers_arr( NumSpecifiers, specs_found );
// <specifiers> ...
}
result = parser_parse_constructor(ctx, specifiers);
parser_pop(& ctx->parser);
}
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeDefine parse_define( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeConstructor parse_constructor(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_constructor: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_constructor_base(ctx, def, info); return cast(CodeConstructor, info->result);
}
void parse_define_base(Context* ctx, Str def, ParseInfo* info) {
CodeDefine result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
if (parse_base_begin(ctx, def, info, "parse_define") == false) return;
/*Parse*/ {
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope);
result = parser_parse_define(ctx);
parser_pop(& ctx->parser);
}
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope);
result = parser_parse_define(ctx);
parser_pop(& ctx->parser);
}
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeDestructor parse_destructor( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeDefine parse_define(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_define: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_define_base(ctx, def, info); return cast(CodeDefine, info->result);
}
void parse_destructor_base(Context* ctx, Str def, ParseInfo* info) {
CodeDestructor result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_destructor") == false) return;
// TODO(Ed): Destructors can have prefix attributes
// TODO(Ed): Destructors can have virtual
result = parser_parse_destructor(ctx, NullCode);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeEnum parse_enum( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeDestructor parse_destructor(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_destructor: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_destructor_base(ctx, def, info); return cast(CodeDestructor, info->result);
}
void parse_enum_base(Context* ctx, Str def, ParseInfo* info) {
CodeEnum result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_enum") == false) return;
result = parser_parse_enum(ctx, parser_not_inplace_def);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeBody parse_export_body( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeEnum parse_enum(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_enum: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_enum_base(ctx, def, info); return cast(CodeEnum, info->result);
}
void parse_export_body_base(Context* ctx, Str def, ParseInfo* info) {
CodeBody result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_export_body") == false) return;
result = parser_parse_export_body(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeExtern parse_extern_link( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeBody parse_export_body(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_export_body: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_export_body_base(ctx, def, info); return cast(CodeBody, info->result);
}
void parse_extern_link_base(Context* ctx, Str def, ParseInfo* info) {
CodeExtern result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_extern_link") == false) return;
result = parser_parse_extern_link(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeFriend parse_friend( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeExtern parse_extern_link(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_extern_link: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_extern_link_base(ctx, def, info); return cast(CodeExtern, info->result);
}
void parse_friend_base(Context* ctx, Str def, ParseInfo* info) {
CodeFriend result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_friend") == false) return;
result = parser_parse_friend(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeFn parse_function( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeFn result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
CodeFriend parse_friend(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_friend: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_friend_base(ctx, def, info); return cast(CodeFriend, info->result);
}
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
void parse_function_base(Context* ctx, Str def, ParseInfo* info) {
CodeFn result = InvalidCode;
if (parse_base_begin(ctx, def, info, "parse_function") == false) return;
result = (CodeFn) parser_parse_function(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeFn parse_function(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts);
Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_function: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = & ctx->parse_info;
parse_function_base(ctx, def, info); return cast(CodeFn, info->result);
}
void parse_global_body_base( Context* ctx, Str def, ParseInfo* info )
{
LexedInfo lexed;
CodeBody body;
if ( ctx == nullptr || info == nullptr )
GEN_FATAL( "parse_global_body_base: null context or info" );
LexedInfo lexed; CodeBody body;
if ( ctx == nullptr || info == nullptr ) GEN_FATAL( "parse_global_body_base: null context or info" );
info[0] = struct_zero( ParseInfo );
lexed = struct_zero( LexedInfo );
@@ -362,6 +276,7 @@ void parse_global_body_base( Context* ctx, Str def, ParseInfo* info )
lexed = lex( ctx, def );
info->lexed = lexed;
ctx->parser.lexed_messages = lexed.messages;
if ( lexed.tokens.num == 0 ) {
info->result = cast( Code, body );
info->messages = ctx->parser.messages;
@@ -378,286 +293,179 @@ void parse_global_body_base( Context* ctx, Str def, ParseInfo* info )
info->messages = ctx->parser.messages;
}
CodeBody parse_global_body( Str def, Opts_parse_global_body opts ) {
Opts_parse_global_body resolved = get_optional( opts );
Context* ctx = resolved.ctx;
ParseInfo* info;
if ( ctx == nullptr ) ctx = _ctx;
if ( ctx == nullptr ) GEN_FATAL( "parse_global_body: no context" );
info = resolved.info; if ( info == nullptr ) info = & ctx->parse_info;
parse_global_body_base( ctx, def, info );
return cast( CodeBody, info->result );
CodeBody parse_global_body( Str def, Opts_parse opts ) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if ( ctx == nullptr ) ctx = _ctx; if ( ctx == nullptr ) GEN_FATAL( "parse_global_body: no context" );
ParseInfo* info = resolved.info; if ( info == nullptr ) info = & ctx->parse_info;
parse_global_body_base( ctx, def, info ); return cast( CodeBody, info->result );
}
CodeNS parse_namespace( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeNS result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
void parse_namespace_base(Context* ctx, Str def, ParseInfo* info) {
CodeNS result = InvalidCode;
if (parse_base_begin(ctx, def, info, "parse_namespace") == false) return;
result = parser_parse_namespace(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeOperator parse_operator( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeNS parse_namespace(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_namespace: no context");
ParseInfo* info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_namespace_base(ctx, def, info); return cast(CodeNS, info->result);
}
void parse_operator_base(Context* ctx, Str def, ParseInfo* info) {
CodeOperator result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_operator") == false) return;
result = (CodeOperator) parser_parse_operator(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeOpCast parse_operator_cast( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeOperator parse_operator(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_operator: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_operator_base(ctx, def, info); return cast(CodeOperator, info->result);
}
void parse_operator_cast_base(Context* ctx, Str def, ParseInfo* info) {
CodeOpCast result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_operator_cast") == false) return;
result = parser_parse_operator_cast(ctx, NullCode);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeStruct parse_struct( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeOpCast parse_operator_cast(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_operator_cast: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_operator_cast_base(ctx, def, info); return cast(CodeOpCast, info->result);
}
void parse_struct_base(Context* ctx, Str def, ParseInfo* info) {
CodeStruct result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
if (parse_base_begin(ctx, def, info, "parse_struct") == false) return;
/*Parse*/ {
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope);
result = (CodeStruct) parse_class_struct( ctx, Tok_Decl_Struct, parser_not_inplace_def );
parser_pop(& ctx->parser);
}
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope);
result = (CodeStruct) parse_class_struct( ctx, Tok_Decl_Struct, parser_not_inplace_def );
parser_pop(& ctx->parser);
}
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeTemplate parse_template( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeStruct parse_struct(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_struct: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_struct_base(ctx, def, info); return cast(CodeStruct, info->result);
}
void parse_template_base(Context* ctx, Str def, ParseInfo* info) {
CodeTemplate result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_template") == false) return;
result = parser_parse_template(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeTypename parse_type( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeTemplate parse_template(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_template: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_template_base(ctx, def, info); return cast(CodeTemplate, info->result);
}
void parse_type_base(Context* ctx, Str def, ParseInfo* info) {
CodeTypename result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
if (parse_base_begin(ctx, def, info, "parse_type") == false) return;
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
result = parser_parse_type( ctx, parser_not_from_template, nullptr);
done:
return result;
result = parser_parse_type(ctx, parser_not_from_template, nullptr);
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeTypedef parse_typedef( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeTypename parse_type(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_type: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_type_base(ctx, def, info); return cast(CodeTypename, info->result);
}
void parse_typedef_base(Context* ctx, Str def, ParseInfo* info) {
CodeTypedef result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_typedef") == false) return;
result = parser_parse_typedef(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeUnion parse_union( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeTypedef parse_typedef(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_typedef: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_typedef_base(ctx, def, info); return cast(CodeTypedef, info->result);
}
void parse_union_base(Context* ctx, Str def, ParseInfo* info) {
CodeUnion result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_union") == false) return;
result = parser_parse_union(ctx, parser_not_inplace_def);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeUsing parse_using( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeUnion parse_union(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_union: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_union_base(ctx, def, info); return cast(CodeUnion, info->result);
}
void parse_using_base(Context* ctx, Str def, ParseInfo* info) {
CodeUsing result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parse_base_begin(ctx, def, info, "parse_using") == false) return;
result = parser_parse_using(ctx);
done:
return result;
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
CodeVar parse_variable( Str def )
{
// TODO(Ed): Lift this.
Context* ctx = _ctx;
CodeVar result = InvalidCode;
LexedInfo lexed = struct_zero(LexedInfo);
CodeUsing parse_using(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_using: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_using_base(ctx, def, info); return cast(CodeUsing, info->result);
}
ctx->parser = struct_zero(ParseContext);
if (check_parse_args(def) == false)
goto done;
lexed = lex(ctx, def);
ctx->parser.tokens = lexed.tokens;
if (lexed.messages != nullptr || ctx->parser.tokens.num == 0) {
if (lexed.messages != nullptr && lexed.messages->content.Ptr != nullptr)
parser_record_failure(ctx, lexed.messages->content);
else
parser_record_failure(ctx, txt("parse: lex produced no tokens"));
goto done;
}
if (parser_slice_is_formatting_only(&ctx->parser)) {
void parse_variable_base(Context* ctx, Str def, ParseInfo* info) {
CodeVar result = InvalidCode;
if (parse_base_begin(ctx, def, info, "parse_variable") == false) return;
if (parser_slice_is_formatting_only(& ctx->parser)) {
parser_record_failure(ctx, txt("parse_variable: only formatting tokens"));
goto done;
}
else {
result = parser_parse_variable(ctx);
}
info->result = cast(Code, result);
info->messages = ctx->parser.messages;
}
result = parser_parse_variable(ctx);
done:
return result;
CodeVar parse_variable(Str def, Opts_parse opts) {
Opts_parse resolved = get_optional(opts); Context* ctx = resolved.ctx; ParseInfo* info;
if (ctx == nullptr) ctx = _ctx; if (ctx == nullptr) GEN_FATAL("parse_variable: no context");
info = resolved.info; if (info == nullptr) info = &ctx->parse_info;
parse_variable_base(ctx, def, info); return cast(CodeVar, info->result);
}
// Undef helper macros
base/components/parser.cpp
+439 -29
View File
@@ -104,14 +104,49 @@ void parser_record_failure(Context* ctx, Str text) {
ctx->parser.messages = msg;
}
internal b32 parser_is_recoverable_body(ParseContext* parser) {
ParseStackNode* scope = parser->scope;
b32 in_function_body = false;
while ( scope )
{
if ( scope->code_rel )
{
CodeType type = scope->code_rel->Type;
if ( type == CT_Function_Body ) {
in_function_body = true;
scope = scope->prev;
continue;
}
if ( in_function_body && type != CT_Global_Body )
return false;
return type == CT_Global_Body || type == CT_Namespace_Body || type == CT_Class_Body || type == CT_Struct_Body;
}
scope = scope->prev;
}
return false;
}
bool lex__eat(Context* ctx, ParseContext* parser, TokType type)
{
b32 ate = false;
Token at_idx;
b32 not_accepted;
b32 is_identifier;
if (parser->tokens.num - parser->token_id <= 0)
s32 start_id = parser->token_id;
if ( parser->token_mismatch )
goto done;
if (parser->tokens.num - parser->token_id <= 0)
{
if ( parser_is_recoverable_body(parser) ) {
StrBuilder message = strbuilder_fmt_buf(ctx->Allocator_Temp, "Parse Error, TokArray::eat, Expected: ' %S ' at end of token slice\n%SB"
, toktype_to_str(type)
, parser_to_strbuilder(parser, ctx->Allocator_Temp)
);
parser_record_failure(ctx, strbuilder_to_str(message));
parser->token_mismatch = true;
}
goto done;
}
at_idx = parser->tokens.ptr[ parser->token_id ];
if ( ( at_idx.Type == Tok_NewLine && type != Tok_NewLine )
@@ -129,15 +164,26 @@ bool lex__eat(Context* ctx, ParseContext* parser, TokType type)
}
if ( not_accepted )
{
Token tok = * lex_current( parser, lex_skip_formatting );
log_failure( "Parse Error, TokArray::eat, Expected: ' %S ' not ' %S ' (%d, %d)`\n%SB"
parser->token_id = start_id;
Token tok = * lex_current( parser, lex_skip_formatting );
StrBuilder message = strbuilder_fmt_buf(ctx->Allocator_Temp, "Parse Error, TokArray::eat, Expected: ' %S ' not ' %S ' (%d, %d)`\n%SB"
, toktype_to_str(type)
, at_idx.Text
, tok.Line
, tok.Column
, parser_to_strbuilder(parser, ctx->Allocator_Temp)
);
GEN_DEBUG_TRAP();
Str message_text = strbuilder_to_str(message);
if ( parser_is_recoverable_body(parser) ) {
parser_record_failure(ctx, message_text);
parser->token_mismatch = true;
}
else if ( parser->lexed_messages )
parser_record_failure(ctx, message_text);
else {
log_failure("%S", message_text);
GEN_DEBUG_TRAP();
}
goto done;
}
@@ -206,7 +252,7 @@ done:
# define nexttok (* lex_next( & ctx->parser, lex_skip_formatting ))
# define nexttok_noskip (* lex_next( & ctx->parser, lex_dont_skip_formatting))
# define eat( Type_ ) lex__eat(ctx, & ctx->parser, Type_ )
# define left ( ctx->parser.tokens.num - ctx->parser.token_id )
# define left ( ! ctx->parser.token_mismatch && ctx->parser.tokens.num - ctx->parser.token_id )
#if GEN_COMPILER_CPP
# define def_assign( ... ) { __VA_ARGS__ }
@@ -233,6 +279,8 @@ done:
// Procedure Forwards ( Entire parser internal parser interface )
internal void parser_recover_body_member(Context* ctx, CodeBody body, Token member_start_token, s32 member_start_id, b32 allow_comma);
internal Code parse_array_decl (Context* ctx);
internal CodeAttributes parse_attributes (Context* ctx);
internal CodeComment parse_comment (Context* ctx);
@@ -533,8 +581,7 @@ StrBuilder parser_strip_formatting(Context* ctx, Str raw_text, bool preserve_new
return content;
}
StrBuilder parser_strip_formatting_2(TokenSlice tokens)
{
StrBuilder parser_strip_formatting_2(TokenSlice tokens) {
// TODO(Ed): Use this to produce strings for validation purposes. We shouldn't serialize down from tokens once we start storing token slices for content.
StrBuilder result = struct_zero(StrBuilder);
return result;
@@ -870,8 +917,14 @@ CodeBody parse_class_struct_body(Context* ctx, TokType which, Token name)
else
result->Type = CT_Struct_Body;
scope.code_rel = cast(Code, result);
while ( left && currtok_noskip.Type != Tok_BraceCurly_Close )
{
ParseMessage* messages_before_member = ctx->parser.messages;
s32 member_start_id = ctx->parser.token_id;
Token member_start_token = currtok_noskip;
Code member = Code_Invalid;
CodeAttributes attributes = { nullptr };
CodeSpecifiers specifiers = { nullptr };
@@ -1180,11 +1233,26 @@ CodeBody parse_class_struct_body(Context* ctx, TokType which, Token name)
break;
}
if ( member == Code_Invalid )
b32 member_is_nested_scope = member != Code_Invalid && (member->Type == CT_Class || member->Type == CT_Struct);
b32 member_failed = ctx->parser.token_mismatch || (ctx->parser.messages != messages_before_member && member_is_nested_scope == false);
if (member_failed) {
if ( ctx->parser.messages != messages_before_member ) ctx->parser.messages->Next = messages_before_member;
ctx->parser.token_id = member_start_id;
ctx->parser.token_mismatch = false;
member = Code_Invalid;
}
if (member == Code_Invalid)
{
log_failure( "Failed to parse member\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) );
parser_pop(& ctx->parser);
return InvalidCode;
if ( member_failed == false ) {
log_failure( "Failed to parse member\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) );
parser_pop(& ctx->parser);
return InvalidCode;
}
ctx->parser.token_id = member_start_id;
ctx->parser.token_mismatch = false;
parser_recover_body_member(ctx, result, member_start_token, member_start_id, false);
continue;
}
body_append(result, member );
}
@@ -1196,10 +1264,8 @@ CodeBody parse_class_struct_body(Context* ctx, TokType which, Token name)
}
internal
CodeComment parse_comment(Context* ctx)
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope );
CodeComment parse_comment(Context* ctx) {
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope );
CodeComment
result = (CodeComment) make_code();
@@ -1215,8 +1281,7 @@ CodeComment parse_comment(Context* ctx)
internal
Code parse_complicated_definition(Context* ctx, TokType which)
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope );
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope );
b32 is_inplace = false;
b32 is_fn_def = false;
@@ -1399,8 +1464,7 @@ Code parse_complicated_definition(Context* ctx, TokType which)
internal inline
Code parse_assignment_expression(Context* ctx)
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope );
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope );
Code expr = { nullptr };
@@ -1445,7 +1509,6 @@ internal inline
Code parse_forward_or_definition(Context* ctx, TokType which, bool is_inplace )
{
Code result = InvalidCode;
switch ( which )
{
case Tok_Decl_Class:
@@ -1483,8 +1546,7 @@ CodeFn parse_function_after_name(Context* ctx
, Token name
)
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope );
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope );
CodeParams params = parse_params(ctx, parser_use_parenthesis);
// <Attributes> <Specifiers> <ReturnType> <Name> ( <Parameters> )
@@ -1639,14 +1701,14 @@ CodeFn parse_function_after_name(Context* ctx
internal
Code parse_function_body(Context* ctx)
{
ParseStackNode scope = NullScope;
parser_push(& ctx->parser, & scope );
ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope );
eat( Tok_BraceCurly_Open );
CodeBody
result = (CodeBody) make_code();
result->Type = CT_Function_Body;
scope.code_rel = cast(Code, result);
// TODO : Support actual parsing of function body
Token start = currtok_noskip;
@@ -1667,8 +1729,7 @@ Code parse_function_body(Context* ctx)
s32 len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)start.Text.Ptr;
if ( len > 0 )
{
if ( len > 0 ) {
Str str = { start.Text.Ptr, len };
body_append( result, cast(Code, def_execution( str )) );
}
@@ -1679,6 +1740,294 @@ Code parse_function_body(Context* ctx)
return cast(Code, result);
}
internal
s32 parser_recovery_scope_close_id(ParseContext const* parser, s32 member_start_id, b32* nested_body, b32* has_scope_close)
{
nested_body[0] = false;
has_scope_close[0] = false;
ParseStackNode* scope = parser->scope;
while ( scope != nullptr )
{
b32 scope_has_code = scope->code_rel != nullptr;
if (scope_has_code)
{
CodeType type = scope->code_rel->Type;
b32 is_nested_scope_body = false;
is_nested_scope_body |= type == CT_Namespace_Body;
is_nested_scope_body |= type == CT_Class_Body;
is_nested_scope_body |= type == CT_Struct_Body;
// Function-body frames have their own brace owner; inspect the enclosing body frame.
if ( type == CT_Function_Body ) { scope = scope->prev; continue; }
b32 nothing_to_recover = type == CT_Global_Body || is_nested_scope_body == false;
// Global recovery uses the full token slice. Other unsupported scope types have no nested boundary.
if (nothing_to_recover) return parser->tokens.num; // TODO(Ed): Drain to single return.
// Nested recovery needs the active body's closing brace as a hard delimiter boundary.
nested_body[0] = true;
s32 scope_start_id = parser->tokens.num;
b32 scope_has_start = scope->start != nullptr;
if (scope_has_start) scope_start_id = scast(s32, scope->start - parser->tokens.ptr);
// Scope start may point at formatting before the body's opening brace.
s32 body_open_id = scope_start_id;
while ( body_open_id < parser->tokens.num ) {
b32 token_opens_body = parser->tokens.ptr[body_open_id].Type == Tok_BraceCurly_Open;
if (token_opens_body == true) break;
++ body_open_id;
}
b32 found_body_open = body_open_id < parser->tokens.num;
if (found_body_open == false) return parser->tokens.num;
// A depth-zero close belongs to this body; nested closes only reduce depth.
s32 body_depth = 0;
for (s32 idx = body_open_id; idx < parser->tokens.num; ++ idx)
{
TokType token_type = parser->tokens.ptr[idx].Type;
if (token_type == Tok_BraceCurly_Open)
++ body_depth;
else if (token_type == Tok_BraceCurly_Close)
{
if (body_depth > 0) // inside nested brace
{
-- body_depth;
if (body_depth == 0) { // found owner close
has_scope_close[0] = true;
// TODO(Ed): Drain to single return.
return idx;
}
}
}
}
s32 fallback_close_id = parser->tokens.num;
if (type == CT_Class_Body || type == CT_Struct_Body)
{
// A malformed member opener can keep the brace-depth scan from reaching the owner close.
// Direct typed entries scan to their last candidate; nested bodies stop at the first candidate.
b32 has_parent_body = false;
for (ParseStackNode* parent = scope->prev; parent != nullptr; parent = parent->prev)
{
b32 parent_has_code = parent->code_rel != nullptr;
if (parent_has_code == false) continue;
b32 parent_is_function_body = parent->code_rel->Type == CT_Function_Body;
if (parent_is_function_body) continue;
CodeType parent_type = parent->code_rel->Type;
b32 parent_owns_body = false;
parent_owns_body |= parent_type == CT_Global_Body;
parent_owns_body |= parent_type == CT_Namespace_Body;
parent_owns_body |= parent_type == CT_Class_Body;
parent_owns_body |= parent_type == CT_Struct_Body;
has_parent_body = parent_owns_body;
break;
}
for (s32 idx = member_start_id; idx < parser->tokens.num; ++ idx)
{
if (parser->tokens.ptr[idx].Type != Tok_BraceCurly_Close) continue;
s32 next_id = idx + 1;
while (next_id < parser->tokens.num) {
TokType next_type = parser->tokens.ptr[next_id].Type;
b32 next_is_formatting = next_type == Tok_NewLine || next_type == Tok_Comment;
if (next_is_formatting == false) break;
++ next_id;
}
b32 next_ends_statement = false;
b32 close_ends_declaration = false;
b32 next_in_slice = next_id < parser->tokens.num;
if (next_in_slice) {
next_ends_statement = parser->tokens.ptr[next_id].Type == Tok_Statement_End;
close_ends_declaration = next_ends_statement;
}
if (close_ends_declaration) {
fallback_close_id = idx;
if (has_parent_body) break;
}
}
}
else
{
// A namespace close follows a member terminator, unlike a nested type close followed by `;`.
for (s32 idx = member_start_id; idx < parser->tokens.num; ++ idx)
{
if (parser->tokens.ptr[idx].Type != Tok_BraceCurly_Close) continue;
s32 prev_id = idx - 1;
while (prev_id >= body_open_id) {
TokType prev_type = parser->tokens.ptr[prev_id].Type;
b32 prev_is_formatting = prev_type == Tok_NewLine || prev_type == Tok_Comment;
if ( prev_is_formatting == false ) break;
-- prev_id;
}
s32 next_id = idx + 1;
while (next_id < parser->tokens.num) {
TokType next_type = parser->tokens.ptr[next_id].Type;
b32 next_is_formatting = next_type == Tok_NewLine || next_type == Tok_Comment;
if ( next_is_formatting == false ) break;
++ next_id;
}
b32 prev_in_body = prev_id >= body_open_id;
b32 prev_ends_member = false;
if (prev_in_body) prev_ends_member = parser->tokens.ptr[prev_id].Type == Tok_Statement_End;
b32 next_ends_declaration = false;
b32 next_past_slice = next_id >= parser->tokens.num;
if (next_past_slice == false) next_ends_declaration = parser->tokens.ptr[next_id].Type == Tok_Statement_End;
b32 member_ended_before_close = false;
b32 close_starts_next_declaration = false;
b32 namespace_owner_close = false;
if (prev_in_body && prev_ends_member) member_ended_before_close = true;
if (next_past_slice == false && next_ends_declaration) close_starts_next_declaration = true;
if (member_ended_before_close && close_starts_next_declaration == false) namespace_owner_close = true;
if (namespace_owner_close) {
fallback_close_id = idx;
break;
}
}
}
if (fallback_close_id < parser->tokens.num) { // found fallback close
has_scope_close[0] = true;
return fallback_close_id;
}
return parser->tokens.num;
}
scope = scope->prev;
}
return parser->tokens.num;
}
internal
b32 parser_recovery_delimiter_is_balanced(ParseContext const* parser
, s32 start_id
, TokType open_type
, TokType close_type
, s32 scope_close_id
, b32 nested_body
, b32 has_scope_close
){
// Inner delimiters shield semicolons only when their close precedes the owning body close.
if ( open_type == Tok_BraceCurly_Open && nested_body && has_scope_close == false ) return false;
s32 delimiter_depth = 0;
s32 curly_depth = 0;
for ( s32 idx = start_id; idx < scope_close_id; ++ idx )
{
TokType type = parser->tokens.ptr[idx].Type;
if ( type == Tok_BraceCurly_Open )
++ curly_depth;
else if ( type == Tok_BraceCurly_Close )
{
if ( curly_depth == 0 ) return false;
-- curly_depth;
}
if ( type == open_type )
++ delimiter_depth;
else if ( type == close_type && delimiter_depth > 0 )
{
-- delimiter_depth;
if ( delimiter_depth == 0 ) return true;
}
}
return false;
}
internal
void parser_recover_body_member(Context* ctx, CodeBody body, Token member_start_token, s32 member_start_id, b32 allow_comma)
{
// Keep the original member start for the untyped source span.
Token recovery_end_token = member_start_token;
b32 has_recovery_tokens = false;
s32 paren_depth = 0;
s32 square_depth = 0;
s32 curly_depth = 0;
b32 nested_body = false;
b32 has_scope_close = false;
// Bound delimiter matches before scanning the rejected member.
s32 scope_close_id = parser_recovery_scope_close_id(& ctx->parser, member_start_id, &nested_body, &has_scope_close);
while ( left )
{
Token token = currtok_noskip;
// Leave the active body's close for its owner.
b32 is_scope_close = token.Type == Tok_BraceCurly_Close && curly_depth == 0;
if (is_scope_close) break;
b32 at_member_start = ctx->parser.token_id == member_start_id;
b32 at_delimiter_depth_zero = paren_depth == 0 && square_depth == 0 && curly_depth == 0;
b32 sync_at_comma = allow_comma && at_member_start && at_delimiter_depth_zero && token.Type == Tok_Comma;
b32 sync_at_statement_end = token.Type == Tok_Statement_End && at_delimiter_depth_zero;
b32 sync_at_member_boundary = sync_at_comma || sync_at_statement_end;
// Semicolons sync at depth zero; only global recovery accepts a leading comma.
if (sync_at_member_boundary)
{
TokType sync_type = Tok_Statement_End;
if (sync_at_comma) sync_type = Tok_Comma;
if (eat( sync_type )) {
recovery_end_token = token;
has_recovery_tokens = true;
}
break;
}
// Add delimiter depth only when its matching close precedes scope_close_id.
switch ( token.Type ) { default: break;
case Tok_Paren_Open: {
b32 res = parser_recovery_delimiter_is_balanced(& ctx->parser, ctx->parser.token_id
, Tok_Paren_Open, Tok_Paren_Close
, scope_close_id, nested_body, has_scope_close);
if (res) ++ paren_depth;
break;
}
case Tok_Paren_Close:
if ( paren_depth > 0 ) -- paren_depth;
break;
case Tok_BraceSquare_Open: {
b32 res = parser_recovery_delimiter_is_balanced(& ctx->parser, ctx->parser.token_id
, Tok_BraceSquare_Open, Tok_BraceSquare_Close
, scope_close_id, nested_body, has_scope_close);
if (res) ++ square_depth;
break;
}
case Tok_BraceSquare_Close:
if ( square_depth > 0 ) -- square_depth;
break;
case Tok_BraceCurly_Open: {
b32 res = parser_recovery_delimiter_is_balanced(& ctx->parser, ctx->parser.token_id
, Tok_BraceCurly_Open, Tok_BraceCurly_Close
, scope_close_id, nested_body, has_scope_close);
if (res) ++ curly_depth;
break;
}
case Tok_BraceCurly_Close:
if ( curly_depth > 0 ) -- curly_depth;
break;
}
if (eat( token.Type )) {
recovery_end_token = token;
has_recovery_tokens = true;
}
}
// Retain the failed source through its synchronization token as untyped code.
if (has_recovery_tokens) body_append(body, untyped_str(token_range_to_str(member_start_token, recovery_end_token)));
}
internal neverinline
CodeBody parse_global_nspace(Context* ctx, CodeType which)
{
@@ -1695,9 +2044,13 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
CodeBody
result = (CodeBody) make_code();
result->Type = which;
scope.code_rel = cast(Code, result);
while ( left && currtok_noskip.Type != Tok_BraceCurly_Close )
{
ParseMessage* messages_before_member = ctx->parser.messages;
s32 member_start_id = ctx->parser.token_id;
Token member_start_token = currtok_noskip;
Code member = Code_Invalid;
CodeAttributes attributes = { nullptr };
CodeSpecifiers specifiers = { nullptr };
@@ -1716,8 +2069,12 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
case Tok_Comma:
{
parser_record_failure(ctx, txt("Dangling comma"));
parser_pop( & ctx->parser);
return result;
if ( which != CT_Global_Body )
{
parser_pop( & ctx->parser);
return result;
}
member = Code_Invalid;
}
break;
case Tok_Statement_End:
@@ -1849,6 +2206,12 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
case Tok_Module_Import: {
// import ...
if ( which == CT_Global_Body )
{
parser_record_failure(ctx, txt("Module import parsing is not implemented"));
member = Code_Invalid;
break;
}
log_failure( "gen::%s: This function is not implemented" );
return InvalidCode;
}
@@ -1858,6 +2221,12 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
{
if ( ! tok_is_attribute(currtok))
{
if ( which == CT_Global_Body )
{
parser_record_failure(ctx, txt("Unbounded macro expression in global body"));
member = Code_Invalid;
break;
}
log_failure("Unbounded macro expression residing in class/struct body\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp));
return InvalidCode;
}
@@ -1878,6 +2247,7 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
{
Specifier specs_found[16] = { Spec_NumSpecifiers };
s32 NumSpecifiers = 0;
b32 invalid_global_specifier = false;
while ( left && tok_is_specifier(currtok) )
{
@@ -1900,11 +2270,22 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
default:
Str spec_str = spec_to_str(spec);
if ( which == CT_Global_Body )
{
StrBuilder message = strbuilder_fmt_buf(ctx->Allocator_Temp, "Invalid specifier %S for variable\n%SB"
, spec_str
, parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)
);
parser_record_failure(ctx, strbuilder_to_str(message));
invalid_global_specifier = true;
break;
}
log_failure( "Invalid specifier %S for variable\n%S", spec_str, strbuilder_to_str( parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)) );
parser_pop(& ctx->parser);
return InvalidCode;
}
if ( invalid_global_specifier )
break;
if (ignore_spec)
break;
@@ -1913,6 +2294,11 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
NumSpecifiers++;
eat( currtok.Type );
}
if ( invalid_global_specifier )
{
member = Code_Invalid;
break;
}
if ( NumSpecifiers )
{
@@ -1979,8 +2365,32 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which)
}
}
b32 member_is_nested_scope = member != Code_Invalid && (member->Type == CT_Class || member->Type == CT_Struct || member->Type == CT_Namespace);
b32 member_failed = ctx->parser.token_mismatch || (ctx->parser.messages != messages_before_member && ! member_is_nested_scope);
b32 can_recover_member = which == CT_Global_Body || which == CT_Namespace_Body;
if (can_recover_member && member_failed) {
if (ctx->parser.messages != messages_before_member) ctx->parser.messages->Next = messages_before_member;
ctx->parser.token_id = member_start_id;
ctx->parser.token_mismatch = false;
member = Code_Invalid;
}
if ( member == Code_Invalid )
{
if ( which == CT_Global_Body || (which == CT_Namespace_Body && member_failed) )
{
if ( ctx->parser.messages == messages_before_member )
{
StrBuilder message = strbuilder_fmt_buf(ctx->Allocator_Temp, "Failed to parse member\nToken: %SB\nContext:\n%SB"
, tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip)
, parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)
);
parser_record_failure(ctx, strbuilder_to_str(message));
}
parser_recover_body_member(ctx, result, member_start_token, member_start_id, which == CT_Global_Body);
continue;
}
log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) );
parser_pop(& ctx->parser);
return InvalidCode;
base/components/parser_types.hpp
+2
View File
@@ -162,6 +162,8 @@ struct ParseContext
// TokArray Tokens;
TokenSlice tokens;
s32 token_id;
LexerMessage* lexed_messages;
b32 token_mismatch;
};
enum MacroType : u16