#ifdef INTELLISENSE_DIRECTIVES #pragma once #include "gen/etoktype.hpp" #include "parser_case_macros.cpp" #include "interface.upfront.cpp" #include "lexer.cpp" #endif // TODO(Ed) : Rename ETok_Capture_Start, ETok_Capture_End to Open_Parenthesis adn Close_Parenthesis constexpr bool lex_dont_skip_formatting = false; constexpr bool lex_skip_formatting = true; void parser_push( ParseContext* ctx, ParseStackNode* node ) { node->prev = ctx->scope; ctx->scope = node; #if 0 && GEN_BUILD_DEBUG log_fmt("\tEntering parser: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); #endif } void parser_pop(ParseContext* ctx) { #if 0 && GEN_BUILD_DEBUG log_fmt("\tPopping parser: %.*s\n", Scope->ProcName.Len, Scope->ProcName.Ptr ); #endif ctx->scope = ctx->scope->prev; } StrBuilder parser_to_strbuilder(ParseContext const* ctx, AllocatorInfo temp) { StrBuilder result; b32 empty_slice = ctx->tokens.ptr == nullptr || ctx->tokens.num <= 0; if (empty_slice == false) empty_slice = ctx->scope->start == nullptr || ctx->scope->start->Text.Ptr == nullptr; // empty_scope if (empty_slice) { result = strbuilder_make_reserve(temp, 64); strbuilder_append_fmt(&result, "\tScope : \n"); goto done; } // non_empty { result = strbuilder_make_reserve( temp, kilobytes(4) ); Token scope_start = * ctx->scope->start; Token last_valid = (ctx->token_id >= ctx->tokens.num) ? ctx->tokens.ptr[ctx->tokens.num - 1] : (*lex_peek(ctx, true)); sptr length = scope_start.Text.Len; char const* current = scope_start.Text.Ptr + length; while ( current <= ctx->tokens.ptr[ctx->tokens.num - 1].Text.Ptr && (* current) != '\n' && length < 74 ) { current++; length++; } Str scope_str = { scope_start.Text.Ptr, length }; StrBuilder line = strbuilder_make_str( temp, scope_str ); strbuilder_append_fmt( & result, "\tScope : %s\n", line ); strbuilder_free(& line); sptr dist = (sptr)last_valid.Text.Ptr - (sptr)scope_start.Text.Ptr + 2; sptr length_from_err = dist; Str err_str = { last_valid.Text.Ptr, length_from_err }; StrBuilder line_from_err = strbuilder_make_str( temp, err_str ); if ( length_from_err < 100 ) strbuilder_append_fmt(& result, "\t(%d, %d):%*c\n", last_valid.Line, last_valid.Column, length_from_err, '^' ); else strbuilder_append_fmt(& result, "\t(%d, %d)\n", last_valid.Line, last_valid.Column ); ParseStackNode* curr_scope = ctx->scope; s32 level = 0; do { if ( curr_scope->name.Ptr ) { strbuilder_append_fmt(& result, "\t%d: %S, AST Name: %S\n", level, curr_scope->proc_name, curr_scope->name ); } else { strbuilder_append_fmt(& result, "\t%d: %S\n", level, curr_scope->proc_name ); } curr_scope = curr_scope->prev; level++; } while ( curr_scope ); } done: return result; } void parser_record_failure(Context* ctx, Str text) { ParseMessage* msg = rcast(ParseMessage*, alloc(ctx->Allocator_DyanmicContainers, size_of(ParseMessage))); if (msg == nullptr) GEN_FATAL("parser_record_failure: allocation failed"); msg->Next = ctx->parser.messages; msg->Scope = nullptr; msg->Content = cache_str(text); msg->Level = LL_Error; ctx->parser.messages = msg; } internal b32 parser_is_recoverable_body(ParseContext* parser) { ParseStackNode* scope = parser->scope; b32 in_function_body = false; while ( scope ) { if ( scope->code_rel ) { CodeType type = scope->code_rel->Type; if ( type == CT_Function_Body ) { in_function_body = true; scope = scope->prev; continue; } if ( in_function_body && type != CT_Global_Body ) return false; return type == CT_Global_Body || type == CT_Namespace_Body || type == CT_Class_Body || type == CT_Struct_Body; } scope = scope->prev; } return false; } bool lex__eat(Context* ctx, ParseContext* parser, TokType type) { b32 ate = false; Token at_idx; b32 not_accepted; b32 is_identifier; s32 start_id = parser->token_id; if ( parser->token_mismatch ) goto done; if (parser->tokens.num - parser->token_id <= 0) { if ( parser_is_recoverable_body(parser) ) { StrBuilder message = strbuilder_fmt_buf(ctx->Allocator_Temp, "Parse Error, TokArray::eat, Expected: ' %S ' at end of token slice\n%SB" , toktype_to_str(type) , parser_to_strbuilder(parser, ctx->Allocator_Temp) ); parser_record_failure(ctx, strbuilder_to_str(message)); parser->token_mismatch = true; } goto done; } at_idx = parser->tokens.ptr[ parser->token_id ]; if ( ( at_idx.Type == Tok_NewLine && type != Tok_NewLine ) || ( at_idx.Type == Tok_Comment && type != Tok_Comment ) ) { parser->token_id ++; } not_accepted = at_idx.Type != type; is_identifier = at_idx.Type == Tok_Identifier; if ( not_accepted ) { Macro* macro = lookup_macro(at_idx.Text); b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier ); not_accepted = type == Tok_Identifier && accept_as_identifier ? false : true; } if ( not_accepted ) { parser->token_id = start_id; Token tok = * lex_current( parser, lex_skip_formatting ); StrBuilder message = strbuilder_fmt_buf(ctx->Allocator_Temp, "Parse Error, TokArray::eat, Expected: ' %S ' not ' %S ' (%d, %d)`\n%SB" , toktype_to_str(type) , at_idx.Text , tok.Line , tok.Column , parser_to_strbuilder(parser, ctx->Allocator_Temp) ); Str message_text = strbuilder_to_str(message); if ( parser_is_recoverable_body(parser) ) { parser_record_failure(ctx, message_text); parser->token_mismatch = true; } else if ( parser->lexed_messages ) parser_record_failure(ctx, message_text); else { log_failure("%S", message_text); GEN_DEBUG_TRAP(); } goto done; } #if 0 && GEN_BUILD_DEBUG log_fmt("Ate: %SB\n", self->Arr[Idx].to_strbuilder() ); #endif parser->token_id ++; ate = true; done: return ate; } internal void parser_init(Context* ctx) { } internal void parser_deinit(Context* ctx) { } #pragma region Helper Macros b32 parser_slice_is_formatting_only(ParseContext const* parser) { b32 formatting_only = true; if (parser->tokens.ptr == nullptr || parser->tokens.num <= 0) goto done; for (s32 idx = 0; idx < parser->tokens.num; ++idx) { TokType type = parser->tokens.ptr[idx].Type; if (type != Tok_NewLine && type != Tok_Comment) { formatting_only = false; goto done; } } done: return formatting_only; } #define check_parse_args( def ) _check_parse_args(ctx, def, __func__) bool _check_parse_args(Context* ctx, Str def, char const* func_name ) { b32 ok = true; char const* text; if ( def.Len <= 0 ) { text = c_str_fmt_buf("gen::%s: length must greater than 0", func_name); parser_record_failure(ctx, to_str_from_c_str(text)); ok = false; goto done; } if ( def.Ptr == nullptr ) { text = c_str_fmt_buf("gen::%s: def was null", func_name); parser_record_failure(ctx, to_str_from_c_str(text)); ok = false; goto done; } done: return ok; } # define currtok_noskip (* lex_current( & ctx->parser, lex_dont_skip_formatting )) # define currtok (* lex_current( & ctx->parser, lex_skip_formatting )) # define peektok (* lex_peek(& ctx->parser, lex_skip_formatting)) # define prevtok (* lex_previous( & ctx->parser, lex_dont_skip_formatting)) # define nexttok (* lex_next( & ctx->parser, lex_skip_formatting )) # define nexttok_noskip (* lex_next( & ctx->parser, lex_dont_skip_formatting)) # define eat( Type_ ) lex__eat(ctx, & ctx->parser, Type_ ) # define left ( ! ctx->parser.token_mismatch && ctx->parser.tokens.num - ctx->parser.token_id ) #if GEN_COMPILER_CPP # define def_assign( ... ) { __VA_ARGS__ } #else # define def_assign( ... ) __VA_ARGS__ #endif #ifdef check #define CHECK_WAS_DEFINED #pragma push_macro("check") #undef check #endif # define check_noskip( Type_ ) ( left && currtok_noskip.Type == Type_ ) # define check( Type_ ) ( left && currtok.Type == Type_ ) #if GEN_COMPILER_CPP # define NullScope { nullptr, {nullptr, 0}, lex_current( & ctx->parser, lex_dont_skip_formatting ), Str{nullptr, 0}, txt( __func__ ), { nullptr} } #else # define NullScope (ParseStackNode){ nullptr, {nullptr, 0}, lex_current( & ctx->parser, lex_dont_skip_formatting ), (Str){nullptr, 0}, txt( __func__ ), { nullptr} } #endif #pragma endregion Helper Macros // Procedure Forwards ( Entire parser internal parser interface ) internal void parser_recover_body_member(Context* ctx, CodeBody body, Token member_start_token, s32 member_start_id, b32 allow_comma); internal Code parse_array_decl (Context* ctx); internal CodeAttributes parse_attributes (Context* ctx); internal CodeComment parse_comment (Context* ctx); internal Code parse_complicated_definition (Context* ctx, TokType which); internal CodeBody parse_class_struct_body (Context* ctx, TokType which, Token name); internal Code parse_class_struct (Context* ctx, TokType which, bool inplace_def); internal Code parse_expression (Context* ctx); internal Code parse_forward_or_definition (Context* ctx, TokType which, bool is_inplace); internal CodeFn parse_function_after_name (Context* ctx, ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type, Token name); internal Code parse_function_body (Context* ctx); internal CodeBody parse_global_nspace (Context* ctx, CodeType which); internal Code parse_global_nspace_constructor_destructor(Context* ctx, CodeSpecifiers specifiers); internal Token parse_identifier (Context* ctx, bool* possible_member_function); internal CodeInclude parse_include (Context* ctx); internal Code parse_macro_as_definiton (Context* ctx, CodeAttributes attributes, CodeSpecifiers specifiers); internal CodeOperator parse_operator_after_ret_type (Context* ctx, ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename ret_type); internal Code parse_operator_function_or_variable(Context* ctx, bool expects_function, CodeAttributes attributes, CodeSpecifiers specifiers); internal CodePragma parse_pragma (Context* ctx); internal CodeParams parse_params (Context* ctx, bool use_template_capture); internal CodePreprocessCond parse_preprocess_cond (Context* ctx); internal Code parse_simple_preprocess (Context* ctx, TokType which); internal Code parse_static_assert (Context* ctx); internal void parse_template_args (Context* ctx, Token* token ); internal CodeVar parse_variable_after_name (Context* ctx, ModuleFlag mflags, CodeAttributes attributes, CodeSpecifiers specifiers, CodeTypename type, Str name); internal CodeVar parse_variable_declaration_list (Context* ctx); internal CodeClass parser_parse_class (Context* ctx, bool inplace_def ); internal CodeConstructor parser_parse_constructor (Context* ctx, CodeSpecifiers specifiers ); internal CodeDefine parser_parse_define (Context* ctx); internal CodeDestructor parser_parse_destructor (Context* ctx, CodeSpecifiers specifiers ); internal CodeEnum parser_parse_enum (Context* ctx, bool inplace_def ); internal CodeBody parser_parse_export_body (Context* ctx); internal CodeBody parser_parse_extern_link_body(Context* ctx); internal CodeExtern parser_parse_extern_link (Context* ctx); internal CodeFriend parser_parse_friend (Context* ctx); internal CodeFn parser_parse_function (Context* ctx); internal CodeNS parser_parse_namespace (Context* ctx); internal CodeOpCast parser_parse_operator_cast (Context* ctx, CodeSpecifiers specifiers ); internal CodeStruct parser_parse_struct (Context* ctx, bool inplace_def ); internal CodeVar parser_parse_variable (Context* ctx); internal CodeTemplate parser_parse_template (Context* ctx); internal CodeTypename parser_parse_type (Context* ctx, bool from_template, bool* is_function ); internal CodeTypedef parser_parse_typedef (Context* ctx); internal CodeUnion parser_parse_union (Context* ctx, bool inplace_def ); internal CodeUsing parser_parse_using (Context* ctx); constexpr bool parser_inplace_def = true; constexpr bool parser_not_inplace_def = false; constexpr bool parser_dont_consume_braces = true; constexpr bool parser_consume_braces = false; constexpr bool parser_not_from_template = false; constexpr bool parser_use_parenthesis = false; // Internal parsing functions constexpr bool parser_strip_formatting_dont_preserve_newlines = false; /* This function was an attempt at stripping formatting from any c++ code. It has edge case failures that prevent it from being used in function bodies. */ internal StrBuilder parser_strip_formatting(Context* ctx, Str raw_text, bool preserve_newlines ) { StrBuilder content = strbuilder_make_reserve( ctx->Allocator_Temp, raw_text.Len ); if ( raw_text.Len == 0 ) return content; #define cut_length ( scanner - raw_text.Ptr - last_cut ) #define cut_ptr ( raw_text.Ptr + last_cut ) #define pos ( rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ) ) #define move_fwd() do { scanner++; tokleft--; } while(0) s32 tokleft = raw_text.Len; sptr last_cut = 0; char const* scanner = raw_text.Ptr; if ( scanner[0] == ' ' ) { move_fwd(); last_cut = 1; } bool within_string = false; bool within_char = false; bool must_keep_newline = false; while ( tokleft ) { // Skip over the content of string literals if ( scanner[0] == '"' ) { move_fwd(); while ( tokleft && ( scanner[0] != '"' || *( scanner - 1 ) == '\\' ) ) { if ( scanner[0] == '\\' && tokleft > 1 ) { scanner += 2; tokleft -= 2; } else { move_fwd(); } } // Skip the closing " if ( tokleft ) move_fwd(); strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); last_cut = rcast(sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } // Skip over the content of character literals if ( scanner[0] == '\'' ) { move_fwd(); while ( tokleft && ( scanner[0] != '\'' || ( *(scanner -1 ) == '\\' ) ) ) { move_fwd(); } // Skip the closing ' if ( tokleft ) move_fwd(); strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } // Block comments if ( tokleft > 1 && scanner[0] == '/' && scanner[1] == '*' ) { while ( tokleft > 1 && !(scanner[0] == '*' && scanner[1] == '/') ) move_fwd(); scanner += 2; tokleft -= 2; strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } // Line comments if ( tokleft > 1 && scanner[0] == '/' && scanner[1] == '/' ) { must_keep_newline = true; scanner += 2; tokleft -= 2; while ( tokleft && scanner[ 0 ] != '\n' ) move_fwd(); if (tokleft) move_fwd(); strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } // Tabs if (scanner[0] == '\t') { if (pos > last_cut) strbuilder_append_c_str_len( & content, cut_ptr, cut_length); if ( * strbuilder_back( content ) != ' ' ) strbuilder_append_char( & content, ' ' ); move_fwd(); last_cut = rcast( sptr, scanner) - rcast( sptr, raw_text.Ptr); continue; } if ( tokleft > 1 && scanner[0] == '\r' && scanner[1] == '\n' ) { if ( must_keep_newline || preserve_newlines ) { must_keep_newline = false; scanner += 2; tokleft -= 2; strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } if ( pos > last_cut ) strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); // Replace with a space if ( * strbuilder_back( content ) != ' ' ) strbuilder_append_char( & content, ' ' ); scanner += 2; tokleft -= 2; last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } if ( scanner[0] == '\n' ) { if ( must_keep_newline || preserve_newlines ) { must_keep_newline = false; move_fwd(); strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } if ( pos > last_cut ) strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); // Replace with a space if ( * strbuilder_back( content ) != ' ' ) strbuilder_append_char( & content, ' ' ); move_fwd(); last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } // Escaped newlines if ( scanner[0] == '\\' ) { strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); s32 amount_to_skip = 1; if ( tokleft > 1 && scanner[1] == '\n' ) { amount_to_skip = 2; } else if ( tokleft > 2 && scanner[1] == '\r' && scanner[2] == '\n' ) { amount_to_skip = 3; } if ( amount_to_skip > 1 && pos == last_cut ) { scanner += amount_to_skip; tokleft -= amount_to_skip; } else move_fwd(); last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); continue; } // Consectuive spaces if ( tokleft > 1 && char_is_space( scanner[0] ) && char_is_space( scanner[ 1 ] ) ) { strbuilder_append_c_str_len( & content, cut_ptr, cut_length ); do { move_fwd(); } while ( tokleft && char_is_space( scanner[0] ) ); last_cut = rcast( sptr, scanner ) - rcast( sptr, raw_text.Ptr ); // Preserve only 1 space of formattting char* last = strbuilder_back(content); if ( last == nullptr || * last != ' ' ) strbuilder_append_char( & content, ' ' ); continue; } move_fwd(); } if ( last_cut < raw_text.Len ) { strbuilder_append_c_str_len( & content, cut_ptr, raw_text.Len - last_cut ); } #undef cut_ptr #undef cut_length #undef pos #undef move_fwd return content; } StrBuilder parser_strip_formatting_2(TokenSlice tokens) { // TODO(Ed): Use this to produce strings for validation purposes. We shouldn't serialize down from tokens once we start storing token slices for content. StrBuilder result = struct_zero(StrBuilder); return result; } internal Code parse_array_decl(Context* ctx) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); if ( check( Tok_Operator ) && currtok.Text.Ptr[0] == '[' && currtok.Text.Ptr[1] == ']' ) { Code array_expr = untyped_str( txt(" ") ); eat( Tok_Operator ); // [] parser_pop(& ctx->parser); return array_expr; } if ( check( Tok_BraceSquare_Open ) ) { eat( Tok_BraceSquare_Open ); // [ if ( left == 0 ) { log_failure( "Error, unexpected end of array declaration ( '[]' scope started )\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } if ( currtok.Type == Tok_BraceSquare_Close ) { log_failure( "Error, empty array expression in definition\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } TokenSlice tokens = { & currtok, 1 }; Token untyped_tok = currtok; while ( left && currtok.Type != Tok_BraceSquare_Close ) { eat( currtok.Type ); ++ tokens.num; } // untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; untyped_tok.Text = token_range_to_str(untyped_tok, prevtok); Code array_expr = untyped_str( untyped_tok.Text ); // Code array_expr = untyped_toks( tokens ); // TODO(Ed): Use token slice instead of untyped strings. // [ if ( left == 0 ) { log_failure( "Error, unexpected end of array declaration, expected ]\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } if ( currtok.Type != Tok_BraceSquare_Close ) { log_failure( "%s: Error, expected ] in array declaration, not %S\n%SB", toktype_to_str( currtok.Type ), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } eat( Tok_BraceSquare_Close ); // [ ] // Its a multi-dimensional array if ( check( Tok_BraceSquare_Open )) { Code adjacent_arr_expr = parse_array_decl(ctx); // [ ][ ]... array_expr->Next = adjacent_arr_expr; } parser_pop(& ctx->parser); return array_expr; } parser_pop(& ctx->parser); return NullCode; } internal inline CodeAttributes parse_attributes(Context* ctx) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope); Token start = currtok; s32 len = 0; // There can be more than one attribute. If there is flatten them to a single string. // TODO(Ed): Support chaining attributes (Use parameter linkage pattern) while ( left && tok_is_attribute(currtok) ) { if ( check( Tok_Attribute_Open ) ) { eat( Tok_Attribute_Open ); // [[ while ( left && currtok.Type != Tok_Attribute_Close ) { eat( currtok.Type ); } // [[ eat( Tok_Attribute_Close ); // [[ ]] len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )start.Text.Ptr; } else if ( check( Tok_Decl_GNU_Attribute ) ) { eat( Tok_Decl_GNU_Attribute ); eat( Tok_Paren_Open ); eat( Tok_Paren_Open ); // __attribute__(( while ( left && currtok.Type != Tok_Paren_Close ) { eat( currtok.Type ); } // __attribute__(( eat( Tok_Paren_Close ); eat( Tok_Paren_Close ); // __attribute__(( )) len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )start.Text.Ptr; } else if ( check( Tok_Decl_MSVC_Attribute ) ) { eat( Tok_Decl_MSVC_Attribute ); eat( Tok_Paren_Open ); // __declspec( while ( left && currtok.Type != Tok_Paren_Close ) { eat( currtok.Type ); } // __declspec( eat( Tok_Paren_Close ); // __declspec( ) len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )start.Text.Ptr; } else if ( tok_is_attribute(currtok) ) { eat( currtok.Type ); // // If its a macro based attribute, this could be a functional macro such as Unreal's UE_DEPRECATED(...) if ( check( Tok_Paren_Open)) { eat( Tok_Paren_Open ); s32 level = 0; while (left && currtok.Type != Tok_Paren_Close && level == 0) { if (currtok.Type == Tok_Paren_Open) ++ level; if (currtok.Type == Tok_Paren_Close) --level; eat(currtok.Type); } eat(Tok_Paren_Close); } len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )start.Text.Ptr; // ( ... ) } } if ( len > 0 ) { Str attribute_txt = { start.Text.Ptr, len }; parser_pop(& ctx->parser); StrBuilder name_stripped = parser_strip_formatting(ctx, attribute_txt, parser_strip_formatting_dont_preserve_newlines ); Code result = make_code(); result->Type = CT_PlatformAttributes; result->Name = cache_str( strbuilder_to_str(name_stripped) ); result->Content = result->Name; // result->Token = return ( CodeAttributes )result; } parser_pop(& ctx->parser); return NullCode; } internal Code parse_class_struct(Context* ctx, TokType which, bool inplace_def) { if ( which != Tok_Decl_Class && which != Tok_Decl_Struct ) { log_failure( "Error, expected class or struct, not %S\n%SB", toktype_to_str( which ), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); return InvalidCode; } Token name = NullToken; AccessSpec access = AccessSpec_Default; CodeTypename parent = { nullptr }; CodeBody body = { nullptr }; CodeAttributes attributes = { nullptr }; ModuleFlag mflags = ModuleFlag_None; Code result = InvalidCode; if ( check(Tok_Module_Export) ) { mflags = ModuleFlag_Export; eat( Tok_Module_Export ); } // eat( which ); // attributes = parse_attributes(ctx); // if ( check( Tok_Identifier ) ) { name = parse_identifier(ctx, nullptr); ctx->parser.scope->name = name.Text; } // CodeSpecifiers specifiers = NullCode; if ( check(Tok_Spec_Final)) { specifiers = def_specifier(Spec_Final); eat(Tok_Spec_Final); } // local_persist char interface_arr_mem[ kilobytes(4) ] = {0}; Array(CodeTypename) interfaces = {nullptr}; // TODO(Ed) : Make an AST_DerivedType, we'll store any arbitary derived type into there as a linear linked list of them. if ( check( Tok_Assign_Classifer ) ) { eat( Tok_Assign_Classifer ); // : if ( tok_is_access_specifier(currtok) ) { access = tok_to_access_specifier(currtok); // : eat( currtok.Type ); } Token parent_tok = parse_identifier(ctx, nullptr); parent = def_type( parent_tok.Text ); // : if (check(Tok_Comma)) { Arena arena = arena_init_from_memory( interface_arr_mem, kilobytes(4) ); interfaces = array_init_reserve(CodeTypename, arena_allocator_info(& arena), 4 ); do { eat( Tok_Comma ); // : , if ( tok_is_access_specifier(currtok) ) { eat(currtok.Type); } Token interface_tok = parse_identifier(ctx, nullptr); array_append( interfaces, def_type( interface_tok.Text ) ); // : , ... } while ( check(Tok_Comma) ); } } if ( check( Tok_BraceCurly_Open ) ) { body = parse_class_struct_body( ctx, which, name ); } // : , ... { } CodeComment inline_cmt = NullCode; if ( ! inplace_def ) { Token stmt_end = currtok; eat( Tok_Statement_End ); // : , ... { }; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(ctx); // : , ... { }; } s32 num_interfaces = scast(s32, interfaces ? array_num(interfaces) : 0); if ( which == Tok_Decl_Class ) result = cast(Code, def_class( name.Text, def_assign( body, parent, access, attributes, interfaces, num_interfaces, specifiers, mflags ) )); else result = cast(Code, def_struct( name.Text, def_assign( body, (CodeTypename)parent, access, attributes, interfaces, num_interfaces, specifiers, mflags ) )); if ( inline_cmt ) result->InlineCmt = cast(Code, inline_cmt); if (interfaces) array_free(interfaces); return result; } internal neverinline CodeBody parse_class_struct_body(Context* ctx, TokType which, Token name) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); eat( Tok_BraceCurly_Open ); // { CodeBody result = (CodeBody) make_code(); if ( which == Tok_Decl_Class ) result->Type = CT_Class_Body; else result->Type = CT_Struct_Body; scope.code_rel = cast(Code, result); while ( left && currtok_noskip.Type != Tok_BraceCurly_Close ) { ParseMessage* messages_before_member = ctx->parser.messages; s32 member_start_id = ctx->parser.token_id; Token member_start_token = currtok_noskip; Code member = Code_Invalid; CodeAttributes attributes = { nullptr }; CodeSpecifiers specifiers = { nullptr }; bool expects_function = false; // ctx->parser.Scope->Start = currtok_noskip; if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); switch ( currtok_noskip.Type ) { case Tok_Statement_End: { // TODO(Ed): Convert this to a general warning procedure log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip)); eat( Tok_Statement_End ); continue; } case Tok_NewLine: { member = fmt_newline; eat( Tok_NewLine ); break; } case Tok_Comment: { member = cast(Code, parse_comment(ctx)); break; } case Tok_Access_Public: { member = access_public; eat( Tok_Access_Public ); eat( Tok_Assign_Classifer ); // public: break; } case Tok_Access_Protected: { member = access_protected; eat( Tok_Access_Protected ); eat( Tok_Assign_Classifer ); // protected: break; } case Tok_Access_Private: { member = access_private; eat( Tok_Access_Private ); eat( Tok_Assign_Classifer ); // private: break; } case Tok_Decl_Class: { member = parse_complicated_definition(ctx, Tok_Decl_Class ); // class break; } case Tok_Decl_Enum: { member = parse_complicated_definition(ctx, Tok_Decl_Enum ); // enum break; } case Tok_Decl_Friend: { member = cast(Code, parser_parse_friend(ctx)); // friend break; } case Tok_Decl_Operator: { member = cast(Code, parser_parse_operator_cast(ctx, NullCode)); // operator () break; } case Tok_Decl_Struct: { member = parse_complicated_definition(ctx, Tok_Decl_Struct ); // struct break; } case Tok_Decl_Template: { member = cast(Code, parser_parse_template(ctx)); // template< ... > break; } case Tok_Decl_Typedef: { member = cast(Code, parser_parse_typedef(ctx)); // typedef break; } case Tok_Decl_Union: { member = parse_complicated_definition(ctx, Tok_Decl_Union ); // union break; } case Tok_Decl_Using: { member = cast(Code, parser_parse_using(ctx)); // using break; } case Tok_Operator: { //if ( currtok.Text[0] != '~' ) //{ // log_failure( "Operator token found in global body but not destructor unary negation\n%s", to_strbuilder(ctx->parser) ); // return InvalidCode; //} member = cast(Code, parser_parse_destructor(ctx, NullCode)); // ~() break; } case Tok_Preprocess_Define: { member = cast(Code, parser_parse_define(ctx)); // #define break; } case Tok_Preprocess_Include: { member = cast(Code, parse_include(ctx)); // #include break; } case Tok_Preprocess_If: case Tok_Preprocess_IfDef: case Tok_Preprocess_IfNotDef: case Tok_Preprocess_ElIf: member = cast(Code, parse_preprocess_cond(ctx)); // # break; case Tok_Preprocess_Else: { member = cast(Code, preprocess_else); eat( Tok_Preprocess_Else ); // #else break; } case Tok_Preprocess_EndIf: { member = cast(Code, preprocess_endif); eat( Tok_Preprocess_EndIf ); // #endif break; } case Tok_Preprocess_Macro_Stmt: { member = cast(Code, parse_simple_preprocess(ctx, Tok_Preprocess_Macro_Stmt )); break; } // case Tok_Preprocess_Macro: // // // macro_found = true; // goto Preprocess_Macro_Bare_In_Body; // break; case Tok_Preprocess_Pragma: { member = cast(Code, parse_pragma(ctx)); // #pragma break; } case Tok_Preprocess_Unsupported: { member = cast(Code, parse_simple_preprocess(ctx, Tok_Preprocess_Unsupported)); // # break; } case Tok_StaticAssert: { member = parse_static_assert(ctx); // static_assert break; } case Tok_Preprocess_Macro_Expr: { if ( ! tok_is_attribute(currtok)) { log_failure("Unbounded macro expression residing in class/struct body\n%S", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); return InvalidCode; } } //! Fallthrough intended case Tok_Attribute_Open: case Tok_Decl_GNU_Attribute: case Tok_Decl_MSVC_Attribute: #define Entry( attribute, str ) case attribute: GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { attributes = parse_attributes(ctx); // } //! Fallthrough intended GEN_PARSER_CLASS_STRUCT_BODY_ALLOWED_MEMBER_TOK_SPECIFIER_CASES: { Specifier specs_found[16] = { Spec_NumSpecifiers }; s32 NumSpecifiers = 0; while ( left && tok_is_specifier(currtok) ) { Specifier spec = str_to_specifier( currtok.Text ); b32 ignore_spec = false; switch ( spec ) { GEN_PARSER_CLASS_STRUCT_BODY_ALLOWED_MEMBER_SPECIFIER_CASES: break; case Spec_Consteval: expects_function = true; break; case Spec_Const : ignore_spec = true; break; default: log_failure( "Invalid specifier %S for class/struct member\n%S", spec_to_str(spec), strbuilder_to_str( parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)) ); parser_pop(& ctx->parser); return InvalidCode; } // Every specifier after would be considered part of the type type signature if (ignore_spec) break; specs_found[NumSpecifiers] = spec; NumSpecifiers++; eat( currtok.Type ); } if ( NumSpecifiers ) { specifiers = def_specifiers_arr( NumSpecifiers, specs_found ); } // if ( tok_is_attribute(currtok) ) { // Unfortuantely Unreal has code where there is attirbutes before specifiers CodeAttributes more_attributes = parse_attributes(ctx); if ( attributes ) { StrBuilder fused = strbuilder_make_reserve( ctx->Allocator_Temp, attributes->Content.Len + more_attributes->Content.Len ); strbuilder_append_fmt( & fused, "%SB %SB", attributes->Content, more_attributes->Content ); Str attrib_name = strbuilder_to_str(fused); attributes->Name = cache_str( attrib_name ); attributes->Content = attributes->Name; // } attributes = more_attributes; } if ( currtok.Type == Tok_Operator && currtok.Text.Ptr[0] == '~' ) { member = cast(Code, parser_parse_destructor(ctx, specifiers )); // ~() break; } if ( currtok.Type == Tok_Decl_Operator ) { member = cast(Code, parser_parse_operator_cast(ctx, specifiers )); // operator () break; } } //! Fallthrough intentional case Tok_Identifier: case Tok_Preprocess_Macro_Typename: case Tok_Spec_Const: case Tok_Type_Unsigned: case Tok_Type_Signed: case Tok_Type_Short: case Tok_Type_Long: case Tok_Type_bool: case Tok_Type_char: case Tok_Type_int: case Tok_Type_double: { if ( nexttok.Type == Tok_Paren_Open && name.Text.Len && currtok.Type == Tok_Identifier ) { if ( c_str_compare_len( name.Text.Ptr, currtok.Text.Ptr, name.Text.Len ) == 0 ) { member = cast(Code, parser_parse_constructor(ctx, specifiers )); // () break; } } member = parse_operator_function_or_variable(ctx, expects_function, attributes, specifiers ); // operator ... // or // ... } break; default: Token untyped_tok = currtok; while ( left && currtok.Type != Tok_BraceCurly_Close ) { untyped_tok.Text.Len = ( (sptr)currtok.Text.Ptr + currtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; eat( currtok.Type ); } member = untyped_str( untyped_tok.Text ); // Something unknown break; } b32 member_is_nested_scope = member != Code_Invalid && (member->Type == CT_Class || member->Type == CT_Struct); b32 member_failed = ctx->parser.token_mismatch || (ctx->parser.messages != messages_before_member && member_is_nested_scope == false); if (member_failed) { if ( ctx->parser.messages != messages_before_member ) ctx->parser.messages->Next = messages_before_member; ctx->parser.token_id = member_start_id; ctx->parser.token_mismatch = false; member = Code_Invalid; } if (member == Code_Invalid) { if ( member_failed == false ) { log_failure( "Failed to parse member\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } ctx->parser.token_id = member_start_id; ctx->parser.token_mismatch = false; parser_recover_body_member(ctx, result, member_start_token, member_start_id, false); continue; } body_append(result, member ); } eat( Tok_BraceCurly_Close ); // { } parser_pop(& ctx->parser); return result; } internal CodeComment parse_comment(Context* ctx) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); CodeComment result = (CodeComment) make_code(); result->Type = CT_Comment; result->Content = cache_str( currtok_noskip.Text ); // result->Token = currtok_noskip; eat( Tok_Comment ); parser_pop(& ctx->parser); return result; } internal Code parse_complicated_definition(Context* ctx, TokType which) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); b32 is_inplace = false; b32 is_fn_def = false; TokenSlice tokens = ctx->parser.tokens; s32 idx = ctx->parser.token_id; s32 level = 0; b32 had_def = false; b32 had_paren = false; for ( ; idx < tokens.num; idx++ ) { if ( tokens.ptr[ idx ].Type == Tok_BraceCurly_Open ) level++; if ( tokens.ptr[ idx ].Type == Tok_BraceCurly_Close ) { level--; had_def = level == 0; } b32 found_fn_def = had_def && had_paren; if ( level == 0 && (tokens.ptr[ idx ].Type == Tok_Statement_End || found_fn_def) ) break; } is_fn_def = had_def && had_paren; if (is_fn_def) { // Function definition with on return type Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode); // (...) ... { ... } parser_pop(& ctx->parser); return result; } if ( ( idx - 2 ) == ctx->parser.token_id ) { // It's a forward declaration only Code result = parse_forward_or_definition(ctx, which, is_inplace ); // ; parser_pop(& ctx->parser); return result; } Token tok = tokens.ptr[ idx - 1 ]; if ( tok_is_specifier(tok) && spec_is_trailing( str_to_specifier( tok.Text)) ) { // (...) ...; s32 spec_idx = idx - 1; Token spec = tokens.ptr[spec_idx]; while ( tok_is_specifier(spec) && spec_is_trailing( str_to_specifier( spec.Text)) ) { -- spec_idx; spec = tokens.ptr[spec_idx]; } if ( tokens.ptr[spec_idx].Type == Tok_Paren_Close ) { // Forward declaration with trailing specifiers for a procedure tok = tokens.ptr[spec_idx]; Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode); // , or Name> ... parser_pop(& ctx->parser); return result; } log_failure( "Unsupported or bad member definition after %S declaration\n%SB", toktype_to_str(which), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } if ( tok.Type == Tok_Identifier ) { tok = tokens.ptr[ idx - 2 ]; bool is_indirection = tok.Type == Tok_Ampersand || tok.Type == Tok_Star; bool ok_to_parse = false; if ( tok.Type == Tok_BraceCurly_Close ) { // Its an inplace definition // { ... } ; ok_to_parse = true; is_inplace = true; CodeTypename type = cast(CodeTypename, parse_forward_or_definition(ctx, which, is_inplace)); // Should be a name right after the type. Token name = parse_identifier(ctx, nullptr); ctx->parser.scope->name = name.Text; CodeVar result = parse_variable_after_name(ctx, ModuleFlag_None, NullCode, NullCode, type, name.Text); parser_pop(& ctx->parser); return (Code) result; } else if ( tok.Type == Tok_Identifier && tokens.ptr[ idx - 3 ].Type == which ) { // Its a variable with type ID using namespace. // ; ok_to_parse = true; } else if ( tok.Type == Tok_Assign_Classifer && ( ( tokens.ptr[idx - 5].Type == which && tokens.ptr[idx - 4].Type == Tok_Decl_Class ) || ( tokens.ptr[idx - 4].Type == which)) ) { // Its a forward declaration of an enum // : ; // : ; ok_to_parse = true; Code result = cast(Code, parser_parse_enum(ctx, ! parser_inplace_def)); parser_pop(& ctx->parser); return result; } else if ( is_indirection ) { // Its a indirection type with type ID using struct namespace. // * ; ok_to_parse = true; } if ( ! ok_to_parse ) { log_failure( "Unsupported or bad member definition after %S declaration\n%SB", toktype_to_str(which), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode ); // , or Name> ... parser_pop(& ctx->parser); return result; } else if ( tok.Type >= Tok_Type_Unsigned && tok.Type <= Tok_Type_MS_W64 ) { tok = tokens.ptr[ idx - 2 ]; if ( tok.Type != Tok_Assign_Classifer || ( ( tokens.ptr[idx - 5].Type != which && tokens.ptr[idx - 4].Type != Tok_Decl_Class ) && ( tokens.ptr[idx - 4].Type != which)) ) { log_failure( "Unsupported or bad member definition after %S declaration\n%SB", toktype_to_str(which), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } // Its a forward declaration of an enum class // : ; // : ; Code result = cast(Code, parser_parse_enum(ctx, ! parser_inplace_def)); parser_pop(& ctx->parser); return result; } else if ( tok.Type == Tok_BraceCurly_Close ) { // Its a definition Code result = parse_forward_or_definition(ctx, which, is_inplace ); // { ... }; parser_pop(& ctx->parser); return result; } else if ( tok.Type == Tok_BraceSquare_Close ) { // Its an array definition Code result = parse_operator_function_or_variable(ctx, false, NullCode, NullCode ); // [ ... ]; parser_pop(& ctx->parser); return result; } else { log_failure( "Unsupported or bad member definition after %S declaration\n%SB", toktype_to_str(which).Ptr, parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } } internal inline Code parse_assignment_expression(Context* ctx) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); Code expr = { nullptr }; eat( Tok_Operator ); // = Token expr_tok = currtok; if ( currtok.Type == Tok_Statement_End && currtok.Type != Tok_Comma ) { log_failure( "Expected expression after assignment operator\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } s32 level = 0; while ( left && currtok.Type != Tok_Statement_End && (currtok.Type != Tok_Comma || level > 0) ) { if (currtok.Type == Tok_BraceCurly_Open ) level++; if (currtok.Type == Tok_BraceCurly_Close ) level--; if (currtok.Type == Tok_Paren_Open) level++; else if (currtok.Type == Tok_Paren_Close) level--; eat( currtok.Type ); } if (left) { expr_tok.Text.Len = ( ( sptr )currtok.Text.Ptr + currtok.Text.Len ) - ( sptr )expr_tok.Text.Ptr - 1; } expr = untyped_str( expr_tok.Text ); // = parser_pop(& ctx->parser); return expr; } internal inline Code parse_forward_or_definition(Context* ctx, TokType which, bool is_inplace ) { Code result = InvalidCode; switch ( which ) { case Tok_Decl_Class: result = cast(Code, parser_parse_class(ctx, is_inplace )); return result; case Tok_Decl_Enum: result = cast(Code, parser_parse_enum(ctx, is_inplace )); return result; case Tok_Decl_Struct: result = cast(Code, parser_parse_struct(ctx, is_inplace )); return result; case Tok_Decl_Union: result = cast(Code, parser_parse_union(ctx, is_inplace )); return result; default: log_failure( "Error, wrong token type given to parse_complicated_definition " "(only supports class, enum, struct, union) \n%SB" , parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); return InvalidCode; } } // Function parsing is handled in multiple places because its initial signature is shared with variable parsing internal inline CodeFn parse_function_after_name(Context* ctx , ModuleFlag mflags , CodeAttributes attributes , CodeSpecifiers specifiers , CodeTypename ret_type , Token name ) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); CodeParams params = parse_params(ctx, parser_use_parenthesis); // ( ) Code suffix_specs = NullCode; // TODO(Ed), Review old comment : These have to be kept separate from the return type's specifiers. while ( left && tok_is_specifier(currtok) ) { // For Unreal's PURE_VIRTUAL Support Macro* macro = lookup_macro( currtok.Text ); if (macro && tok_is_specifier(currtok)) { suffix_specs = parse_simple_preprocess(ctx, Tok_Preprocess_Macro_Expr); continue; } if ( specifiers == nullptr ) { specifiers = def_specifier( str_to_specifier( currtok.Text) ); eat( currtok.Type ); continue; } specifiers_append(specifiers, str_to_specifier( currtok.Text) ); eat( currtok.Type ); } // ( ) // Check for trailing specifiers... CodeAttributes post_rt_attributes = parse_attributes(ctx); if (post_rt_attributes) { if (attributes) { StrBuilder merged = strbuilder_fmt_buf(ctx->Allocator_Temp, "%S %S", attributes->Content, post_rt_attributes->Content); attributes->Content = cache_str(strbuilder_to_str(merged)); } else { attributes = post_rt_attributes; } } // ( ) CodeBody body = NullCode; CodeComment inline_cmt = NullCode; if ( check( Tok_BraceCurly_Open ) ) { body = cast(CodeBody, parse_function_body(ctx)); if ( cast(Code, body) == Code_Invalid ) { parser_pop(& ctx->parser); return InvalidCode; } // ( ) { } } else if ( check(Tok_Operator) && currtok.Text.Ptr[0] == '=' ) { eat(Tok_Operator); if ( specifiers == nullptr ) { specifiers = (CodeSpecifiers) make_code(); specifiers->Type = CT_Specifiers; } if ( str_are_equal(nexttok.Text, txt("delete"))) { specifiers_append(specifiers, Spec_Delete); eat(currtok.Type); // ( ) = delete } else { specifiers_append(specifiers, Spec_Pure ); eat( Tok_Number); // ( ) = 0 } Token stmt_end = currtok; eat( Tok_Statement_End ); if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(ctx); // ( ) < = 0 or delete > ; } if (body == nullptr) { Token stmt_end = currtok; eat( Tok_Statement_End ); // ( ) < = 0 or delete > ; if ( currtok_noskip.Type == Tok_Comment && currtok_noskip.Line == stmt_end.Line ) inline_cmt = parse_comment(ctx); // ( ) < = 0 or delete > ; } StrBuilder name_stripped = strbuilder_make_str( ctx->Allocator_Temp, name.Text ); strbuilder_strip_space(name_stripped); CodeFn result = (CodeFn) make_code(); result->Name = cache_str( strbuilder_to_str(name_stripped) ); result->ModuleFlags = mflags; if ( body ) { switch ( body->Type ) { case CT_Function_Body: case CT_Untyped: break; default: { log_failure("Body must be either of Function_Body or Untyped type, %S\n%SB", code_debug_str(body), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); parser_pop(& ctx->parser); return InvalidCode; } } result->Type = CT_Function; result->Body = body; } else { result->Type = CT_Function_Fwd; } if ( attributes ) result->Attributes = attributes; if ( specifiers ) result->Specs = specifiers; if ( suffix_specs ) result->SuffixSpecs = suffix_specs; result->ReturnType = ret_type; if ( params ) result->Params = params; if ( inline_cmt ) result->InlineCmt = inline_cmt; parser_pop(& ctx->parser); return result; } internal Code parse_function_body(Context* ctx) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); eat( Tok_BraceCurly_Open ); CodeBody result = (CodeBody) make_code(); result->Type = CT_Function_Body; scope.code_rel = cast(Code, result); // TODO : Support actual parsing of function body Token start = currtok_noskip; s32 level = 0; while ( left && ( currtok_noskip.Type != Tok_BraceCurly_Close || level > 0 ) ) { if ( currtok_noskip.Type == Tok_BraceCurly_Open ) level++; else if ( currtok_noskip.Type == Tok_BraceCurly_Close && level > 0 ) level--; eat( currtok_noskip.Type ); } Token past = prevtok; s32 len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)start.Text.Ptr; if ( len > 0 ) { Str str = { start.Text.Ptr, len }; body_append( result, cast(Code, def_execution( str )) ); } eat( Tok_BraceCurly_Close ); parser_pop(& ctx->parser); return cast(Code, result); } internal s32 parser_recovery_scope_close_id(ParseContext const* parser, s32 member_start_id, b32* nested_body, b32* has_scope_close) { nested_body[0] = false; has_scope_close[0] = false; ParseStackNode* scope = parser->scope; while ( scope != nullptr ) { b32 scope_has_code = scope->code_rel != nullptr; if (scope_has_code) { CodeType type = scope->code_rel->Type; b32 is_nested_scope_body = false; is_nested_scope_body |= type == CT_Namespace_Body; is_nested_scope_body |= type == CT_Class_Body; is_nested_scope_body |= type == CT_Struct_Body; // Function-body frames have their own brace owner; inspect the enclosing body frame. if ( type == CT_Function_Body ) { scope = scope->prev; continue; } b32 nothing_to_recover = type == CT_Global_Body || is_nested_scope_body == false; // Global recovery uses the full token slice. Other unsupported scope types have no nested boundary. if (nothing_to_recover) return parser->tokens.num; // TODO(Ed): Drain to single return. // Nested recovery needs the active body's closing brace as a hard delimiter boundary. nested_body[0] = true; s32 scope_start_id = parser->tokens.num; b32 scope_has_start = scope->start != nullptr; if (scope_has_start) scope_start_id = scast(s32, scope->start - parser->tokens.ptr); // Scope start may point at formatting before the body's opening brace. s32 body_open_id = scope_start_id; while ( body_open_id < parser->tokens.num ) { b32 token_opens_body = parser->tokens.ptr[body_open_id].Type == Tok_BraceCurly_Open; if (token_opens_body == true) break; ++ body_open_id; } b32 found_body_open = body_open_id < parser->tokens.num; if (found_body_open == false) return parser->tokens.num; // A depth-zero close belongs to this body; nested closes only reduce depth. s32 body_depth = 0; for (s32 idx = body_open_id; idx < parser->tokens.num; ++ idx) { TokType token_type = parser->tokens.ptr[idx].Type; if (token_type == Tok_BraceCurly_Open) ++ body_depth; else if (token_type == Tok_BraceCurly_Close) { if (body_depth > 0) // inside nested brace { -- body_depth; if (body_depth == 0) { // found owner close has_scope_close[0] = true; // TODO(Ed): Drain to single return. return idx; } } } } s32 fallback_close_id = parser->tokens.num; if (type == CT_Class_Body || type == CT_Struct_Body) { // A malformed member opener can keep the brace-depth scan from reaching the owner close. // Direct typed entries scan to their last candidate; nested bodies stop at the first candidate. b32 has_parent_body = false; for (ParseStackNode* parent = scope->prev; parent != nullptr; parent = parent->prev) { b32 parent_has_code = parent->code_rel != nullptr; if (parent_has_code == false) continue; b32 parent_is_function_body = parent->code_rel->Type == CT_Function_Body; if (parent_is_function_body) continue; CodeType parent_type = parent->code_rel->Type; b32 parent_owns_body = false; parent_owns_body |= parent_type == CT_Global_Body; parent_owns_body |= parent_type == CT_Namespace_Body; parent_owns_body |= parent_type == CT_Class_Body; parent_owns_body |= parent_type == CT_Struct_Body; has_parent_body = parent_owns_body; break; } for (s32 idx = member_start_id; idx < parser->tokens.num; ++ idx) { if (parser->tokens.ptr[idx].Type != Tok_BraceCurly_Close) continue; s32 next_id = idx + 1; while (next_id < parser->tokens.num) { TokType next_type = parser->tokens.ptr[next_id].Type; b32 next_is_formatting = next_type == Tok_NewLine || next_type == Tok_Comment; if (next_is_formatting == false) break; ++ next_id; } b32 next_ends_statement = false; b32 close_ends_declaration = false; b32 next_in_slice = next_id < parser->tokens.num; if (next_in_slice) { next_ends_statement = parser->tokens.ptr[next_id].Type == Tok_Statement_End; close_ends_declaration = next_ends_statement; } if (close_ends_declaration) { fallback_close_id = idx; if (has_parent_body) break; } } } else { // A namespace close follows a member terminator, unlike a nested type close followed by `;`. for (s32 idx = member_start_id; idx < parser->tokens.num; ++ idx) { if (parser->tokens.ptr[idx].Type != Tok_BraceCurly_Close) continue; s32 prev_id = idx - 1; while (prev_id >= body_open_id) { TokType prev_type = parser->tokens.ptr[prev_id].Type; b32 prev_is_formatting = prev_type == Tok_NewLine || prev_type == Tok_Comment; if ( prev_is_formatting == false ) break; -- prev_id; } s32 next_id = idx + 1; while (next_id < parser->tokens.num) { TokType next_type = parser->tokens.ptr[next_id].Type; b32 next_is_formatting = next_type == Tok_NewLine || next_type == Tok_Comment; if ( next_is_formatting == false ) break; ++ next_id; } b32 prev_in_body = prev_id >= body_open_id; b32 prev_ends_member = false; if (prev_in_body) prev_ends_member = parser->tokens.ptr[prev_id].Type == Tok_Statement_End; b32 next_ends_declaration = false; b32 next_past_slice = next_id >= parser->tokens.num; if (next_past_slice == false) next_ends_declaration = parser->tokens.ptr[next_id].Type == Tok_Statement_End; b32 member_ended_before_close = false; b32 close_starts_next_declaration = false; b32 namespace_owner_close = false; if (prev_in_body && prev_ends_member) member_ended_before_close = true; if (next_past_slice == false && next_ends_declaration) close_starts_next_declaration = true; if (member_ended_before_close && close_starts_next_declaration == false) namespace_owner_close = true; if (namespace_owner_close) { fallback_close_id = idx; break; } } } if (fallback_close_id < parser->tokens.num) { // found fallback close has_scope_close[0] = true; return fallback_close_id; } return parser->tokens.num; } scope = scope->prev; } return parser->tokens.num; } internal b32 parser_recovery_delimiter_is_balanced(ParseContext const* parser , s32 start_id , TokType open_type , TokType close_type , s32 scope_close_id , b32 nested_body , b32 has_scope_close ){ // Inner delimiters shield semicolons only when their close precedes the owning body close. if ( open_type == Tok_BraceCurly_Open && nested_body && has_scope_close == false ) return false; s32 delimiter_depth = 0; s32 curly_depth = 0; for ( s32 idx = start_id; idx < scope_close_id; ++ idx ) { TokType type = parser->tokens.ptr[idx].Type; if ( type == Tok_BraceCurly_Open ) ++ curly_depth; else if ( type == Tok_BraceCurly_Close ) { if ( curly_depth == 0 ) return false; -- curly_depth; } if ( type == open_type ) ++ delimiter_depth; else if ( type == close_type && delimiter_depth > 0 ) { -- delimiter_depth; if ( delimiter_depth == 0 ) return true; } } return false; } internal void parser_recover_body_member(Context* ctx, CodeBody body, Token member_start_token, s32 member_start_id, b32 allow_comma) { // Keep the original member start for the untyped source span. Token recovery_end_token = member_start_token; b32 has_recovery_tokens = false; s32 paren_depth = 0; s32 square_depth = 0; s32 curly_depth = 0; b32 nested_body = false; b32 has_scope_close = false; // Bound delimiter matches before scanning the rejected member. s32 scope_close_id = parser_recovery_scope_close_id(& ctx->parser, member_start_id, &nested_body, &has_scope_close); while ( left ) { Token token = currtok_noskip; // Leave the active body's close for its owner. b32 is_scope_close = token.Type == Tok_BraceCurly_Close && curly_depth == 0; if (is_scope_close) break; b32 at_member_start = ctx->parser.token_id == member_start_id; b32 at_delimiter_depth_zero = paren_depth == 0 && square_depth == 0 && curly_depth == 0; b32 sync_at_comma = allow_comma && at_member_start && at_delimiter_depth_zero && token.Type == Tok_Comma; b32 sync_at_statement_end = token.Type == Tok_Statement_End && at_delimiter_depth_zero; b32 sync_at_member_boundary = sync_at_comma || sync_at_statement_end; // Semicolons sync at depth zero; only global recovery accepts a leading comma. if (sync_at_member_boundary) { TokType sync_type = Tok_Statement_End; if (sync_at_comma) sync_type = Tok_Comma; if (eat( sync_type )) { recovery_end_token = token; has_recovery_tokens = true; } break; } // Add delimiter depth only when its matching close precedes scope_close_id. switch ( token.Type ) { default: break; case Tok_Paren_Open: { b32 res = parser_recovery_delimiter_is_balanced(& ctx->parser, ctx->parser.token_id , Tok_Paren_Open, Tok_Paren_Close , scope_close_id, nested_body, has_scope_close); if (res) ++ paren_depth; break; } case Tok_Paren_Close: if ( paren_depth > 0 ) -- paren_depth; break; case Tok_BraceSquare_Open: { b32 res = parser_recovery_delimiter_is_balanced(& ctx->parser, ctx->parser.token_id , Tok_BraceSquare_Open, Tok_BraceSquare_Close , scope_close_id, nested_body, has_scope_close); if (res) ++ square_depth; break; } case Tok_BraceSquare_Close: if ( square_depth > 0 ) -- square_depth; break; case Tok_BraceCurly_Open: { b32 res = parser_recovery_delimiter_is_balanced(& ctx->parser, ctx->parser.token_id , Tok_BraceCurly_Open, Tok_BraceCurly_Close , scope_close_id, nested_body, has_scope_close); if (res) ++ curly_depth; break; } case Tok_BraceCurly_Close: if ( curly_depth > 0 ) -- curly_depth; break; } if (eat( token.Type )) { recovery_end_token = token; has_recovery_tokens = true; } } // Retain the failed source through its synchronization token as untyped code. if (has_recovery_tokens) body_append(body, untyped_str(token_range_to_str(member_start_token, recovery_end_token))); } internal neverinline CodeBody parse_global_nspace(Context* ctx, CodeType which) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); if ( which != CT_Namespace_Body && which != CT_Global_Body && which != CT_Export_Body && which != CT_Extern_Linkage_Body ) return InvalidCode; if ( which != CT_Global_Body ) eat( Tok_BraceCurly_Open ); // { CodeBody result = (CodeBody) make_code(); result->Type = which; scope.code_rel = cast(Code, result); while ( left && currtok_noskip.Type != Tok_BraceCurly_Close ) { ParseMessage* messages_before_member = ctx->parser.messages; s32 member_start_id = ctx->parser.token_id; Token member_start_token = currtok_noskip; Code member = Code_Invalid; CodeAttributes attributes = { nullptr }; CodeSpecifiers specifiers = { nullptr }; bool expects_function = false; // ctx->parser.Scope->Start = currtok_noskip; if ( currtok_noskip.Type == Tok_Preprocess_Hash ) eat( Tok_Preprocess_Hash ); b32 macro_found = false; switch ( currtok_noskip.Type ) { case Tok_Comma: { parser_record_failure(ctx, txt("Dangling comma")); if ( which != CT_Global_Body ) { parser_pop( & ctx->parser); return result; } member = Code_Invalid; } break; case Tok_Statement_End: { // TODO(Ed): Convert this to a general warning procedure log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip)); eat( Tok_Statement_End ); continue; } case Tok_NewLine: // Empty lines are auto skipped by Tokens.current() member = fmt_newline; eat( Tok_NewLine ); break; case Tok_Comment: member = cast(Code, parse_comment(ctx)); break; case Tok_Decl_Class: member = parse_complicated_definition(ctx, Tok_Decl_Class ); // class break; case Tok_Decl_Enum: member = parse_complicated_definition(ctx, Tok_Decl_Enum ); // enum break; case Tok_Decl_Extern_Linkage: if ( which == CT_Extern_Linkage_Body ) log_failure( "Nested extern linkage\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); member = cast(Code, parser_parse_extern_link(ctx)); // extern "..." { ... } break; case Tok_Decl_Namespace: member = cast(Code, parser_parse_namespace(ctx)); // namespace { ... } break; case Tok_Decl_Struct: member = parse_complicated_definition(ctx, Tok_Decl_Struct ); // struct ... break; case Tok_Decl_Template: member = cast(Code, parser_parse_template(ctx)); // template<...> ... break; case Tok_Decl_Typedef: member = cast(Code, parser_parse_typedef(ctx)); // typedef ... break; case Tok_Decl_Union: member = parse_complicated_definition(ctx, Tok_Decl_Union ); // union ... break; case Tok_Decl_Using: member = cast(Code, parser_parse_using(ctx)); // using ... break; case Tok_Preprocess_Define: member = cast(Code, parser_parse_define(ctx)); // #define ... break; case Tok_Preprocess_Include: member = cast(Code, parse_include(ctx)); // #include ... break; case Tok_Preprocess_If: case Tok_Preprocess_IfDef: case Tok_Preprocess_IfNotDef: case Tok_Preprocess_ElIf: member = cast(Code, parse_preprocess_cond(ctx)); // # ... break; case Tok_Preprocess_Else: member = cast(Code, preprocess_else); eat( Tok_Preprocess_Else ); // #else break; case Tok_Preprocess_EndIf: member = cast(Code, preprocess_endif); eat( Tok_Preprocess_EndIf ); // #endif break; case Tok_Preprocess_Macro_Stmt: { member = cast(Code, parse_simple_preprocess(ctx, Tok_Preprocess_Macro_Stmt )); break; } case Tok_Preprocess_Pragma: { member = cast(Code, parse_pragma(ctx)); // #pragma ... } break; case Tok_Preprocess_Unsupported: { member = cast(Code, parse_simple_preprocess(ctx, Tok_Preprocess_Unsupported )); // # ... } break; case Tok_StaticAssert: { member = cast(Code, parse_static_assert(ctx)); // static_assert( , ... ); } break; case Tok_Module_Export: { if ( which == CT_Export_Body ) log_failure( "Nested export declaration\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); member = cast(Code, parser_parse_export_body(ctx)); // export { ... } } break; case Tok_Module_Import: { // import ... if ( which == CT_Global_Body ) { parser_record_failure(ctx, txt("Module import parsing is not implemented")); member = Code_Invalid; break; } log_failure( "gen::%s: This function is not implemented" ); return InvalidCode; } break; case Tok_Preprocess_Macro_Expr: { if ( ! tok_is_attribute(currtok)) { if ( which == CT_Global_Body ) { parser_record_failure(ctx, txt("Unbounded macro expression in global body")); member = Code_Invalid; break; } log_failure("Unbounded macro expression residing in class/struct body\n%SB", parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); return InvalidCode; } } //! Fallthrough intentional case Tok_Attribute_Open: case Tok_Decl_GNU_Attribute: case Tok_Decl_MSVC_Attribute: #define Entry( attribute, str ) case attribute: GEN_DEFINE_ATTRIBUTE_TOKENS #undef Entry { attributes = parse_attributes(ctx); // } //! Fallthrough intentional GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_TOK_SPECIFIER_CASES: { Specifier specs_found[16] = { Spec_NumSpecifiers }; s32 NumSpecifiers = 0; b32 invalid_global_specifier = false; while ( left && tok_is_specifier(currtok) ) { Specifier spec = str_to_specifier( currtok.Text ); bool ignore_spec = false; switch ( spec ) { GEN_PARSER_CLASS_GLOBAL_NSPACE_ALLOWED_MEMBER_SPECIFIER_CASES: break; case Spec_Consteval: expects_function = true; break; case Spec_Const: ignore_spec = true; break; default: Str spec_str = spec_to_str(spec); if ( which == CT_Global_Body ) { StrBuilder message = strbuilder_fmt_buf(ctx->Allocator_Temp, "Invalid specifier %S for variable\n%SB" , spec_str , parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_record_failure(ctx, strbuilder_to_str(message)); invalid_global_specifier = true; break; } log_failure( "Invalid specifier %S for variable\n%S", spec_str, strbuilder_to_str( parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)) ); parser_pop(& ctx->parser); return InvalidCode; } if ( invalid_global_specifier ) break; if (ignore_spec) break; specs_found[NumSpecifiers] = spec; NumSpecifiers++; eat( currtok.Type ); } if ( invalid_global_specifier ) { member = Code_Invalid; break; } if ( NumSpecifiers ) { specifiers = def_specifiers_arr( NumSpecifiers, specs_found ); } // } //! Fallthrough intentional case Tok_Identifier: case Tok_Preprocess_Macro_Typename: case Tok_Spec_Const: case Tok_Type_Long: case Tok_Type_Short: case Tok_Type_Signed: case Tok_Type_Unsigned: case Tok_Type_bool: case Tok_Type_char: case Tok_Type_double: case Tok_Type_int: { // This s only in a scope so that Preprocess_Macro_Bare_In_Body works without microsoft extension warnings { Code constructor_destructor = parse_global_nspace_constructor_destructor( ctx, specifiers ); // Possible constructor implemented at global file scope. if ( constructor_destructor ) { member = constructor_destructor; break; } bool found_operator_cast_outside_class_implmentation = false; s32 idx = ctx->parser.token_id; for ( ; idx < ctx->parser.tokens.num; idx++ ) { Token tok = ctx->parser.tokens.ptr[ idx ]; if ( tok.Type == Tok_Identifier ) { idx++; tok = ctx->parser.tokens.ptr[ idx ]; if ( tok.Type == Tok_Access_StaticSymbol ) continue; break; } if ( tok.Type == Tok_Decl_Operator ) found_operator_cast_outside_class_implmentation = true; break; } if ( found_operator_cast_outside_class_implmentation ) { member = cast(Code, parser_parse_operator_cast(ctx, specifiers )); // ::operator () { ... } break; } } member = parse_operator_function_or_variable(ctx, expects_function, attributes, specifiers ); // ... } } b32 member_is_nested_scope = member != Code_Invalid && (member->Type == CT_Class || member->Type == CT_Struct || member->Type == CT_Namespace); b32 member_failed = ctx->parser.token_mismatch || (ctx->parser.messages != messages_before_member && ! member_is_nested_scope); b32 can_recover_member = which == CT_Global_Body || which == CT_Namespace_Body; if (can_recover_member && member_failed) { if (ctx->parser.messages != messages_before_member) ctx->parser.messages->Next = messages_before_member; ctx->parser.token_id = member_start_id; ctx->parser.token_mismatch = false; member = Code_Invalid; } if ( member == Code_Invalid ) { if ( which == CT_Global_Body || (which == CT_Namespace_Body && member_failed) ) { if ( ctx->parser.messages == messages_before_member ) { StrBuilder message = strbuilder_fmt_buf(ctx->Allocator_Temp, "Failed to parse member\nToken: %SB\nContext:\n%SB" , tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip) , parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_record_failure(ctx, strbuilder_to_str(message)); } parser_recover_body_member(ctx, result, member_start_token, member_start_id, which == CT_Global_Body); continue; } log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } // log_fmt("Global Body Member: %s", member->debug_str()); body_append(result, member ); } if ( which != CT_Global_Body ) eat( Tok_BraceCurly_Close ); // { } parser_pop(& ctx->parser); return result; } internal inline Code parse_global_nspace_constructor_destructor(Context* ctx, CodeSpecifiers specifiers) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); Code result = { nullptr }; /* To check if a definition is for a constructor we can go straight to the opening parenthesis for its parameters From There we work backwards to see if we come across two identifiers with the same name between an member access :: operator, there can be template parameters on the left of the :: so we ignore those. Whats important is that its back to back. This has multiple possible faults. What we parse using this method may not filter out if something has a "return type" This is bad since technically you could have a namespace nested into another namespace with the same name. If this awful pattern is done the only way to distiguish with this coarse parse is to know there is no return type defined. TODO(Ed): We could fix this by attempting to parse a type, but we would have to have a way to have it soft fail and rollback. */ TokenSlice tokens = ctx->parser.tokens; s32 idx = ctx->parser.token_id; Token nav = tokens.ptr[ idx ]; for ( ; idx < tokens.num; idx++, nav = tokens.ptr[ idx ] ) { if ( nav.Text.Ptr[0] == '<' ) { // Skip templated expressions as they mey have expressions with the () operators s32 capture_level = 0; s32 template_level = 0; for ( ; idx < tokens.num; idx++, nav = tokens.ptr[idx] ) { if (nav.Text.Ptr[ 0 ] == '<') ++ template_level; if (nav.Text.Ptr[ 0 ] == '>') -- template_level; if (nav.Type == Tok_Operator && nav.Text.Ptr[1] == '>') -- template_level; if ( nav.Type == Tok_Paren_Open) { if (template_level != 0 ) ++ capture_level; else break; } if ( template_level != 0 && nav.Type == Tok_Paren_Close) -- capture_level; } } if ( nav.Type == Tok_Paren_Open ) break; } -- idx; Token tok_right = tokens.ptr[idx]; Token tok_left = NullToken; if (tok_right.Type != Tok_Identifier) { parser_pop(& ctx->parser); // We're not dealing with a constructor if there is no identifier right before the opening of a parameter's scope. return result; } -- idx; tok_left = tokens.ptr[idx]; // ... bool possible_destructor = false; if ( tok_left.Type == Tok_Operator && tok_left.Text.Ptr[0] == '~') { possible_destructor = true; -- idx; tok_left = tokens.ptr[idx]; } if ( tok_left.Type != Tok_Access_StaticSymbol ) { parser_pop(& ctx->parser); return result; } -- idx; tok_left = tokens.ptr[idx]; // ... :: // We search toward the left until we find the next valid identifier s32 capture_level = 0; s32 template_level = 0; while ( idx != ctx->parser.token_id ) { if (tok_left.Text.Ptr[ 0 ] == '<') ++ template_level; if (tok_left.Text.Ptr[ 0 ] == '>') -- template_level; if (tok_left.Type == Tok_Operator && tok_left.Text.Ptr[1] == '>') -- template_level; if ( template_level != 0 && tok_left.Type == Tok_Paren_Open) ++ capture_level; if ( template_level != 0 && tok_left.Type == Tok_Paren_Close) -- capture_level; if ( capture_level == 0 && template_level == 0 && tok_left.Type == Tok_Identifier ) break; -- idx; tok_left = tokens.ptr[idx]; } bool is_same = c_str_compare_len( tok_right.Text.Ptr, tok_left.Text.Ptr, tok_right.Text.Len ) == 0; if (tok_left.Type == Tok_Identifier && is_same) { // We have found the pattern we desired if (possible_destructor) { // :: ~ ( result = cast(Code, parser_parse_destructor(ctx, specifiers )); } else { // :: ( result = cast(Code, parser_parse_constructor(ctx, specifiers )); } } parser_pop(& ctx->parser); return result; } // TODO(Ed): I want to eventually change the identifier to its own AST type. // This would allow distinction of the qualifier for a symbol :: // This would also allow internal Token parse_identifier(Context* ctx, bool* possible_member_function) { ParseStackNode scope = NullScope; parser_push(& ctx->parser, & scope ); Token name = currtok; ctx->parser.scope->name = name.Text; Macro* macro = lookup_macro(currtok.Text); b32 accept_as_identifier = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Allow_As_Identifier ); b32 is_decarator = macro && bitfield_is_set(MacroFlags, macro->Flags, MF_Identifier_Decorator ); // Typename can be: '::' // If that is the case first option will be Tok_Access_StaticSymbol below if (check(Tok_Identifier) || accept_as_identifier) { if (is_decarator) { Code name_macro = parse_simple_preprocess(ctx, currtok.Type); name.Text.Len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )name.Text.Ptr; } else { eat(Tok_Identifier); } } // parse_template_args(ctx, & name); //