diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 492a8ff..4009ec0 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -26,88 +26,84 @@ enum TokFlags : u32 struct Token { - char const* Text; - sptr Length; - TokType Type; - s32 Line; - s32 Column; - u32 Flags; + Str Text; + TokType Type; + s32 Line; + s32 Column; + u32 Flags; }; constexpr Token NullToken { nullptr, 0, Tok_Invalid, false, 0, TF_Null }; -AccessSpec tok_to_access_specifier(Token tok) -{ +forceinline +AccessSpec tok_to_access_specifier(Token tok) { return scast(AccessSpec, tok.Type); } -Str tok_to_str(Token tok) -{ - Str str = { tok.Text, tok.Length }; - return str; +forceinline +Str tok_to_str(Token tok) { + return tok.Text; } -bool tok_is_valid( Token tok ) -{ - return tok.Text && tok.Length && tok.Type != Tok_Invalid; +forceinline +bool tok_is_valid( Token tok ) { + return tok.Text.Ptr && tok.Text.Len && tok.Type != Tok_Invalid; } -bool tok_is_access_operator(Token tok) -{ +forceinline +bool tok_is_access_operator(Token tok) { return bitfield_is_equal( u32, tok.Flags, TF_AccessOperator ); } -bool tok_is_access_specifier(Token tok) -{ +forceinline +bool tok_is_access_specifier(Token tok) { return bitfield_is_equal( u32, tok.Flags, TF_AccessSpecifier ); } -bool tok_is_attribute(Token tok) -{ +forceinline +bool tok_is_attribute(Token tok) { return bitfield_is_equal( u32, tok.Flags, TF_Attribute ); } -bool tok_is_operator(Token tok) -{ +forceinline +bool tok_is_operator(Token tok) { return bitfield_is_equal( u32, tok.Flags, TF_Operator ); } -bool tok_is_preprocessor(Token tok) -{ +forceinline +bool tok_is_preprocessor(Token tok) { return bitfield_is_equal( u32, tok.Flags, TF_Preprocess ); } -bool tok_is_preprocess_cond(Token tok) -{ +forceinline +bool tok_is_preprocess_cond(Token tok) { return bitfield_is_equal( u32, tok.Flags, TF_Preprocess_Cond ); } -bool tok_is_specifier(Token tok) -{ +forceinline +bool tok_is_specifier(Token tok) { return bitfield_is_equal( u32, tok.Flags, TF_Specifier ); } -bool tok_is_end_definition(Token tok) -{ +forceinline +bool tok_is_end_definition(Token tok) { return bitfield_is_equal( u32, tok.Flags, TF_EndDefinition ); } StrBuilder tok_to_strbuilder(Token tok) { - StrBuilder result = strbuilder_make_reserve( GlobalAllocator, kilobytes(4) ); - - Str type_str = toktype_to_str( tok.Type ); + StrBuilder result = strbuilder_make_reserve( GlobalAllocator, kilobytes(4) ); + Str type_str = toktype_to_str( tok.Type ); strbuilder_append_fmt( & result, "Line: %d Column: %d, Type: %.*s Content: %.*s" , tok.Line, tok.Column , type_str.Len, type_str.Ptr - , tok.Length, tok.Text + , tok.Text.Len, tok.Text.Ptr ); - return result; } -struct TokArray +struct TokArray { Array(Token) Arr; s32 Idx; @@ -122,14 +118,12 @@ Token* lex_current(TokArray* self, bool skip_formatting ) while ( self->Arr[self->Idx].Type == Tok_NewLine || self->Arr[self->Idx].Type == Tok_Comment ) self->Idx++; } - return & self->Arr[self->Idx]; } Token* lex_peek(TokArray self, bool skip_formatting) { s32 idx = self.Idx; - if ( skip_formatting ) { while ( self.Arr[idx].Type == Tok_NewLine ) @@ -137,14 +131,12 @@ Token* lex_peek(TokArray self, bool skip_formatting) return & self.Arr[idx]; } - return & self.Arr[idx]; } Token* lex_previous(TokArray self, bool skip_formatting) { s32 idx = self.Idx; - if ( skip_formatting ) { while ( self.Arr[idx].Type == Tok_NewLine ) @@ -152,14 +144,12 @@ Token* lex_previous(TokArray self, bool skip_formatting) return & self.Arr[idx]; } - return & self.Arr[idx - 1]; } Token* lex_next(TokArray self, bool skip_formatting) { s32 idx = self.Idx; - if ( skip_formatting ) { while ( self.Arr[idx].Type == Tok_NewLine ) @@ -167,7 +157,6 @@ Token* lex_next(TokArray self, bool skip_formatting) return & self.Arr[idx + 1]; } - return & self.Arr[idx + 1]; } @@ -234,17 +223,17 @@ forceinline s32 lex_preprocessor_directive( LexContext* ctx ) { char const* hash = ctx->scanner; - Token hash_tok = { hash, 1, Tok_Preprocess_Hash, ctx->line, ctx->column, TF_Preprocess }; + Token hash_tok = { { hash, 1 }, Tok_Preprocess_Hash, ctx->line, ctx->column, TF_Preprocess }; array_append( Lexer_Tokens, hash_tok ); move_forward(); skip_whitespace(); - ctx->token.Text = ctx->scanner; + ctx->token.Text.Ptr = ctx->scanner; while (ctx->left && ! char_is_space((* ctx->scanner)) ) { move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; } ctx->token.Type = str_to_toktype( tok_to_str(ctx->token) ); @@ -268,18 +257,18 @@ s32 lex_preprocessor_directive( LexContext* ctx ) if ( * ctx->scanner == '\\' && ! within_string && ! within_char ) { move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; if ( (* ctx->scanner) == '\r' ) { move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; } if ( (* ctx->scanner) == '\n' ) { move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; continue; } else @@ -295,22 +284,22 @@ s32 lex_preprocessor_directive( LexContext* ctx ) if ( (* ctx->scanner) == '\r' ) { move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; } if ( (* ctx->scanner) == '\n' ) { move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; break; } move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; } - ctx->token.Length = ctx->token.Length + ctx->token.Text - hash; - ctx->token.Text = hash; + ctx->token.Text.Len = ctx->token.Text.Len + ctx->token.Text.Ptr - hash; + ctx->token.Text.Ptr = hash; array_append( Lexer_Tokens, ctx->token ); return Lex_Continue; // Skip found token, its all handled here. } @@ -333,31 +322,31 @@ s32 lex_preprocessor_directive( LexContext* ctx ) if ( ctx->token.Type == Tok_Preprocess_Define ) { - Token name = { ctx->scanner, 0, Tok_Identifier, ctx->line, ctx->column, TF_Preprocess }; + Token name = { { ctx->scanner, 0 }, Tok_Identifier, ctx->line, ctx->column, TF_Preprocess }; - name.Text = ctx->scanner; - name.Length = 1; + name.Text.Ptr = ctx->scanner; + name.Text.Len = 1; move_forward(); while ( ctx->left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) { move_forward(); - name.Length++; + name.Text.Len++; } if ( ctx->left && (* ctx->scanner) == '(' ) { move_forward(); - name.Length++; + name.Text.Len++; } array_append( Lexer_Tokens, name ); - u64 key = crc32( name.Text, name.Length ); + u64 key = crc32( name.Text.Ptr, name.Text.Len ); hashtable_set(ctx->defines, key, tok_to_str(name) ); } - Token preprocess_content = { ctx->scanner, 0, Tok_Preprocess_Content, ctx->line, ctx->column, TF_Preprocess }; + Token preprocess_content = { { ctx->scanner, 0 }, Tok_Preprocess_Content, ctx->line, ctx->column, TF_Preprocess }; if ( ctx->token.Type == Tok_Preprocess_Include ) { @@ -365,7 +354,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) if ( (* ctx->scanner) != '"' && (* ctx->scanner) != '<' ) { - StrBuilder directive_str = strbuilder_fmt_buf( GlobalAllocator, "%.*s", min( 80, ctx->left + preprocess_content.Length ), ctx->token.Text ); + StrBuilder directive_str = strbuilder_fmt_buf( GlobalAllocator, "%.*s", min( 80, ctx->left + preprocess_content.Text.Len ), ctx->token.Text.Ptr ); log_failure( "gen::Parser::lex: Expected '\"' or '<' after #include, not '%c' (%d, %d)\n%s" , (* ctx->scanner) @@ -376,16 +365,16 @@ s32 lex_preprocessor_directive( LexContext* ctx ) return Lex_ReturnNull; } move_forward(); - preprocess_content.Length++; + preprocess_content.Text.Len++; while ( ctx->left && (* ctx->scanner) != '"' && (* ctx->scanner) != '>' ) { move_forward(); - preprocess_content.Length++; + preprocess_content.Text.Len++; } move_forward(); - preprocess_content.Length++; + preprocess_content.Text.Len++; if ( (* ctx->scanner) == '\r' && ctx->scanner[1] == '\n' ) { @@ -416,24 +405,24 @@ s32 lex_preprocessor_directive( LexContext* ctx ) if ( (* ctx->scanner) == '\\' && ! within_string && ! within_char ) { move_forward(); - preprocess_content.Length++; + preprocess_content.Text.Len++; if ( (* ctx->scanner) == '\r' ) { move_forward(); - preprocess_content.Length++; + preprocess_content.Text.Len++; } if ( (* ctx->scanner) == '\n' ) { move_forward(); - preprocess_content.Length++; + preprocess_content.Text.Len++; continue; } else { - StrBuilder directive_str = strbuilder_make_length( GlobalAllocator, ctx->token.Text, ctx->token.Length ); - StrBuilder content_str = strbuilder_fmt_buf( GlobalAllocator, "%.*s", min( 400, ctx->left + preprocess_content.Length ), preprocess_content.Text ); + StrBuilder directive_str = strbuilder_make_length( GlobalAllocator, ctx->token.Text.Ptr, ctx->token.Text.Len ); + StrBuilder content_str = strbuilder_fmt_buf( GlobalAllocator, "%.*s", min( 400, ctx->left + preprocess_content.Text.Len ), preprocess_content.Text.Ptr ); log_failure( "gen::Parser::lex: Invalid escape sequence '\\%c' (%d, %d)" " in preprocessor directive '%s' (%d, %d)\n%s" @@ -457,7 +446,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) } move_forward(); - preprocess_content.Length++; + preprocess_content.Text.Len++; } array_append( Lexer_Tokens, preprocess_content ); @@ -520,9 +509,9 @@ void lex_found_token( LexContext* ctx ) u64 key = 0; if ( (* ctx->scanner) == '(') - key = crc32( ctx->token.Text, ctx->token.Length + 1 ); + key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len + 1 ); else - key = crc32( ctx->token.Text, ctx->token.Length ); + key = crc32( ctx->token.Text.Ptr, ctx->token.Text.Len ); Str* define = hashtable_get(ctx->defines, key ); if ( define ) @@ -533,7 +522,7 @@ void lex_found_token( LexContext* ctx ) if ( ctx->left && (* ctx->scanner) == '(' ) { move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; s32 level = 0; while ( ctx->left && ((* ctx->scanner) != ')' || level > 0) ) @@ -545,22 +534,22 @@ void lex_found_token( LexContext* ctx ) level--; move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; } move_forward(); - ctx->token.Length++; + ctx->token.Text.Len++; } //if ( (* ctx->scanner) == '\r' && ctx->scanner[1] == '\n' ) //{ // move_forward(); - // ctx->token.Length++; + // ctx->token..Text.Length++; //} //else if ( (* ctx->scanner) == '\n' ) //{ // move_forward(); - // ctx->token.Length++; + // ctx->token..Text.Length++; //} } else @@ -625,7 +614,7 @@ TokArray lex( Str content ) #endif { - Token thanks_c = { c.scanner, 0, Tok_Invalid, c.line, c.column, TF_Null }; + Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; c.token = thanks_c; } @@ -636,7 +625,7 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == '\r') { move_forward(); - c.token.Length = 1; + c.token.Text.Len = 1; } if ( (* ctx->scanner) == '\n' ) @@ -644,14 +633,14 @@ TokArray lex( Str content ) move_forward(); c.token.Type = Tok_NewLine; - c.token.Length++; + c.token.Text.Len++; array_append( Lexer_Tokens, c.token ); continue; } } - c.token.Length = 0; + c.token.Text.Len = 0; skip_whitespace(); if ( c.left <= 0 ) @@ -670,19 +659,19 @@ TokArray lex( Str content ) //if ( last_type == Tok_Preprocess_Pragma ) { { - Token thanks_c = { c.scanner, 0, Tok_Invalid, c.line, c.column, TF_Null }; + Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; c.token = thanks_c; } if ( (* ctx->scanner) == '\r') { move_forward(); - c.token.Length = 1; + c.token.Text.Len = 1; } if ( (* ctx->scanner) == '\n' ) { c.token.Type = Tok_NewLine; - c.token.Length++; + c.token.Text.Len++; move_forward(); array_append( Lexer_Tokens, c.token ); @@ -700,8 +689,8 @@ TokArray lex( Str content ) } case '.': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Access_MemberSymbol; c.token.Flags = TF_AccessOperator; @@ -714,9 +703,9 @@ TokArray lex( Str content ) move_forward(); if( (* ctx->scanner) == '.' ) { - c.token.Length = 3; - c.token.Type = Tok_Varadic_Argument; - c.token.Flags = TF_Null; + c.token.Text.Len = 3; + c.token.Type = Tok_Varadic_Argument; + c.token.Flags = TF_Null; move_forward(); } else @@ -731,8 +720,8 @@ TokArray lex( Str content ) } case '&' : { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Ampersand; c.token.Flags |= TF_Operator; c.token.Flags |= TF_Specifier; @@ -742,8 +731,8 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == '&' ) // && { - c.token.Length = 2; - c.token.Type = Tok_Ampersand_DBL; + c.token.Text.Len = 2; + c.token.Type = Tok_Ampersand_DBL; if (c.left) move_forward(); @@ -753,9 +742,9 @@ TokArray lex( Str content ) } case ':': { - c.token.Text = c.scanner; - c.token.Length = 1; - c.token.Type = Tok_Assign_Classifer; + Str text = { c.scanner, 1 }; + c.token.Text = text; + c.token.Type = Tok_Assign_Classifer; // Can be either a classifier (ParentType, Bitfield width), or ternary else // token.Type = Tok_Colon; @@ -765,15 +754,15 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == ':' ) { move_forward(); - c.token.Type = Tok_Access_StaticSymbol; - c.token.Length++; + c.token.Type = Tok_Access_StaticSymbol; + c.token.Text.Len++; } goto FoundToken; } case '{': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_BraceCurly_Open; if (c.left) @@ -782,8 +771,8 @@ TokArray lex( Str content ) } case '}': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_BraceCurly_Close; c.token.Flags = TF_EndDefinition; @@ -795,8 +784,8 @@ TokArray lex( Str content ) } case '[': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_BraceSquare_Open; if ( c.left ) { @@ -804,8 +793,8 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == ']' ) { - c.token.Length = 2; - c.token.Type = Tok_Operator; + c.token.Text.Len = 2; + c.token.Type = Tok_Operator; move_forward(); } } @@ -813,8 +802,8 @@ TokArray lex( Str content ) } case ']': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_BraceSquare_Close; if (c.left) @@ -823,8 +812,8 @@ TokArray lex( Str content ) } case '(': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Capture_Start; if (c.left) @@ -833,8 +822,8 @@ TokArray lex( Str content ) } case ')': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Capture_End; if (c.left) @@ -843,8 +832,8 @@ TokArray lex( Str content ) } case '\'': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Char; c.token.Flags = TF_Literal; @@ -853,32 +842,32 @@ TokArray lex( Str content ) if ( c.left && (* ctx->scanner) == '\\' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; if ( (* ctx->scanner) == '\'' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } } while ( c.left && (* ctx->scanner) != '\'' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } if ( c.left ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } goto FoundToken; } case ',': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Comma; c.token.Flags = TF_Operator; @@ -888,8 +877,8 @@ TokArray lex( Str content ) } case '*': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Star; c.token.Flags |= TF_Specifier; c.token.Flags |= TF_Operator; @@ -899,7 +888,7 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == '=' ) { - c.token.Length++; + c.token.Text.Len++; c.token.Flags |= TF_Assign; // c.token.Type = Tok_Assign_Multiply; @@ -911,8 +900,8 @@ TokArray lex( Str content ) } case ';': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Statement_End; c.token.Flags = TF_EndDefinition; @@ -924,8 +913,8 @@ TokArray lex( Str content ) } case '"': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_String; c.token.Flags |= TF_Literal; @@ -941,25 +930,25 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == '\\' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; if ( c.left ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } continue; } move_forward(); - c.token.Length++; + c.token.Text.Len++; } goto FoundToken; } case '?': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Operator; // c.token.Type = Tok_Ternary; c.token.Flags = TF_Operator; @@ -971,8 +960,8 @@ TokArray lex( Str content ) } case '=': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Operator; // c.token.Type = Tok_Assign; c.token.Flags = TF_Operator; @@ -983,7 +972,7 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == '=' ) { - c.token.Length++; + c.token.Text.Len++; c.token.Flags = TF_Operator; if (c.left) @@ -1027,8 +1016,8 @@ TokArray lex( Str content ) } case '|': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Operator; c.token.Flags = TF_Operator; // token.Type = Tok_L_Or; @@ -1038,7 +1027,7 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == '=' ) { - c.token.Length++; + c.token.Text.Len++; c.token.Flags |= TF_Assign; // token.Flags |= TokFlags::Assignment; // token.Type = Tok_Assign_L_Or; @@ -1046,9 +1035,9 @@ TokArray lex( Str content ) if (c.left) move_forward(); } - else while ( c.left && (* ctx->scanner) == *(c.scanner - 1) && c.token.Length < 3 ) + else while ( c.left && (* ctx->scanner) == *(c.scanner - 1) && c.token.Text.Len < 3 ) { - c.token.Length++; + c.token.Text.Len++; if (c.left) move_forward(); @@ -1059,8 +1048,8 @@ TokArray lex( Str content ) // Dash is unfortunatlly a bit more complicated... case '-': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Operator; // token.Type = Tok_Subtract; c.token.Flags = TF_Operator; @@ -1070,7 +1059,7 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == '>' ) { - c.token.Length++; + c.token.Text.Len++; // token.Type = Tok_Access_PointerToMemberSymbol; c.token.Flags |= TF_AccessOperator; move_forward(); @@ -1078,22 +1067,22 @@ TokArray lex( Str content ) if ( (* ctx->scanner) == '*' ) { // token.Type = Tok_Access_PointerToMemberOfPointerSymbol; - c.token.Length++; + c.token.Text.Len++; move_forward(); } } else if ( (* ctx->scanner) == '=' ) { - c.token.Length++; + c.token.Text.Len++; // token.Type = Tok_Assign_Subtract; c.token.Flags |= TF_Assign; if (c.left) move_forward(); } - else while ( c.left && (* ctx->scanner) == *(c.scanner - 1) && c.token.Length < 3 ) + else while ( c.left && (* ctx->scanner) == *(c.scanner - 1) && c.token.Text.Len < 3 ) { - c.token.Length++; + c.token.Text.Len++; if (c.left) move_forward(); @@ -1103,8 +1092,8 @@ TokArray lex( Str content ) } case '/': { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Operator; // token.Type = Tok_Divide; c.token.Flags = TF_Operator; @@ -1116,40 +1105,40 @@ TokArray lex( Str content ) { // token.Type = TokeType::Assign_Divide; move_forward(); - c.token.Length++; + c.token.Text.Len++; c.token.Flags = TF_Assign; } else if ( (* ctx->scanner) == '/' ) { - c.token.Type = Tok_Comment; - c.token.Length = 2; - c.token.Flags = TF_Null; + c.token.Type = Tok_Comment; + c.token.Text.Len = 2; + c.token.Flags = TF_Null; move_forward(); while ( c.left && (* ctx->scanner) != '\n' && (* ctx->scanner) != '\r' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } if ( (* ctx->scanner) == '\r' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } if ( (* ctx->scanner) == '\n' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } array_append( Lexer_Tokens, c.token ); continue; } else if ( (* ctx->scanner) == '*' ) { - c.token.Type = Tok_Comment; - c.token.Length = 2; - c.token.Flags = TF_Null; + c.token.Type = Tok_Comment; + c.token.Text.Len = 2; + c.token.Flags = TF_Null; move_forward(); bool star = (* ctx->scanner) == '*'; @@ -1158,25 +1147,25 @@ TokArray lex( Str content ) while ( c.left && ! at_end ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; star = (* ctx->scanner) == '*'; slash = c.scanner[1] == '/'; at_end = star && slash; } - c.token.Length += 2; + c.token.Text.Len += 2; move_forward(); move_forward(); if ( (* ctx->scanner) == '\r' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } if ( (* ctx->scanner) == '\n' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } array_append( Lexer_Tokens, c.token ); // end_line(); @@ -1189,14 +1178,14 @@ TokArray lex( Str content ) if ( char_is_alpha( (* ctx->scanner) ) || (* ctx->scanner) == '_' ) { - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; move_forward(); while ( c.left && ( char_is_alphanumeric((* ctx->scanner)) || (* ctx->scanner) == '_' ) ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } goto FoundToken; @@ -1205,8 +1194,8 @@ TokArray lex( Str content ) { // This is a very brute force lex, no checks are done for validity of literal. - c.token.Text = c.scanner; - c.token.Length = 1; + Str text = { c.scanner, 1 }; + c.token.Text = text; c.token.Type = Tok_Number; c.token.Flags = TF_Literal; move_forward(); @@ -1218,12 +1207,12 @@ TokArray lex( Str content ) ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; while ( c.left && char_is_hex_digit((* ctx->scanner)) ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } goto FoundToken; @@ -1232,18 +1221,18 @@ TokArray lex( Str content ) while ( c.left && char_is_digit((* ctx->scanner)) ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } if ( c.left && (* ctx->scanner) == '.' ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; while ( c.left && char_is_digit((* ctx->scanner)) ) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } // Handle number literal suffixes in a botched way @@ -1256,13 +1245,13 @@ TokArray lex( Str content ) { char prev = (* ctx->scanner); move_forward(); - c.token.Length++; + c.token.Text.Len++; // Handle 'll'/'LL' as a special case when we just processed an 'l'/'L' if (c.left && (prev == 'l' || prev == 'L') && ((* ctx->scanner) == 'l' || (* ctx->scanner) == 'L')) { move_forward(); - c.token.Length++; + c.token.Text.Len++; } } } @@ -1278,7 +1267,7 @@ TokArray lex( Str content ) log_fmt( "Token %d Type: %s : %.*s\n" , idx , toktype_to_str( Lexer_Tokens[ idx ].Type ).Ptr - , Lexer_Tokens[ idx ].Length, Lexer_Tokens[ idx ].Text + , Lexer_Tokens[ idx ].Text.Len, Lexer_Tokens[ idx ].Text.Ptr ); } @@ -1298,18 +1287,18 @@ TokArray lex( Str content ) TokType last_type = array_back(Lexer_Tokens)->Type; if ( last_type == Tok_Preprocess_Macro ) { - Token thanks_c = { c.scanner, 0, Tok_Invalid, c.line, c.column, TF_Null }; + Token thanks_c = { { c.scanner, 0 }, Tok_Invalid, c.line, c.column, TF_Null }; c.token = thanks_c; if ( (* ctx->scanner) == '\r') { move_forward(); - c.token.Length = 1; + c.token.Text.Len = 1; } if ( (* ctx->scanner) == '\n' ) { c.token.Type = Tok_NewLine; - c.token.Length++; + c.token.Text.Len++; move_forward(); array_append( Lexer_Tokens, c.token ); diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 671afea..aad9116 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -52,23 +52,23 @@ StrBuilder parser_to_strbuilder(ParseContext ctx) Token scope_start = ctx.Scope->Start; Token last_valid = ctx.Tokens.Idx >= array_num(ctx.Tokens.Arr) ? ctx.Tokens.Arr[array_num(ctx.Tokens.Arr) -1] : (* lex_current(& ctx.Tokens, true)); - sptr length = scope_start.Length; - char const* current = scope_start.Text + length; - while ( current <= array_back( ctx.Tokens.Arr)->Text && *current != '\n' && length < 74 ) + sptr length = scope_start.Text.Len; + char const* current = scope_start.Text.Ptr + length; + while ( current <= array_back( ctx.Tokens.Arr)->Text.Ptr && (* current) != '\n' && length < 74 ) { current++; length++; } - Str scope_str = { scope_start.Text, length }; + Str scope_str = { scope_start.Text.Ptr, length }; StrBuilder line = strbuilder_make_str( GlobalAllocator, scope_str ); strbuilder_append_fmt( & result, "\tScope : %s\n", line ); strbuilder_free(& line); - sptr dist = (sptr)last_valid.Text - (sptr)scope_start.Text + 2; + sptr dist = (sptr)last_valid.Text.Ptr - (sptr)scope_start.Text.Ptr + 2; sptr length_from_err = dist; - Str err_str = { last_valid.Text, length_from_err }; + Str err_str = { last_valid.Text.Ptr, length_from_err }; StrBuilder line_from_err = strbuilder_make_str( GlobalAllocator, err_str ); if ( length_from_err < 100 ) @@ -82,7 +82,7 @@ StrBuilder parser_to_strbuilder(ParseContext ctx) { if ( tok_is_valid(curr_scope->Name) ) { - strbuilder_append_fmt(& result, "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Length, curr_scope->Name.Text ); + strbuilder_append_fmt(& result, "\t%d: %s, AST Name: %.*s\n", level, curr_scope->ProcName.Ptr, curr_scope->Name.Text.Len, curr_scope->Name.Text.Ptr ); } else { @@ -119,7 +119,7 @@ bool lex__eat(TokArray* self, TokType type ) Token tok = * lex_current( self, lex_skip_formatting ); log_failure( "Parse Error, TokArray::eat, Expected: ' %s ' not ' %.*s ' (%d, %d)`\n%s" , toktype_to_str(type).Ptr - , at_idx.Length, at_idx.Text + , at_idx.Text.Len, at_idx.Text.Ptr , tok.Line , tok.Column , parser_to_strbuilder(Context) @@ -513,7 +513,7 @@ Code parse_array_decl() { push_scope(); - if ( check( Tok_Operator ) && currtok.Text[0] == '[' && currtok.Text[1] == ']' ) + if ( check( Tok_Operator ) && currtok.Text.Ptr[0] == '[' && currtok.Text.Ptr[1] == ']' ) { Code array_expr = untyped_str( txt(" ") ); eat( Tok_Operator ); @@ -549,7 +549,7 @@ Code parse_array_decl() eat( currtok.Type ); } - untyped_tok.Length = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)untyped_tok.Text; + untyped_tok.Text.Len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; Code array_expr = untyped_str( tok_to_str(untyped_tok) ); // [ @@ -614,7 +614,7 @@ CodeAttributes parse_attributes() eat( Tok_Attribute_Close ); // [[ ]] - len = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )start.Text; + len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )start.Text.Ptr; } else if ( check( Tok_Decl_GNU_Attribute ) ) { @@ -633,7 +633,7 @@ CodeAttributes parse_attributes() eat( Tok_Capture_End ); // __attribute__(( )) - len = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )start.Text; + len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )start.Text.Ptr; } else if ( check( Tok_Decl_MSVC_Attribute ) ) { @@ -650,7 +650,7 @@ CodeAttributes parse_attributes() eat( Tok_Capture_End ); // __declspec( ) - len = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )start.Text; + len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )start.Text.Ptr; } else if ( tok_is_attribute(currtok) ) { @@ -674,14 +674,14 @@ CodeAttributes parse_attributes() eat(Tok_Capture_End); } - len = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )start.Text; + len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )start.Text.Ptr; // ( ... ) } } if ( len > 0 ) { - Str attribute_txt = { start.Text, len }; + Str attribute_txt = { start.Text.Ptr, len }; parser_pop(& Context); StrBuilder name_stripped = parser_strip_formatting( attribute_txt, parser_strip_formatting_dont_preserve_newlines ); @@ -1081,7 +1081,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) attributes = more_attributes; } - if ( currtok.Type == Tok_Operator && currtok.Text[0] == '~' ) + if ( currtok.Type == Tok_Operator && currtok.Text.Ptr[0] == '~' ) { member = cast(Code, parser_parse_destructor( specifiers )); // ~() @@ -1107,9 +1107,9 @@ CodeBody parse_class_struct_body( TokType which, Token name ) case Tok_Type_int: case Tok_Type_double: { - if ( nexttok.Type == Tok_Capture_Start && name.Length && currtok.Type == Tok_Identifier ) + if ( nexttok.Type == Tok_Capture_Start && name.Text.Len && currtok.Type == Tok_Identifier ) { - if ( c_str_compare_len( name.Text, currtok.Text, name.Length ) == 0 ) + if ( c_str_compare_len( name.Text.Ptr, currtok.Text.Ptr, name.Text.Len ) == 0 ) { member = cast(Code, parser_parse_constructor( specifiers )); // () @@ -1151,7 +1151,7 @@ CodeBody parse_class_struct_body( TokType which, Token name ) while ( left && currtok.Type != Tok_BraceCurly_Close ) { - untyped_tok.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)untyped_tok.Text; + untyped_tok.Text.Len = ( (sptr)currtok.Text.Ptr + currtok.Text.Len ) - (sptr)untyped_tok.Text.Ptr; eat( currtok.Type ); } @@ -1379,7 +1379,7 @@ CodeDefine parse_define() return InvalidCode; } - if ( currtok.Length == 0 ) + if ( currtok.Text.Len == 0 ) { define->Content = get_cached_string( tok_to_str(currtok) ); eat( Tok_Preprocess_Content ); @@ -1429,8 +1429,8 @@ Code parse_assignment_expression() eat( currtok.Type ); } - expr_tok.Length = ( ( sptr )currtok.Text + currtok.Length ) - ( sptr )expr_tok.Text - 1; - expr = untyped_str( tok_to_str(expr_tok) ); + expr_tok.Text.Len = ( ( sptr )currtok.Text.Ptr + currtok.Text.Len ) - ( sptr )expr_tok.Text.Ptr - 1; + expr = untyped_str( tok_to_str(expr_tok) ); // = return expr; } @@ -1508,7 +1508,7 @@ CodeFn parse_function_after_name( } // ( ) { } } - else if ( check(Tok_Operator) && currtok.Text[0] == '=' ) + else if ( check(Tok_Operator) && currtok.Text.Ptr[0] == '=' ) { eat(Tok_Operator); specifiers_append(specifiers, Spec_Pure ); @@ -1612,11 +1612,11 @@ Code parse_function_body() Token past = prevtok; - s32 len = ( (sptr)prevtok.Text + prevtok.Length ) - (sptr)start.Text; + s32 len = ( (sptr)prevtok.Text.Ptr + prevtok.Text.Len ) - (sptr)start.Text.Ptr; if ( len > 0 ) { - Str str = { start.Text, len }; + Str str = { start.Text.Ptr, len }; body_append( result, cast(Code, def_execution( str )) ); } @@ -2000,19 +2000,19 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) Token nav = tokens.Arr[ idx ]; for ( ; idx < array_num(tokens.Arr); idx++, nav = tokens.Arr[ idx ] ) { - if ( nav.Text[0] == '<' ) + if ( nav.Text.Ptr[0] == '<' ) { // Skip templated expressions as they mey have expressions with the () operators s32 capture_level = 0; s32 template_level = 0; for ( ; idx < array_num(tokens.Arr); idx++, nav = tokens.Arr[idx] ) { - if (nav.Text[ 0 ] == '<') + if (nav.Text.Ptr[ 0 ] == '<') ++ template_level; - if (nav.Text[ 0 ] == '>') + if (nav.Text.Ptr[ 0 ] == '>') -- template_level; - if (nav.Type == Tok_Operator && nav.Text[1] == '>') + if (nav.Type == Tok_Operator && nav.Text.Ptr[1] == '>') -- template_level; if ( nav.Type == Tok_Capture_Start) @@ -2047,7 +2047,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) // ... bool possible_destructor = false; - if ( tok_left.Type == Tok_Operator && tok_left.Text[0] == '~') + if ( tok_left.Type == Tok_Operator && tok_left.Text.Ptr[0] == '~') { possible_destructor = true; -- idx; @@ -2066,12 +2066,12 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) s32 template_level = 0; while ( idx != tokens.Idx ) { - if (tok_left.Text[ 0 ] == '<') + if (tok_left.Text.Ptr[ 0 ] == '<') ++ template_level; - if (tok_left.Text[ 0 ] == '>') + if (tok_left.Text.Ptr[ 0 ] == '>') -- template_level; - if (tok_left.Type == Tok_Operator && tok_left.Text[1] == '>') + if (tok_left.Type == Tok_Operator && tok_left.Text.Ptr[1] == '>') -- template_level; if ( template_level != 0 && tok_left.Type == Tok_Capture_Start) @@ -2087,7 +2087,7 @@ Code parse_global_nspace_constructor_destructor( CodeSpecifiers specifiers ) tok_left = tokens.Arr[idx]; } - bool is_same = c_str_compare_len( tok_right.Text, tok_left.Text, tok_right.Length ) == 0; + bool is_same = c_str_compare_len( tok_right.Text.Ptr, tok_left.Text.Ptr, tok_right.Text.Len ) == 0; if (tok_left.Type == Tok_Identifier && is_same) { // We have found the pattern we desired @@ -2134,12 +2134,12 @@ Token parse_identifier( bool* possible_member_function ) return invalid; } - if ( currtok.Type == Tok_Operator && currtok.Text[0] == '~' ) + if ( currtok.Type == Tok_Operator && currtok.Text.Ptr[0] == '~' ) { bool is_destructor = str_are_equal( Context.Scope->Prev->ProcName, txt("parser_parse_destructor")); if (is_destructor) { - name.Length = ( ( sptr )prevtok.Text + prevtok.Length ) - ( sptr )name.Text; + name.Text.Len = ( ( sptr )prevtok.Text.Ptr + prevtok.Text.Len ) - ( sptr )name.Text.Ptr; parser_pop(& Context); return name; } @@ -2149,7 +2149,7 @@ Token parse_identifier( bool* possible_member_function ) return invalid; } - if ( currtok.Type == Tok_Operator && currtok.Text[0] == '*' && currtok.Length == 1 ) + if ( currtok.Type == Tok_Operator && currtok.Text.Ptr[0] == '*' && currtok.Text.Len == 1 ) { if ( possible_member_function ) *possible_member_function = true; @@ -2169,7 +2169,7 @@ Token parse_identifier( bool* possible_member_function ) return invalid; } - name.Length = ( (sptr)currtok.Text + currtok.Length ) - (sptr)name.Text; + name.Text.Len = ( (sptr)currtok.Text.Ptr + currtok.Text.Len ) - (sptr)name.Text.Ptr; eat( Tok_Identifier ); //