From 822a3b07dc3efd667f1b5908373e36c9d44a64b8 Mon Sep 17 00:00:00 2001 From: Ed_ Date: Sun, 15 Jun 2025 17:56:26 -0400 Subject: [PATCH] misc changes (technically, old was never pushed...) --- .gitignore | 2 +- base/components/code_types.hpp | 2 +- base/components/lexer.cpp | 25 +++++++++++++------------ base/components/parser.cpp | 8 ++++---- base/components/parser_types.hpp | 3 ++- scripts/helpers/vendor_toolchain.ps1 | 2 ++ 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 7202e1c..3aaaa91 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,4 @@ test/cpp_library/gen !scripts/helpers/refactor.exe -ai/** +# ai/** diff --git a/base/components/code_types.hpp b/base/components/code_types.hpp index a394d1e..59b8a77 100644 --- a/base/components/code_types.hpp +++ b/base/components/code_types.hpp @@ -1076,7 +1076,7 @@ forceinline void to_strbuilder(CodeParams params, StrBuilder& result ) { forceinline bool append (CodeSpecifiers specifiers, Specifier spec) { return specifiers_append(specifiers, spec); } forceinline bool has (CodeSpecifiers specifiers, Specifier spec) { return specifiers_has(specifiers, spec); } -forceinline s32 specifiers_index_of(CodeSpecifiers specifiers, Specifier spec) { return specifiers_index_of(specifiers, spec); } +forceinline s32 index_of (CodeSpecifiers specifiers, Specifier spec) { return specifiers_index_of(specifiers, spec); } forceinline s32 remove (CodeSpecifiers specifiers, Specifier to_remove ) { return specifiers_remove(specifiers, to_remove); } forceinline StrBuilder to_strbuilder (CodeSpecifiers specifiers) { return specifiers_to_strbuilder(specifiers); } forceinline void to_strbuilder (CodeSpecifiers specifiers, StrBuilder& result) { return specifiers_to_strbuilder_ref(specifiers, & result); } diff --git a/base/components/lexer.cpp b/base/components/lexer.cpp index 3014041..6c74241 100644 --- a/base/components/lexer.cpp +++ b/base/components/lexer.cpp @@ -4,9 +4,9 @@ #include "gen/etoktype.hpp" #endif -StrBuilder tok_to_strbuilder(Token tok) +StrBuilder tok_to_strbuilder(AllocatorInfo ainfo, Token tok) { - StrBuilder result = strbuilder_make_reserve( _ctx->Allocator_Temp, kilobytes(4) ); + StrBuilder result = strbuilder_make_reserve( ainfo, kilobytes(4) ); Str type_str = toktype_to_str( tok.Type ); strbuilder_append_fmt( & result, "Line: %d Column: %d, Type: %.*s Content: %.*s" @@ -379,7 +379,7 @@ s32 lex_preprocessor_directive( LexContext* ctx ) if ( (* ctx->scanner) != '"' && (* ctx->scanner) != '<' ) { - StrBuilder directive_str = strbuilder_fmt_buf( _ctx->Allocator_Temp, "%.*s", min( 80, ctx->left + preprocess_content.Text.Len ), ctx->token.Text.Ptr ); + StrBuilder directive_str = strbuilder_fmt_buf( ctx->allocator_temp, "%.*s", min( 80, ctx->left + preprocess_content.Text.Len ), ctx->token.Text.Ptr ); log_failure( "gen::Parser::lex: Expected '\"' or '<' after #include, not '%c' (%d, %d)\n%s" , (* ctx->scanner) @@ -446,8 +446,8 @@ s32 lex_preprocessor_directive( LexContext* ctx ) } else { - StrBuilder directive_str = strbuilder_make_length( _ctx->Allocator_Temp, ctx->token.Text.Ptr, ctx->token.Text.Len ); - StrBuilder content_str = strbuilder_fmt_buf( _ctx->Allocator_Temp, "%.*s", min( 400, ctx->left + preprocess_content.Text.Len ), preprocess_content.Text.Ptr ); + StrBuilder directive_str = strbuilder_make_length( ctx->allocator_temp, ctx->token.Text.Ptr, ctx->token.Text.Len ); + StrBuilder content_str = strbuilder_fmt_buf( ctx->allocator_temp, "%.*s", min( 400, ctx->left + preprocess_content.Text.Len ), preprocess_content.Text.Ptr ); log_failure( "gen::Parser::lex: Invalid escape sequence '\\%c' (%d, %d)" " in preprocessor directive '%s' (%d, %d)\n%s" @@ -574,12 +574,13 @@ LexedInfo lex(Context* lib_ctx, Str content) LexedInfo info = struct_zero(LexedInfo); LexContext c = struct_zero(LexContext); LexContext* ctx = & c; - c.content = content; - c.left = content.Len; - c.scanner = content.Ptr; - c.line = 1; - c.column = 1; - c.tokens = array_init_reserve(Token, lib_ctx->Allocator_DyanmicContainers, lib_ctx->InitSize_LexerTokens ); + c.allocator_temp = lib_ctx->Allocator_Temp; + c.content = content; + c.left = content.Len; + c.scanner = content.Ptr; + c.line = 1; + c.column = 1; + c.tokens = array_init_reserve(Token, lib_ctx->Allocator_DyanmicContainers, lib_ctx->InitSize_LexerTokens ); // TODO(Ed): Re-implement to new constraints: // 1. Ability to continue on error @@ -688,7 +689,7 @@ LexedInfo lex(Context* lib_ctx, Str content) } else { - StrBuilder context_str = strbuilder_fmt_buf( _ctx->Allocator_Temp, "%s", c.scanner, min( 100, c.left ) ); + StrBuilder context_str = strbuilder_fmt_buf( lib_ctx->Allocator_Temp, "%s", c.scanner, min( 100, c.left ) ); log_failure( "gen::lex: invalid varadic argument, expected '...' got '..%c' (%d, %d)\n%s", (* ctx->scanner), c.line, c.column, context_str ); } diff --git a/base/components/parser.cpp b/base/components/parser.cpp index 58ed1a4..254353e 100644 --- a/base/components/parser.cpp +++ b/base/components/parser.cpp @@ -841,7 +841,7 @@ CodeBody parse_class_struct_body(Context* ctx, TokType which, Token name) { case Tok_Statement_End: { // TODO(Ed): Convert this to a general warning procedure - log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(currtok_noskip)); + log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip)); eat( Tok_Statement_End ); continue; } @@ -1669,7 +1669,7 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which) { case Tok_Comma: { - log_failure("Dangling comma found: %SB\nContext:\n%SB", tok_to_strbuilder(currtok), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); + log_failure("Dangling comma found: %SB\nContext:\n%SB", tok_to_strbuilder(ctx->Allocator_Temp, currtok), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp)); parser_pop( & ctx->parser); return InvalidCode; } @@ -1677,7 +1677,7 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which) case Tok_Statement_End: { // TODO(Ed): Convert this to a general warning procedure - log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(currtok_noskip)); + log_fmt("Dangling end statement found %SB\n", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip)); eat( Tok_Statement_End ); continue; } @@ -1935,7 +1935,7 @@ CodeBody parse_global_nspace(Context* ctx, CodeType which) if ( member == Code_Invalid ) { - log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(currtok_noskip), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); + log_failure( "Failed to parse member\nToken: %SB\nContext:\n%SB", tok_to_strbuilder(ctx->Allocator_Temp, currtok_noskip), parser_to_strbuilder(& ctx->parser, ctx->Allocator_Temp) ); parser_pop(& ctx->parser); return InvalidCode; } diff --git a/base/components/parser_types.hpp b/base/components/parser_types.hpp index 1a8f5a4..492310c 100644 --- a/base/components/parser_types.hpp +++ b/base/components/parser_types.hpp @@ -89,7 +89,7 @@ bool tok_is_end_definition(Token tok) { return bitfield_is_set( u32, tok.Flags, TF_EndDefinition ); } -StrBuilder tok_to_strbuilder(Token tok); +StrBuilder tok_to_strbuilder(AllocatorInfo ainfo, Token tok); struct TokenSlice { @@ -128,6 +128,7 @@ struct LexerMessage struct LexContext { + AllocatorInfo allocator_temp; LexerMessage* messages; Str content; s32 left; diff --git a/scripts/helpers/vendor_toolchain.ps1 b/scripts/helpers/vendor_toolchain.ps1 index 9d5f005..c3c7e96 100644 --- a/scripts/helpers/vendor_toolchain.ps1 +++ b/scripts/helpers/vendor_toolchain.ps1 @@ -132,6 +132,7 @@ if ( $vendor -match "clang" ) $flag_all_c = @('-x', 'c') $flag_c11 = '-std=c11' $flag_all_cpp = '-x c++' + $flag_charset_utf8 = '-utf-8' $flag_compile = '-c' $flag_color_diagnostics = '-fcolor-diagnostics' $flag_no_color_diagnostics = '-fno-color-diagnostics' @@ -371,6 +372,7 @@ if ( $vendor -match "msvc" ) $flag_all_c = '/TC' $flag_c11 = '/std:c11' $flag_all_cpp = '/TP' + $flag_charset_utf8 = '/utf-8' $flag_compile = '/c' $flag_debug = '/Zi' $flag_define = '/D'