From 9ed36445b924c028acf72854e6ca735abf5838cc Mon Sep 17 00:00:00 2001 From: Abdelrahman Farid Date: Tue, 19 Sep 2023 22:18:23 +0300 Subject: [PATCH 1/4] Add test for utf8 multibyte strings --- tests/core/encoding/json/test_core_json.odin | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/core/encoding/json/test_core_json.odin b/tests/core/encoding/json/test_core_json.odin index 937d1c738..23361d694 100644 --- a/tests/core/encoding/json/test_core_json.odin +++ b/tests/core/encoding/json/test_core_json.odin @@ -33,6 +33,7 @@ main :: proc() { marshal_json(&t) unmarshal_json(&t) surrogate(&t) + utf8_string_of_multibyte_characters(&t) fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) if TEST_fail > 0 { @@ -359,3 +360,10 @@ surrogate :: proc(t: ^testing.T) { expect(t, uerr == nil, fmt.tprintf("Expected `json.unmarshal(%q)` to return a nil error, got %v", string(out), uerr)) expect(t, back == input, fmt.tprintf("Expected `json.unmarshal(%q)` to return %q, got %v", string(out), input, uerr)) } + +@test +utf8_string_of_multibyte_characters :: proc(t: ^testing.T) { + _, err := json.parse_string(`"🐛✅"`) + msg := fmt.tprintf("Expected `json.parse` to return nil, got %v", err) + expect(t, err == nil, msg) +} From f1872f495ad1e6abc695b78ee2cc3b1c0444cb8b Mon Sep 17 00:00:00 2001 From: Abdelrahman Farid Date: Tue, 19 Sep 2023 22:19:05 +0300 Subject: [PATCH 2/4] Fix bug with index increment in `unquote_string` --- core/encoding/json/parser.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index d007e16d7..bc381efee 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -343,7 +343,7 @@ unquote_string :: proc(token: Token, spec: Specification, allocator := context.a i += 1 continue } - r, w := utf8.decode_rune_in_string(s) + r, w := utf8.decode_rune_in_string(s[i:]) if r == utf8.RUNE_ERROR && w == 1 { break } From 39c85cafa20b669c19cd4cfc18bc525a12da8d0b Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Fri, 22 Sep 2023 08:21:06 +0200 Subject: [PATCH 3/4] Fix for crash when using io.write_u128/io.write_i128 due to buffer being too small --- core/io/util.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/io/util.odin b/core/io/util.odin index c77d0be9d..c24eb99c5 100644 --- a/core/io/util.odin +++ b/core/io/util.odin @@ -39,12 +39,12 @@ write_int :: proc(w: Writer, i: int, base: int = 10, n_written: ^int = nil) -> ( } write_u128 :: proc(w: Writer, i: u128, base: int = 10, n_written: ^int = nil) -> (n: int, err: Error) { - buf: [32]byte + buf: [39]byte s := strconv.append_bits_128(buf[:], i, base, false, 128, strconv.digits, nil) return write_string(w, s, n_written) } write_i128 :: proc(w: Writer, i: i128, base: int = 10, n_written: ^int = nil) -> (n: int, err: Error) { - buf: [32]byte + buf: [40]byte s := strconv.append_bits_128(buf[:], u128(i), base, true, 128, strconv.digits, nil) return write_string(w, s, n_written) } From 0b7854479638603d877912682e5cff5910ed4e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tar=C4=B1k=20B?= Date: Fri, 22 Sep 2023 17:12:16 +0200 Subject: [PATCH 4/4] Fix various mismatches with the specified -max-error-count flag --- src/error.cpp | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/src/error.cpp b/src/error.cpp index 6a039006b..e63682829 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -356,7 +356,9 @@ gb_internal void error_out_coloured(char const *str, TerminalStyle style, Termin gb_internal void error_va(TokenPos const &pos, TokenPos end, char const *fmt, va_list va) { global_error_collector.count.fetch_add(1); - + if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { + gb_exit(1); + } mutex_lock(&global_error_collector.mutex); // NOTE(bill): Duplicate error, skip it if (pos.line == 0) { @@ -372,11 +374,10 @@ gb_internal void error_va(TokenPos const &pos, TokenPos end, char const *fmt, va error_out_va(fmt, va); error_out("\n"); show_error_on_line(pos, end); + } else { + global_error_collector.count.fetch_sub(1); } mutex_unlock(&global_error_collector.mutex); - if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { - gb_exit(1); - } } gb_internal void warning_va(TokenPos const &pos, TokenPos end, char const *fmt, va_list va) { @@ -410,8 +411,11 @@ gb_internal void error_line_va(char const *fmt, va_list va) { } gb_internal void error_no_newline_va(TokenPos const &pos, char const *fmt, va_list va) { - mutex_lock(&global_error_collector.mutex); global_error_collector.count.fetch_add(1); + if (global_error_collector.count.load() > MAX_ERROR_COLLECTOR_COUNT()) { + gb_exit(1); + } + mutex_lock(&global_error_collector.mutex); // NOTE(bill): Duplicate error, skip it if (pos.line == 0) { error_out_coloured("Error: ", TerminalStyle_Normal, TerminalColour_Red); @@ -425,15 +429,15 @@ gb_internal void error_no_newline_va(TokenPos const &pos, char const *fmt, va_li error_out_va(fmt, va); } mutex_unlock(&global_error_collector.mutex); - if (global_error_collector.count.load() > MAX_ERROR_COLLECTOR_COUNT()) { - gb_exit(1); - } } gb_internal void syntax_error_va(TokenPos const &pos, TokenPos end, char const *fmt, va_list va) { + global_error_collector.count.fetch_add(1); + if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { + gb_exit(1); + } mutex_lock(&global_error_collector.mutex); - global_error_collector.count++; // NOTE(bill): Duplicate error, skip it if (global_error_collector.prev != pos) { global_error_collector.prev = pos; @@ -447,16 +451,14 @@ gb_internal void syntax_error_va(TokenPos const &pos, TokenPos end, char const * error_out_va(fmt, va); error_out("\n"); } - mutex_unlock(&global_error_collector.mutex); - if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { - gb_exit(1); - } } gb_internal void syntax_error_with_verbose_va(TokenPos const &pos, TokenPos end, char const *fmt, va_list va) { global_error_collector.count.fetch_add(1); - + if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { + gb_exit(1); + } mutex_lock(&global_error_collector.mutex); // NOTE(bill): Duplicate error, skip it if (pos.line == 0) { @@ -474,9 +476,6 @@ gb_internal void syntax_error_with_verbose_va(TokenPos const &pos, TokenPos end, show_error_on_line(pos, end); } mutex_unlock(&global_error_collector.mutex); - if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { - gb_exit(1); - } } @@ -578,7 +577,3 @@ gb_internal void compiler_error(char const *fmt, ...) { GB_DEBUG_TRAP(); gb_exit(1); } - - - -