From a58a08c0c3aaf91bec0b02e53bbca64af4ded256 Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Wed, 1 Nov 2023 00:23:17 +0100 Subject: [PATCH 1/6] JSON: Option to sort marshaled maps before outputting. Also added a json.clone_value proc --- core/encoding/json/marshal.odin | 144 ++++++++++++++++++++++---------- core/encoding/json/types.odin | 27 ++++++ 2 files changed, 127 insertions(+), 44 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 85eca50b6..6922f9b77 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -7,6 +7,7 @@ import "core:strconv" import "core:strings" import "core:reflect" import "core:io" +import "core:slice" Marshal_Data_Error :: enum { None, @@ -18,29 +19,40 @@ Marshal_Error :: union #shared_nil { io.Error, } -// careful with MJSON maps & non quotes usage as keys without whitespace will lead to bad results +// careful with MJSON maps & non quotes usage as keys with whitespace will lead to bad results Marshal_Options :: struct { // output based on spec spec: Specification, - // use line breaks & tab|spaces + // Use line breaks & tabs/spaces pretty: bool, - // spacing + // Use spaces for indentation instead of tabs use_spaces: bool, + + // Given use_spaces true, use this many spaces per indent level. 0 means 4 spaces. spaces: int, - // state - indentation: int, - - // option to output uint in JSON5 & MJSON + // Output uint as hex in JSON5 & MJSON write_uint_as_hex: bool, - // mjson output options + // If spec is MJSON and this is true, then keys will be quoted. + // + // WARNING: If your keys contain whitespace and this is false, then the + // output will be bad. mjson_keys_use_quotes: bool, + + // If spec is MJSON and this is true, then use '=' as delimiter between + // keys and values, otherwise ':' is used. mjson_keys_use_equal_sign: bool, - // mjson state + // When outputting a map, sort the output by key. + // + // NOTE: This will temp allocate and sort a list for each map. + sort_maps_by_key: bool, + + // Internal state + indentation: int, mjson_skipped_first_braces_start: bool, mjson_skipped_first_braces_end: bool, } @@ -263,36 +275,81 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: map_cap := uintptr(runtime.map_cap(m^)) ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(m^, info.map_info) - i := 0 - for bucket_index in 0.. bool { return i.key < j.key }) + + for s, i in sorted { + opt_write_iteration(w, opt, i) or_return + opt_write_key(w, opt, s.key) or_return + marshal_to_writer(w, s.value, opt) or_return + } + } else { + i := 0 + for bucket_index in 0.. (err // insert start byte and increase indentation on pretty opt_write_start :: proc(w: io.Writer, opt: ^Marshal_Options, c: byte) -> (err: io.Error) { - // skip mjson starting braces - if opt.spec == .MJSON && !opt.mjson_skipped_first_braces_start { + // Skip MJSON starting braces. We make sure to only do this for c == '{', + // skipping a starting '[' is not allowed. + if opt.spec == .MJSON && !opt.mjson_skipped_first_braces_start && opt.indentation == 0 && c == '{' { opt.mjson_skipped_first_braces_start = true return } @@ -473,11 +531,9 @@ opt_write_iteration :: proc(w: io.Writer, opt: ^Marshal_Options, iteration: int) // decrease indent, write spacing and insert end byte opt_write_end :: proc(w: io.Writer, opt: ^Marshal_Options, c: byte) -> (err: io.Error) { - if opt.spec == .MJSON && opt.mjson_skipped_first_braces_start && !opt.mjson_skipped_first_braces_end { - if opt.indentation == 0 { - opt.mjson_skipped_first_braces_end = true - return - } + if opt.spec == .MJSON && opt.mjson_skipped_first_braces_start && !opt.mjson_skipped_first_braces_end && opt.indentation == 0 && c == '}' { + opt.mjson_skipped_first_braces_end = true + return } opt.indentation -= 1 diff --git a/core/encoding/json/types.odin b/core/encoding/json/types.odin index 089fd9c9b..60b3defa1 100644 --- a/core/encoding/json/types.odin +++ b/core/encoding/json/types.odin @@ -1,5 +1,7 @@ package json +import "core:strings" + /* JSON strict JSON @@ -104,4 +106,29 @@ destroy_value :: proc(value: Value, allocator := context.allocator) { case String: delete(v) } +} + +clone_value :: proc(value: Value, allocator := context.allocator) -> Value { + context.allocator = allocator + + #partial switch &v in value { + case Object: + new_o := make(Object, len(v)) + for key, elem in v { + new_o[strings.clone(key)] = clone_value(elem) + } + return new_o + case Array: + len := len(v) + new_a := make(Array, len) + vv := v + for elem, idx in vv { + new_a[idx] = clone_value(elem) + } + return new_a + case String: + return strings.clone(v) + } + + return value } \ No newline at end of file From a73ff00b0207ed211f2732ca4832ea6dd93e2d2f Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Wed, 1 Nov 2023 00:33:59 +0100 Subject: [PATCH 2/6] Indentation fix. --- core/encoding/json/marshal.odin | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 6922f9b77..e53ab9a7c 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -311,13 +311,13 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: append(&sorted, Entry { key = name, value = any{value, info.value.id}}) } - slice.sort_by(sorted[:], proc(i, j: Entry) -> bool { return i.key < j.key }) + slice.sort_by(sorted[:], proc(i, j: Entry) -> bool { return i.key < j.key }) - for s, i in sorted { + for s, i in sorted { opt_write_iteration(w, opt, i) or_return opt_write_key(w, opt, s.key) or_return marshal_to_writer(w, s.value, opt) or_return - } + } } else { i := 0 for bucket_index in 0.. Date: Wed, 1 Nov 2023 00:37:27 +0100 Subject: [PATCH 3/6] Reversed order of sort_maps_by_key check in marshal.odin to make PR comparison clearer. --- core/encoding/json/marshal.odin | 66 ++++++++++++++++----------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index e53ab9a7c..80b2e4dd4 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -275,7 +275,39 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: map_cap := uintptr(runtime.map_cap(m^)) ks, vs, hs, _, _ := runtime.map_kvh_data_dynamic(m^, info.map_info) - if opt.sort_maps_by_key { + if !opt.sort_maps_by_key { + i := 0 + for bucket_index in 0.. (err: opt_write_key(w, opt, s.key) or_return marshal_to_writer(w, s.value, opt) or_return } - } else { - i := 0 - for bucket_index in 0.. Date: Wed, 1 Nov 2023 00:46:01 +0100 Subject: [PATCH 4/6] Added temp allocator guard to json.marshal, in case we temp alloc when sorting map kesy --- core/encoding/json/marshal.odin | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 80b2e4dd4..9ef78b95d 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -83,6 +83,9 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: return } + // temp guard in case we are sorting map keys, which will use temp allocations + runtime.DEFAULT_TEMP_ALLOCATOR_TEMP_GUARD() + ti := runtime.type_info_base(type_info_of(v.id)) a := any{v.data, ti.id} From 75cb2c68cc1f75760d39cd7a41f91d65496843dd Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Wed, 1 Nov 2023 00:57:27 +0100 Subject: [PATCH 5/6] Cleanup of json.clone_value --- core/encoding/json/types.odin | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/core/encoding/json/types.odin b/core/encoding/json/types.odin index 60b3defa1..20c806236 100644 --- a/core/encoding/json/types.odin +++ b/core/encoding/json/types.odin @@ -119,10 +119,8 @@ clone_value :: proc(value: Value, allocator := context.allocator) -> Value { } return new_o case Array: - len := len(v) - new_a := make(Array, len) - vv := v - for elem, idx in vv { + new_a := make(Array, len(v)) + for elem, idx in v { new_a[idx] = clone_value(elem) } return new_a From 9d067ae562d60ae3e0eb29770693d6fc26e793f1 Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Mon, 22 Jan 2024 14:35:05 +0100 Subject: [PATCH 6/6] Made sure temp guard for sorting map keys in json marshal code ignores temp allocator --- core/encoding/json/marshal.odin | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 9ef78b95d..ab2af9561 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -62,6 +62,9 @@ marshal :: proc(v: any, opt: Marshal_Options = {}, allocator := context.allocato defer if err != nil { strings.builder_destroy(&b) } + + // temp guard in case we are sorting map keys, which will use temp allocations + runtime.DEFAULT_TEMP_ALLOCATOR_TEMP_GUARD(ignore = allocator == context.temp_allocator) opt := opt marshal_to_builder(&b, v, &opt) or_return @@ -83,9 +86,6 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: return } - // temp guard in case we are sorting map keys, which will use temp allocations - runtime.DEFAULT_TEMP_ALLOCATOR_TEMP_GUARD() - ti := runtime.type_info_base(type_info_of(v.id)) a := any{v.data, ti.id}