From 614ea5c16888fccbedf521dc2f7e0e891284f66f Mon Sep 17 00:00:00 2001 From: zhibog Date: Fri, 1 Nov 2019 22:34:19 +0100 Subject: [PATCH 01/10] Added official test vectors from the RFC --- core/encoding/base64/base64.odin | 37 +++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/core/encoding/base64/base64.odin b/core/encoding/base64/base64.odin index 0224e93e4..b33f9cd5f 100644 --- a/core/encoding/base64/base64.odin +++ b/core/encoding/base64/base64.odin @@ -90,4 +90,39 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato out[j + 2] = byte(b2); } return out; -} \ No newline at end of file +} + +// @note(zh): Test inputs. Taken from RFC4648 +/* +import "core:fmt" +main :: proc() { + Test :: struct { + plain: string, + encoded: string, + }; + + test_vectors := [?]Test { + Test{"", ""}, + Test{"f", "Zg=="}, + Test{"fo", "Zm8="}, + Test{"foo", "Zm9v"}, + Test{"foob", "Zm9vYg=="}, + Test{"fooba", "Zm9vYmE="}, + Test{"foobar", "Zm9vYmFy"}, + }; + + // Encode test + for v in test_vectors { + enc := encode(([]byte)(v.plain)); + fmt.printf("encode(\"%s\") => \"%s\" \t| want: \"%s\"\n", v.plain, enc, v.encoded); + delete(enc); + } + + // Decode test + for v in test_vectors { + dec := decode(v.encoded); + fmt.printf("decode(\"%s\") => \"%s\" \t| want: \"%s\"\n", v.encoded, string(dec), v.plain); + delete(dec); + } +} +*/ \ No newline at end of file From 20db0e7f09f26884fb53c6872fc7633c48b94b4c Mon Sep 17 00:00:00 2001 From: zhibog Date: Fri, 1 Nov 2019 22:35:09 +0100 Subject: [PATCH 02/10] Added Base32 + official test vectors from the RFC --- core/encoding/base32/base32.odin | 180 +++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 core/encoding/base32/base32.odin diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin new file mode 100644 index 000000000..ed27e9b3a --- /dev/null +++ b/core/encoding/base32/base32.odin @@ -0,0 +1,180 @@ +package base32 + +// @note(zh): Encoding utility for Base32 +// A secondary param can be used to supply a custom alphabet to +// @link(encode) and a matching decoding table to @link(decode). +// If none is supplied it just uses the standard Base32 alphabet. +// Incase your specific version does not use padding, you may +// truncate it from the encoded output. + +ENC_TABLE := [32]byte { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', '2', '3', '4', '5', '6', '7' +}; + +PADDING :: '='; + +DEC_TABLE := [?]u8 { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 26, 27, 28, 29, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +}; + +encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string { + out_length := (len(data) + 4) / 5 * 8; + out := make([]byte, out_length); + _encode(out, data); + return string(out); +} + +@private +_encode :: inline proc "contextless"(out, data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) { + out := out; + data := data; + + for len(data) > 0 { + carry: byte; + switch len(data) { + case: + out[7] = ENC_TABLE[data[4] & 0x1f]; + carry = data[4] >> 5; + fallthrough; + case 4: + out[6] = ENC_TABLE[carry | (data[3] << 3) & 0x1f]; + out[5] = ENC_TABLE[(data[3] >> 2) & 0x1f]; + carry = data[3] >> 7; + fallthrough; + case 3: + out[4] = ENC_TABLE[carry | (data[2] << 1) & 0x1f]; + carry = (data[2] >> 4) & 0x1f; + fallthrough; + case 2: + out[3] = ENC_TABLE[carry | (data[1] << 4) & 0x1f]; + out[2] = ENC_TABLE[(data[1] >> 1) & 0x1f]; + carry = (data[1] >> 6) & 0x1f; + fallthrough; + case 1: + out[1] = ENC_TABLE[carry | (data[0] << 2) & 0x1f]; + out[0] = ENC_TABLE[data[0] >> 3]; + } + + if len(data) < 5 { + out[7] = byte(PADDING); + if len(data) < 4 { + out[6] = byte(PADDING); + out[5] = byte(PADDING); + if len(data) < 3 { + out[4] = byte(PADDING); + if len(data) < 2 { + out[3] = byte(PADDING); + out[2] = byte(PADDING); + } + } + } + break; + } + data = data[5:]; + out = out[8:]; + } +} + +decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> []byte #no_bounds_check{ + if len(data) == 0 do return []byte{}; + + outi := 0; + olen := len(data); + data := data; + + out := make([]byte, len(data) / 8 * 5, allocator); + end := false; + for len(data) > 0 && !end { + dbuf : [8]byte; + dlen := 8; + + for j := 0; j < 8; { + if len(data) == 0 { + dlen, end = j, true; + break; + } + input := data[0]; + data = data[1:]; + if input == byte(PADDING) && j >= 2 && len(data) < 8 { + assert(!(len(data) + j < 8 - 1), "Corrupted input"); + for k := 0; k < 8-1-j; k +=1 do assert(len(data) < k || data[k] == byte(PADDING), "Corrupted input"); + dlen, end = j, true; + assert(dlen != 1 && dlen != 3 && dlen != 6, "Corrupted input"); + break; + } + dbuf[j] = DEC_TABLE[input]; + assert(dbuf[j] != 0xff, "Corrupted input"); + j += 1; + } + + switch dlen { + case 8: + out[outi + 4] = dbuf[6] << 5 | dbuf[7]; + fallthrough; + case 7: + out[outi + 3] = dbuf[4] << 7 | dbuf[5] << 2 | dbuf[6] >> 3; + fallthrough; + case 5: + out[outi + 2] = dbuf[3] << 4 | dbuf[4] >> 1; + fallthrough; + case 4: + out[outi + 1] = dbuf[1] << 6 | dbuf[2] << 1 | dbuf[3] >> 4; + fallthrough; + case 2: + out[outi + 0] = dbuf[0] << 3 | dbuf[1] >> 2; + } + outi += 5; + } + return out; +} + +// @note(zh): Test inputs. Taken from RFC4648 +/* +import "core:fmt" +main :: proc() { + Test :: struct { + plain: string, + encoded: string, + }; + + test_vectors := [?]Test { + Test{"", ""}, + Test{"f", "MY======"}, + Test{"fo", "MZXQ===="}, + Test{"foo", "MZXW6==="}, + Test{"foob", "MZXW6YQ="}, + Test{"fooba", "MZXW6YTB"}, + Test{"foobar", "MZXW6YTBOI======"}, + }; + + // Encode test + for v in test_vectors { + enc := encode(([]byte)(v.plain)); + fmt.printf("encode(\"%s\") => \"%s\" \t| want: \"%s\"\n", v.plain, enc, v.encoded); + delete(enc); + } + + // Decode test + for v in test_vectors { + dec := decode(v.encoded); + fmt.printf("decode(\"%s\") => \"%s\" \t| want: \"%s\"\n", v.encoded, string(dec), v.plain); + delete(dec); + } +} +*/ \ No newline at end of file From 4b718aae7556511e7461faf89b147a3416ea8f8e Mon Sep 17 00:00:00 2001 From: zhibog Date: Fri, 1 Nov 2019 22:35:46 +0100 Subject: [PATCH 03/10] Added an implementation for reading and writing csv files --- core/encoding/csv/csv.odin | 106 +++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 core/encoding/csv/csv.odin diff --git a/core/encoding/csv/csv.odin b/core/encoding/csv/csv.odin new file mode 100644 index 000000000..a8f387e69 --- /dev/null +++ b/core/encoding/csv/csv.odin @@ -0,0 +1,106 @@ +package csv + +// @note(zh): Encoding utility for csv files +// You may use the provided struct to build your csv dynamically. +// If you have a string with the whole content already, just use write_raw. +// You are able to read the csv with the headers included in the data or omitted by providing +// a bool parameter to the proc as shown down below. +// Example useage: +/* +import "core:fmt" +main :: proc() { + ctx: CSV; + ctx.data = {{"Col 1", "Col 2", "Col 3"}, {"aaa", "bbb", "ccc"}, {"ddd", "eee", "fff"}, {"ggg", "hhh", "iii"}}; + ctx.line_ending = CRLF; + file_name := "test.csv"; + + // Write file and read with the headers omitted + if isOkWrite := write(file_name, &ctx); isOkWrite { + if content, col_count, isOkRead := read(file_name, DELIMITER, true); isOkRead { + fmt.println("Column count(no headers): ", col_count); + fmt.println(content); + } + } + + // Write file and read with the headers being read as well + if isOkWrite := write(file_name, &ctx); isOkWrite { + if content, col_count, isOkRead := read(file_name); isOkRead { + fmt.println("Column count(with headers): ", col_count); + fmt.println(content); + } + } +} +*/ + +import "core:os" +import "core:strings" + +CSV :: struct { + data: [][]string, + line_ending: string, + delimiter: string, +}; + +LF :: "\n"; +CRLF :: "\r\n"; +DELIMITER :: ","; + +write :: proc(path: string, ctx: ^CSV) -> bool { + b := strings.make_builder(); + defer strings.destroy_builder(&b); + + if ctx.line_ending == "" do ctx.line_ending = LF; + if ctx.delimiter == "" do ctx.delimiter = DELIMITER; + + for row in ctx.data { + for col, i in row { + strings.write_string(&b, col); + if i + 1 < len(row) do strings.write_string(&b, ctx.delimiter); + } + strings.write_string(&b, ctx.line_ending); + } + return write_raw(path, b.buf[:]); +} + +write_raw :: proc(path: string, data: []byte) -> bool { + file, err := os.open(path, os.O_RDWR | os.O_CREATE | os.O_TRUNC); + if err != os.ERROR_NONE do return false; + defer os.close(file); + + if _, err := os.write(file, data); err != os.ERROR_NONE do return false; + return true; +} + +read :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> ([]string, int, bool) { + if bytes, isOk := os.read_entire_file(path); isOk { + cols: [dynamic]string; + defer delete(cols); + out: [dynamic]string; + col_count := 0; + prev_index := 0; + for i := 0; i < len(bytes); i += 1 { + if bytes[i] == '\n' { + append(&cols, string(bytes[prev_index:i])); + i += 1; + prev_index = i; + col_count += 1; + } else if bytes[i] == '\r' { + if bytes[i + 1] == '\n' { + append(&cols, string(bytes[prev_index:i])); + i += 2; + prev_index = i; + col_count += 1; + } else { + append(&cols, string(bytes[prev_index:i])); + i += 1; + prev_index = i; + col_count += 1; + } + } + } + for col in cols do append(&out, ..strings.split(col, delimiter)); + if skip_header do return out[col_count:], col_count - 1, true; + else do return out[:], col_count, true; + } + return nil, -1, false; +} \ No newline at end of file From dc2d5239c5801fb8ac7fea5b4b9717067eb5ce8f Mon Sep 17 00:00:00 2001 From: zhibog Date: Fri, 8 Nov 2019 20:06:58 +0100 Subject: [PATCH 04/10] Removed comments regarding RFC test vectors --- core/encoding/base32/base32.odin | 37 +------------------------------- core/encoding/base64/base64.odin | 37 +------------------------------- 2 files changed, 2 insertions(+), 72 deletions(-) diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index ed27e9b3a..f88c21e2e 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -142,39 +142,4 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato outi += 5; } return out; -} - -// @note(zh): Test inputs. Taken from RFC4648 -/* -import "core:fmt" -main :: proc() { - Test :: struct { - plain: string, - encoded: string, - }; - - test_vectors := [?]Test { - Test{"", ""}, - Test{"f", "MY======"}, - Test{"fo", "MZXQ===="}, - Test{"foo", "MZXW6==="}, - Test{"foob", "MZXW6YQ="}, - Test{"fooba", "MZXW6YTB"}, - Test{"foobar", "MZXW6YTBOI======"}, - }; - - // Encode test - for v in test_vectors { - enc := encode(([]byte)(v.plain)); - fmt.printf("encode(\"%s\") => \"%s\" \t| want: \"%s\"\n", v.plain, enc, v.encoded); - delete(enc); - } - - // Decode test - for v in test_vectors { - dec := decode(v.encoded); - fmt.printf("decode(\"%s\") => \"%s\" \t| want: \"%s\"\n", v.encoded, string(dec), v.plain); - delete(dec); - } -} -*/ \ No newline at end of file +} \ No newline at end of file diff --git a/core/encoding/base64/base64.odin b/core/encoding/base64/base64.odin index b33f9cd5f..0224e93e4 100644 --- a/core/encoding/base64/base64.odin +++ b/core/encoding/base64/base64.odin @@ -90,39 +90,4 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato out[j + 2] = byte(b2); } return out; -} - -// @note(zh): Test inputs. Taken from RFC4648 -/* -import "core:fmt" -main :: proc() { - Test :: struct { - plain: string, - encoded: string, - }; - - test_vectors := [?]Test { - Test{"", ""}, - Test{"f", "Zg=="}, - Test{"fo", "Zm8="}, - Test{"foo", "Zm9v"}, - Test{"foob", "Zm9vYg=="}, - Test{"fooba", "Zm9vYmE="}, - Test{"foobar", "Zm9vYmFy"}, - }; - - // Encode test - for v in test_vectors { - enc := encode(([]byte)(v.plain)); - fmt.printf("encode(\"%s\") => \"%s\" \t| want: \"%s\"\n", v.plain, enc, v.encoded); - delete(enc); - } - - // Decode test - for v in test_vectors { - dec := decode(v.encoded); - fmt.printf("decode(\"%s\") => \"%s\" \t| want: \"%s\"\n", v.encoded, string(dec), v.plain); - delete(dec); - } -} -*/ \ No newline at end of file +} \ No newline at end of file From 80cdf8b6a803cd510c8eff29e883124d2608a184 Mon Sep 17 00:00:00 2001 From: zhibog Date: Fri, 8 Nov 2019 20:16:56 +0100 Subject: [PATCH 05/10] Should be row_count obviously --- core/encoding/csv/csv.odin | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/core/encoding/csv/csv.odin b/core/encoding/csv/csv.odin index a8f387e69..80af438c3 100644 --- a/core/encoding/csv/csv.odin +++ b/core/encoding/csv/csv.odin @@ -16,16 +16,16 @@ main :: proc() { // Write file and read with the headers omitted if isOkWrite := write(file_name, &ctx); isOkWrite { - if content, col_count, isOkRead := read(file_name, DELIMITER, true); isOkRead { - fmt.println("Column count(no headers): ", col_count); + if content, row_count, isOkRead := read(file_name, DELIMITER, true); isOkRead { + fmt.println("Column count(no headers): ", row_count); fmt.println(content); } } // Write file and read with the headers being read as well if isOkWrite := write(file_name, &ctx); isOkWrite { - if content, col_count, isOkRead := read(file_name); isOkRead { - fmt.println("Column count(with headers): ", col_count); + if content, row_count, isOkRead := read(file_name); isOkRead { + fmt.println("Column count(with headers): ", row_count); fmt.println(content); } } @@ -76,31 +76,31 @@ read :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> ([]s cols: [dynamic]string; defer delete(cols); out: [dynamic]string; - col_count := 0; + row_count := 0; prev_index := 0; for i := 0; i < len(bytes); i += 1 { if bytes[i] == '\n' { append(&cols, string(bytes[prev_index:i])); i += 1; prev_index = i; - col_count += 1; + row_count += 1; } else if bytes[i] == '\r' { if bytes[i + 1] == '\n' { append(&cols, string(bytes[prev_index:i])); i += 2; prev_index = i; - col_count += 1; + row_count += 1; } else { append(&cols, string(bytes[prev_index:i])); i += 1; prev_index = i; - col_count += 1; + row_count += 1; } } } for col in cols do append(&out, ..strings.split(col, delimiter)); - if skip_header do return out[col_count:], col_count - 1, true; - else do return out[:], col_count, true; + if skip_header do return out[row_count:], row_count - 1, true; + else do return out[:], row_count, true; } return nil, -1, false; } \ No newline at end of file From 672cfd51c3fe26b906e84315de6cd0460420d2f4 Mon Sep 17 00:00:00 2001 From: zhibog Date: Fri, 8 Nov 2019 20:21:18 +0100 Subject: [PATCH 06/10] Added names to return values --- core/encoding/csv/csv.odin | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/encoding/csv/csv.odin b/core/encoding/csv/csv.odin index 80af438c3..580767078 100644 --- a/core/encoding/csv/csv.odin +++ b/core/encoding/csv/csv.odin @@ -71,11 +71,11 @@ write_raw :: proc(path: string, data: []byte) -> bool { return true; } -read :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> ([]string, int, bool) { +read :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> (content: []string, row_count: int, is_ok: bool) { if bytes, isOk := os.read_entire_file(path); isOk { cols: [dynamic]string; defer delete(cols); - out: [dynamic]string; + content: [dynamic]string; row_count := 0; prev_index := 0; for i := 0; i < len(bytes); i += 1 { @@ -98,9 +98,9 @@ read :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> ([]s } } } - for col in cols do append(&out, ..strings.split(col, delimiter)); - if skip_header do return out[row_count:], row_count - 1, true; - else do return out[:], row_count, true; + for col in cols do append(&content, ..strings.split(col, delimiter)); + if skip_header do return content[row_count:], row_count - 1, true; + else do return content[:], row_count, true; } return nil, -1, false; } \ No newline at end of file From 803f6a665133af8ed3864a3f5c92cb479a826530 Mon Sep 17 00:00:00 2001 From: zhibog Date: Fri, 8 Nov 2019 22:17:24 +0100 Subject: [PATCH 07/10] Added procs to read and write just the data, without any file loading / writing --- core/encoding/csv/csv.odin | 56 ++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/core/encoding/csv/csv.odin b/core/encoding/csv/csv.odin index 580767078..04a776fcd 100644 --- a/core/encoding/csv/csv.odin +++ b/core/encoding/csv/csv.odin @@ -2,7 +2,7 @@ package csv // @note(zh): Encoding utility for csv files // You may use the provided struct to build your csv dynamically. -// If you have a string with the whole content already, just use write_raw. +// If you have a string with the whole content already, just use write_file_raw to write it to a file. // You are able to read the csv with the headers included in the data or omitted by providing // a bool parameter to the proc as shown down below. // Example useage: @@ -15,20 +15,33 @@ main :: proc() { file_name := "test.csv"; // Write file and read with the headers omitted - if isOkWrite := write(file_name, &ctx); isOkWrite { - if content, row_count, isOkRead := read(file_name, DELIMITER, true); isOkRead { - fmt.println("Column count(no headers): ", row_count); + if isOkWrite := write_file(file_name, &ctx); isOkWrite { + if content, row_count, isOkRead := read_file(file_name, DELIMITER, true); isOkRead { + fmt.println("Row count(no headers): ", row_count); fmt.println(content); } } // Write file and read with the headers being read as well - if isOkWrite := write(file_name, &ctx); isOkWrite { - if content, row_count, isOkRead := read(file_name); isOkRead { - fmt.println("Column count(with headers): ", row_count); + if isOkWrite := write_file(file_name, &ctx); isOkWrite { + if content, row_count, isOkRead := read_file(file_name); isOkRead { + fmt.println("Row count(with headers): ", row_count); fmt.println(content); } } + + // Write slice to a csv string + csv_string_data: [9]string = {"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii"}; + col_count := 3; + csv_string := write_string(csv_string_data[:], col_count); + fmt.println(csv_string); + + // Read data into a slice when you have read the csv file yourself + csv_data := "aaa, bbb, ccc\n ddd, eee, fff\n ggg, hhh, iii"; + if content, row_count, isOkRead := read_string(csv_data); isOkRead { + fmt.println("Row count(from string): ", row_count); + fmt.println(content); + } } */ @@ -45,7 +58,17 @@ LF :: "\n"; CRLF :: "\r\n"; DELIMITER :: ","; -write :: proc(path: string, ctx: ^CSV) -> bool { +write_string :: proc(data: []string, col_count: int, delimiter := DELIMITER, line_ending := LF) -> string { + b := strings.make_builder(); + for i := 0; i < len(data); i += 1 { + if i >= col_count && i % col_count == 0 do strings.write_string(&b, line_ending); + strings.write_string(&b, data[i]); + if len(data) - i > 1 do strings.write_string(&b, delimiter); + } + return strings.to_string(b); +} + +write_file :: proc(path: string, ctx: ^CSV) -> bool { b := strings.make_builder(); defer strings.destroy_builder(&b); @@ -59,10 +82,10 @@ write :: proc(path: string, ctx: ^CSV) -> bool { } strings.write_string(&b, ctx.line_ending); } - return write_raw(path, b.buf[:]); + return write_file_raw(path, b.buf[:]); } -write_raw :: proc(path: string, data: []byte) -> bool { +write_file_raw :: proc(path: string, data: []byte) -> bool { file, err := os.open(path, os.O_RDWR | os.O_CREATE | os.O_TRUNC); if err != os.ERROR_NONE do return false; defer os.close(file); @@ -71,13 +94,14 @@ write_raw :: proc(path: string, data: []byte) -> bool { return true; } -read :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> (content: []string, row_count: int, is_ok: bool) { - if bytes, isOk := os.read_entire_file(path); isOk { +read_string ::proc(data: string, delimiter := DELIMITER, skip_header := false) -> (content: []string, row_count: int, is_ok: bool) { + if data != "" { cols: [dynamic]string; defer delete(cols); content: [dynamic]string; row_count := 0; prev_index := 0; + bytes := ([]byte)(data); for i := 0; i < len(bytes); i += 1 { if bytes[i] == '\n' { append(&cols, string(bytes[prev_index:i])); @@ -101,6 +125,14 @@ read :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> (con for col in cols do append(&content, ..strings.split(col, delimiter)); if skip_header do return content[row_count:], row_count - 1, true; else do return content[:], row_count, true; + } else { + return nil, -1, false; + } +} + +read_file :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> (content: []string, row_count: int, is_ok: bool) { + if bytes, isOk := os.read_entire_file(path); isOk { + return read_string(string(bytes), delimiter, skip_header); } return nil, -1, false; } \ No newline at end of file From 0451c88ab662b55861efe33ca3f1aef2dec54bd0 Mon Sep 17 00:00:00 2001 From: zhibog Date: Sat, 9 Nov 2019 18:04:30 +0100 Subject: [PATCH 08/10] Fixed indenting --- core/encoding/base32/base32.odin | 88 ++++++++++++++++---------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index f88c21e2e..c7dc9870f 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -17,19 +17,19 @@ ENC_TABLE := [32]byte { PADDING :: '='; DEC_TABLE := [?]u8 { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 27, 28, 29, 30, 31, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; @@ -47,7 +47,7 @@ _encode :: inline proc "contextless"(out, data: []byte, ENC_TBL := ENC_TABLE, al for len(data) > 0 { carry: byte; - switch len(data) { + switch len(data) { case: out[7] = ENC_TABLE[data[4] & 0x1f]; carry = data[4] >> 5; @@ -69,23 +69,23 @@ _encode :: inline proc "contextless"(out, data: []byte, ENC_TBL := ENC_TABLE, al case 1: out[1] = ENC_TABLE[carry | (data[0] << 2) & 0x1f]; out[0] = ENC_TABLE[data[0] >> 3]; - } + } if len(data) < 5 { - out[7] = byte(PADDING); - if len(data) < 4 { - out[6] = byte(PADDING); - out[5] = byte(PADDING); - if len(data) < 3 { - out[4] = byte(PADDING); - if len(data) < 2 { - out[3] = byte(PADDING); - out[2] = byte(PADDING); - } - } - } - break; - } + out[7] = byte(PADDING); + if len(data) < 4 { + out[6] = byte(PADDING); + out[5] = byte(PADDING); + if len(data) < 3 { + out[4] = byte(PADDING); + if len(data) < 2 { + out[3] = byte(PADDING); + out[2] = byte(PADDING); + } + } + } + break; + } data = data[5:]; out = out[8:]; } @@ -101,29 +101,29 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato out := make([]byte, len(data) / 8 * 5, allocator); end := false; for len(data) > 0 && !end { - dbuf : [8]byte; - dlen := 8; + dbuf : [8]byte; + dlen := 8; - for j := 0; j < 8; { - if len(data) == 0 { - dlen, end = j, true; - break; - } - input := data[0]; - data = data[1:]; - if input == byte(PADDING) && j >= 2 && len(data) < 8 { + for j := 0; j < 8; { + if len(data) == 0 { + dlen, end = j, true; + break; + } + input := data[0]; + data = data[1:]; + if input == byte(PADDING) && j >= 2 && len(data) < 8 { assert(!(len(data) + j < 8 - 1), "Corrupted input"); - for k := 0; k < 8-1-j; k +=1 do assert(len(data) < k || data[k] == byte(PADDING), "Corrupted input"); - dlen, end = j, true; + for k := 0; k < 8-1-j; k +=1 do assert(len(data) < k || data[k] == byte(PADDING), "Corrupted input"); + dlen, end = j, true; assert(dlen != 1 && dlen != 3 && dlen != 6, "Corrupted input"); - break; - } - dbuf[j] = DEC_TABLE[input]; + break; + } + dbuf[j] = DEC_TABLE[input]; assert(dbuf[j] != 0xff, "Corrupted input"); - j += 1; - } + j += 1; + } - switch dlen { + switch dlen { case 8: out[outi + 4] = dbuf[6] << 5 | dbuf[7]; fallthrough; @@ -138,8 +138,8 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato fallthrough; case 2: out[outi + 0] = dbuf[0] << 3 | dbuf[1] >> 2; - } - outi += 5; - } + } + outi += 5; + } return out; } \ No newline at end of file From 694ee02247b5b45da92d319d4134b38415d5764a Mon Sep 17 00:00:00 2001 From: zhibog Date: Sat, 9 Nov 2019 18:06:33 +0100 Subject: [PATCH 09/10] Missed one --- core/encoding/base32/base32.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index c7dc9870f..3dfa800a5 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -95,7 +95,7 @@ decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocato if len(data) == 0 do return []byte{}; outi := 0; - olen := len(data); + olen := len(data); data := data; out := make([]byte, len(data) / 8 * 5, allocator); From 2484f4d04baf3256e005f1ffd17d661db1723931 Mon Sep 17 00:00:00 2001 From: zhibog Date: Sun, 17 Nov 2019 20:09:00 +0100 Subject: [PATCH 10/10] Removed CSV stuff. --- core/encoding/csv/csv.odin | 138 ------------------------------------- 1 file changed, 138 deletions(-) delete mode 100644 core/encoding/csv/csv.odin diff --git a/core/encoding/csv/csv.odin b/core/encoding/csv/csv.odin deleted file mode 100644 index 04a776fcd..000000000 --- a/core/encoding/csv/csv.odin +++ /dev/null @@ -1,138 +0,0 @@ -package csv - -// @note(zh): Encoding utility for csv files -// You may use the provided struct to build your csv dynamically. -// If you have a string with the whole content already, just use write_file_raw to write it to a file. -// You are able to read the csv with the headers included in the data or omitted by providing -// a bool parameter to the proc as shown down below. -// Example useage: -/* -import "core:fmt" -main :: proc() { - ctx: CSV; - ctx.data = {{"Col 1", "Col 2", "Col 3"}, {"aaa", "bbb", "ccc"}, {"ddd", "eee", "fff"}, {"ggg", "hhh", "iii"}}; - ctx.line_ending = CRLF; - file_name := "test.csv"; - - // Write file and read with the headers omitted - if isOkWrite := write_file(file_name, &ctx); isOkWrite { - if content, row_count, isOkRead := read_file(file_name, DELIMITER, true); isOkRead { - fmt.println("Row count(no headers): ", row_count); - fmt.println(content); - } - } - - // Write file and read with the headers being read as well - if isOkWrite := write_file(file_name, &ctx); isOkWrite { - if content, row_count, isOkRead := read_file(file_name); isOkRead { - fmt.println("Row count(with headers): ", row_count); - fmt.println(content); - } - } - - // Write slice to a csv string - csv_string_data: [9]string = {"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii"}; - col_count := 3; - csv_string := write_string(csv_string_data[:], col_count); - fmt.println(csv_string); - - // Read data into a slice when you have read the csv file yourself - csv_data := "aaa, bbb, ccc\n ddd, eee, fff\n ggg, hhh, iii"; - if content, row_count, isOkRead := read_string(csv_data); isOkRead { - fmt.println("Row count(from string): ", row_count); - fmt.println(content); - } -} -*/ - -import "core:os" -import "core:strings" - -CSV :: struct { - data: [][]string, - line_ending: string, - delimiter: string, -}; - -LF :: "\n"; -CRLF :: "\r\n"; -DELIMITER :: ","; - -write_string :: proc(data: []string, col_count: int, delimiter := DELIMITER, line_ending := LF) -> string { - b := strings.make_builder(); - for i := 0; i < len(data); i += 1 { - if i >= col_count && i % col_count == 0 do strings.write_string(&b, line_ending); - strings.write_string(&b, data[i]); - if len(data) - i > 1 do strings.write_string(&b, delimiter); - } - return strings.to_string(b); -} - -write_file :: proc(path: string, ctx: ^CSV) -> bool { - b := strings.make_builder(); - defer strings.destroy_builder(&b); - - if ctx.line_ending == "" do ctx.line_ending = LF; - if ctx.delimiter == "" do ctx.delimiter = DELIMITER; - - for row in ctx.data { - for col, i in row { - strings.write_string(&b, col); - if i + 1 < len(row) do strings.write_string(&b, ctx.delimiter); - } - strings.write_string(&b, ctx.line_ending); - } - return write_file_raw(path, b.buf[:]); -} - -write_file_raw :: proc(path: string, data: []byte) -> bool { - file, err := os.open(path, os.O_RDWR | os.O_CREATE | os.O_TRUNC); - if err != os.ERROR_NONE do return false; - defer os.close(file); - - if _, err := os.write(file, data); err != os.ERROR_NONE do return false; - return true; -} - -read_string ::proc(data: string, delimiter := DELIMITER, skip_header := false) -> (content: []string, row_count: int, is_ok: bool) { - if data != "" { - cols: [dynamic]string; - defer delete(cols); - content: [dynamic]string; - row_count := 0; - prev_index := 0; - bytes := ([]byte)(data); - for i := 0; i < len(bytes); i += 1 { - if bytes[i] == '\n' { - append(&cols, string(bytes[prev_index:i])); - i += 1; - prev_index = i; - row_count += 1; - } else if bytes[i] == '\r' { - if bytes[i + 1] == '\n' { - append(&cols, string(bytes[prev_index:i])); - i += 2; - prev_index = i; - row_count += 1; - } else { - append(&cols, string(bytes[prev_index:i])); - i += 1; - prev_index = i; - row_count += 1; - } - } - } - for col in cols do append(&content, ..strings.split(col, delimiter)); - if skip_header do return content[row_count:], row_count - 1, true; - else do return content[:], row_count, true; - } else { - return nil, -1, false; - } -} - -read_file :: proc(path: string, delimiter := DELIMITER, skip_header := false) -> (content: []string, row_count: int, is_ok: bool) { - if bytes, isOk := os.read_entire_file(path); isOk { - return read_string(string(bytes), delimiter, skip_header); - } - return nil, -1, false; -} \ No newline at end of file