From b5c828fe4ee3f0942b2eda1dc5753e4ad6d38ea9 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 30 Nov 2021 23:01:22 +0100 Subject: [PATCH 01/79] [xml] Initial implementation of `core:encoding/xml`. A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816). Features: - Supports enough of the XML 1.0/1.1 spec to handle the 99.9% of XML documents in common current usage. - Simple to understand and use. Small. Caveats: - We do NOT support HTML in this package, as that may or may not be valid XML. If it works, great. If it doesn't, that's not considered a bug. - We do NOT support UTF-16. If you have a UTF-16 XML file, please convert it to UTF-8 first. Also, our condolences. - <[!ELEMENT and <[!ATTLIST are not supported, and will be either ignored or return an error depending on the parser options. TODO: - Optional CDATA unboxing. - Optional `>`, ` `, ` ` and other escape substitution in tag bodies. - Test suite MAYBE: - XML writer? - Serialize/deserialize Odin types? --- core/encoding/xml/debug_print.odin | 73 ++ core/encoding/xml/example/xml_example.odin | 55 ++ core/encoding/xml/tokenizer.odin | 339 +++++++++ core/encoding/xml/xml_reader.odin | 651 ++++++++++++++++++ tests/core/Makefile | 17 +- tests/core/assets/xml/nl_NL-qt-ts.ts | 35 + tests/core/assets/xml/nl_NL-xliff-1.0.xliff | 38 + tests/core/assets/xml/nl_NL-xliff-2.0.xliff | 52 ++ tests/core/assets/xml/utf8.xml | 8 + tests/core/build.bat | 15 +- .../encoding/{ => json}/test_core_json.odin | 36 +- tests/core/encoding/xml/test_core_xml.odin | 264 +++++++ 12 files changed, 1553 insertions(+), 30 deletions(-) create mode 100644 core/encoding/xml/debug_print.odin create mode 100644 core/encoding/xml/example/xml_example.odin create mode 100644 core/encoding/xml/tokenizer.odin create mode 100644 core/encoding/xml/xml_reader.odin create mode 100644 tests/core/assets/xml/nl_NL-qt-ts.ts create mode 100644 tests/core/assets/xml/nl_NL-xliff-1.0.xliff create mode 100644 tests/core/assets/xml/nl_NL-xliff-2.0.xliff create mode 100644 tests/core/assets/xml/utf8.xml rename tests/core/encoding/{ => json}/test_core_json.odin (63%) create mode 100644 tests/core/encoding/xml/test_core_xml.odin diff --git a/core/encoding/xml/debug_print.odin b/core/encoding/xml/debug_print.odin new file mode 100644 index 000000000..0b7ffa822 --- /dev/null +++ b/core/encoding/xml/debug_print.odin @@ -0,0 +1,73 @@ +package xml +/* + An XML 1.0 / 1.1 parser + + Copyright 2021 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816). + + List of contributors: + Jeroen van Rijn: Initial implementation. +*/ +import "core:fmt" + +/* + Just for debug purposes. +*/ +print :: proc(doc: ^Document) { + assert(doc != nil) + + using fmt + println("[XML Prolog]") + + for attr in doc.prolog { + printf("\t%v: %v\n", attr.key, attr.val) + } + + printf("[Encoding] %v\n", doc.encoding) + printf("[DOCTYPE] %v\n", doc.doctype.ident) + + if len(doc.doctype.rest) > 0 { + printf("\t%v\n", doc.doctype.rest) + } + + if doc.root != nil { + println(" --- ") + print_element(0, doc.root) + println(" --- ") + } +} + +print_element :: proc(indent: int, element: ^Element) { + if element == nil { return } + using fmt + + tab :: proc(indent: int) { + tabs := "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" + + i := max(0, min(indent, len(tabs))) + printf("%v", tabs[:i]) + } + + tab(indent) + + if element.kind == .Element { + printf("<%v>\n", element.ident) + if len(element.value) > 0 { + tab(indent + 1) + printf("[Value] %v\n", element.value) + } + + for attr in element.attribs { + tab(indent + 1) + printf("[Attr] %v: %v\n", attr.key, attr.val) + } + + for child in element.children { + print_element(indent + 1, child) + } + } else if element.kind == .Comment { + printf("[COMMENT] %v\n", element.value) + } +} \ No newline at end of file diff --git a/core/encoding/xml/example/xml_example.odin b/core/encoding/xml/example/xml_example.odin new file mode 100644 index 000000000..24a277de6 --- /dev/null +++ b/core/encoding/xml/example/xml_example.odin @@ -0,0 +1,55 @@ +package xml_example + +import "core:encoding/xml" +import "core:mem" +import "core:fmt" + +Error_Handler :: proc(pos: xml.Pos, fmt: string, args: ..any) { + +} + +FILENAME :: "../../../../tests/core/assets/xml/nl_NL-xliff-1.0.xliff" +DOC :: #load(FILENAME) + +OPTIONS :: xml.Options{ + flags = { + .Ignore_Unsupported, .Intern_Comments, + }, + expected_doctype = "", +} + +_main :: proc() { + using fmt + + println("--- DOCUMENT TO PARSE ---") + println(string(DOC)) + println("--- /DOCUMENT TO PARSE ---\n") + + doc, err := xml.parse(DOC, OPTIONS, FILENAME, Error_Handler) + defer xml.destroy(doc) + + xml.print(doc) + + if err != .None { + printf("Parse error: %v\n", err) + } else { + println("DONE!") + } +} + +main :: proc() { + using fmt + + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + _main() + + if len(track.allocation_map) > 0 { + println() + for _, v in track.allocation_map { + printf("%v Leaked %v bytes.\n", v.location, v.size) + } + } +} \ No newline at end of file diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin new file mode 100644 index 000000000..a63dca5bd --- /dev/null +++ b/core/encoding/xml/tokenizer.odin @@ -0,0 +1,339 @@ +package xml + +import "core:fmt" +import "core:unicode" +import "core:unicode/utf8" + +Error_Handler :: #type proc(pos: Pos, fmt: string, args: ..any) + +Token :: struct { + kind: Token_Kind, + text: string, + pos: Pos, +} + +Pos :: struct { + file: string, + offset: int, // starting at 0 + line: int, // starting at 1 + column: int, // starting at 1 +} + +Token_Kind :: enum { + Invalid, + + Ident, + Literal, + Rune, + String, + + Double_Quote, // " + Single_Quote, // ' + Colon, // : + + Eq, // = + Lt, // < + Gt, // > + Exclaim, // ! + Question, // ? + Hash, // # + Slash, // / + Dash, // - + + Open_Bracket, // [ + Close_Bracket, // ] + + EOF, +} + +CDATA_START :: "" + +Tokenizer :: struct { + // Immutable data + path: string, + src: string, + err: Error_Handler, + + // Tokenizing state + ch: rune, + offset: int, + read_offset: int, + line_offset: int, + line_count: int, + + // Mutable data + error_count: int, +} + +init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = default_error_handler) { + t.src = src + t.err = err + t.ch = ' ' + t.offset = 0 + t.read_offset = 0 + t.line_offset = 0 + t.line_count = len(src) > 0 ? 1 : 0 + t.error_count = 0 + t.path = path + + advance_rune(t) + if t.ch == utf8.RUNE_BOM { + advance_rune(t) + } +} + +@(private) +offset_to_pos :: proc(t: ^Tokenizer, offset: int) -> Pos { + line := t.line_count + column := offset - t.line_offset + 1 + + return Pos { + file = t.path, + offset = offset, + line = line, + column = column, + } +} + +default_error_handler :: proc(pos: Pos, msg: string, args: ..any) { + fmt.eprintf("%s(%d:%d) ", pos.file, pos.line, pos.column) + fmt.eprintf(msg, ..args) + fmt.eprintf("\n") +} + +error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) { + pos := offset_to_pos(t, offset) + if t.err != nil { + t.err(pos, msg, ..args) + } + t.error_count += 1 +} + +advance_rune :: proc(using t: ^Tokenizer) { + if read_offset < len(src) { + offset = read_offset + if ch == '\n' { + line_offset = offset + line_count += 1 + } + r, w := rune(src[read_offset]), 1 + switch { + case r == 0: + error(t, t.offset, "illegal character NUL") + case r >= utf8.RUNE_SELF: + r, w = utf8.decode_rune_in_string(src[read_offset:]) + if r == utf8.RUNE_ERROR && w == 1 { + error(t, t.offset, "illegal UTF-8 encoding") + } else if r == utf8.RUNE_BOM && offset > 0 { + error(t, t.offset, "illegal byte order mark") + } + } + read_offset += w + ch = r + } else { + offset = len(src) + if ch == '\n' { + line_offset = offset + line_count += 1 + } + ch = -1 + } +} + +peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte { + if t.read_offset+offset < len(t.src) { + return t.src[t.read_offset+offset] + } + return 0 +} + +skip_whitespace :: proc(t: ^Tokenizer) { + for { + switch t.ch { + case ' ', '\t', '\r', '\n': + advance_rune(t) + case: + return + } + } +} + +is_letter :: proc(r: rune) -> bool { + if r < utf8.RUNE_SELF { + switch r { + case '_': + return true + case 'A'..='Z', 'a'..='z': + return true + } + } + return unicode.is_letter(r) +} + +is_valid_identifier_rune :: proc(r: rune) -> bool { + if r < utf8.RUNE_SELF { + switch r { + case '_', '-', ':': return true + case 'A'..='Z', 'a'..='z': return true + case '0'..'9': return true + } + } + + if unicode.is_digit(r) || unicode.is_letter(r) { + return true + } + return false +} + +scan_identifier :: proc(t: ^Tokenizer) -> string { + offset := t.offset + namespaced := false + + for is_valid_identifier_rune(t.ch) { + advance_rune(t) + if t.ch == ':' { + /* + A namespaced attr can have at most two parts, `namespace:ident`. + */ + if namespaced { + break + } + namespaced = true + } + } + return string(t.src[offset : t.offset]) +} + +scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false) -> (value: string, err: Error) { + err = .None + in_cdata := false + + loop: for { + ch := t.ch + + switch ch { + case -1: + error(t, t.offset, "[scan_string] Premature end of file.\n") + return "", .Premature_EOF + + case '<': + /* + Might be the start of a CDATA tag. + */ + if t.read_offset + len(CDATA_START) < len(t.src) { + if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START { + in_cdata = true + } + } + + case ']': + /* + Might be the end of a CDATA tag. + */ + if t.read_offset + len(CDATA_END) < len(t.src) { + if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END { + in_cdata = false + } + } + + case '\n': + if !in_cdata { + error(t, offset, string(t.src[offset : t.offset])) + error(t, offset, "[scan_string] Not terminated\n") + err = .Invalid_Tag_Value + break loop + } + } + + if ch == close && !in_cdata { + /* + If it's not a CDATA tag, it's the end of this body. + */ + break loop + } + + advance_rune(t) + } + + lit := string(t.src[offset : t.offset]) + if consume_close { + advance_rune(t) + } + + /* + TODO: Handle decoding escape characters and unboxing CDATA. + */ + + return lit, err +} + +peek :: proc(t: ^Tokenizer) -> (token: Token) { + old := t^ + token = scan(t) + t^ = old + return token +} + +scan :: proc(t: ^Tokenizer) -> Token { + skip_whitespace(t) + + offset := t.offset + + kind: Token_Kind + err: Error + lit: string + pos := offset_to_pos(t, offset) + + switch ch := t.ch; true { + case is_letter(ch): + lit = scan_identifier(t) + kind = .Ident + + case: + advance_rune(t) + switch ch { + case -1: + kind = .EOF + + case '<': kind = .Lt + case '>': kind = .Gt + case '!': kind = .Exclaim + case '?': kind = .Question + case '=': kind = .Eq + case '#': kind = .Hash + case '/': kind = .Slash + case '-': kind = .Dash + case ':': kind = .Colon + + case '"', '\'': + lit, err = scan_string(t, t.offset, ch, true) + if err == .None { + kind = .String + } else { + kind = .Invalid + } + + case '\n': + lit = "\n" + + case '\\': + token := scan(t) + if token.pos.line == pos.line { + error(t, token.pos.offset, "expected a newline after \\") + } + return token + + case: + if ch != utf8.RUNE_BOM { + // error(t, t.offset, "illegal character '%r': %d", ch, ch) + } + kind = .Invalid + } + } + + if lit == "" { + lit = string(t.src[offset : t.offset]) + } + return Token{kind, lit, pos} +} \ No newline at end of file diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin new file mode 100644 index 000000000..526be5856 --- /dev/null +++ b/core/encoding/xml/xml_reader.odin @@ -0,0 +1,651 @@ +package xml +/* + An XML 1.0 / 1.1 parser + + Copyright 2021 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + A from-scratch XML implementation, loosely modelled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816). + + Features: + - Supports enough of the XML 1.0/1.1 spec to handle the 99.9% of XML documents in common current usage. + - Simple to understand and use. Small. + + Caveats: + - We do NOT support HTML in this package, as that may or may not be valid XML. + If it works, great. If it doesn't, that's not considered a bug. + + - We do NOT support UTF-16. If you have a UTF-16 XML file, please convert it to UTF-8 first. Also, our condolences. + - <[!ELEMENT and <[!ATTLIST are not supported, and will be either ignored or return an error depending on the parser options. + + TODO: + - Optional CDATA unboxing. + - Optional `>`, ` `, ` ` and other escape substitution in tag bodies. + + MAYBE: + - XML writer? + - Serialize/deserialize Odin types? + + List of contributors: + Jeroen van Rijn: Initial implementation. +*/ + +import "core:strings" +import "core:mem" +import "core:os" + +DEFAULT_Options :: Options{ + flags = { + .Ignore_Unsupported, + }, + expected_doctype = "", +} + +Option_Flag :: enum { + /* + Document MUST start with ` (doc: ^Document, err: Error) { + context.allocator = allocator + + opts := validate_options(options) or_return + + t := &Tokenizer{} + init(t, string(data), path, error_handler) + + doc = new(Document) + doc.allocator = allocator + doc.tokenizer = t + + strings.intern_init(&doc.intern, allocator, allocator) + + err = .Unexpected_Token + element, parent: ^Element + + /* + If a DOCTYPE is present, the root tag has to match. + If an expected DOCTYPE is given in options (i.e. it's non-empty), the DOCTYPE (if present) and root tag have to match. + */ + expected_doctype := options.expected_doctype + + loop: for { + tok := scan(t) + #partial switch tok.kind { + + case .Lt: + open := scan(t) + #partial switch open.kind { + + case .Question: + /* + 0 { + /* + We've already seen a prolog. + */ + return doc, .Too_Many_Prologs + } else { + error(t, t.offset, "Expected \" 0 { + return doc, .Too_Many_DocTypes + } + if doc.root != nil { + return doc, .DocType_Must_Proceed_Elements + } + parse_doctype(doc) or_return + + if len(expected_doctype) > 0 && expected_doctype != doc.doctype.ident { + error(t, t.offset, "Invalid DOCTYPE. Expected: %v, got: %v\n", expected_doctype, doc.doctype.ident) + return doc, .Invalid_DocType + } + expected_doctype = doc.doctype.ident + + case: + if .Error_on_Unsupported in opts.flags { + error(t, t.offset, "Unhandled: . + The grammar does not allow a comment to end in ---> + */ + if doc.root == nil { + return doc, .Comment_Before_Root_Element + } + + expect(t, .Dash) + offset := t.offset + + for { + advance_rune(t) + ch := t.ch + + /* + A comment ends when we see -->, preceded by a character that's not a dash. + "For compatibility, the string "--" (double-hyphen) must not occur within comments." + + See: https://www.w3.org/TR/2006/REC-xml11-20060816/#dt-comment + + Thanks to the length (4) of the comment start, we also have enough lookback, + and the peek at the next byte asserts that there's at least one more character + that's a `>`. + */ + if ch < 0 { + error(t, offset, "[parse] Comment was not terminated\n") + return doc, .Unclosed_Comment + } + + if string(t.src[t.offset - 1:][:2]) == "--" { + if peek_byte(t) == '>' { + break + } else { + error(t, t.offset - 1, "Invalid -- sequence in comment.\n") + return doc, .Invalid_Sequence_In_Comment + } + } + } + + if .Intern_Comments in opts.flags { + el := new(Element) + + el.parent = element + el.kind = .Comment + el.value = strings.intern_get(&doc.intern, string(t.src[offset : t.offset - 1])) + append(&element.children, el) + } + + expect(t, .Dash) + expect(t, .Gt) + + case: + error(t, t.offset, "Invalid Token after 0 && expected_doctype != open.text { + error(t, t.offset, "Root Tag doesn't match DOCTYPE. Expected: %v, got: %v\n", expected_doctype, open.text) + return doc, .Invalid_DocType + } + } + + /* + One of these should follow: + - `>`, which means we've just opened this tag and expect a later element to close it. + - `/>`, which means this is an 'empty' or self-closing tag. + */ + end_token := scan(t) + + #partial switch end_token.kind { + case .Gt: + /* + We're now the new parent. + */ + parent = element + + case .Slash: + /* + Empty tag? + */ + expect(t, .Gt) or_return + + case: + error(t, t.offset, "Expected close tag, got: %#v\n", end_token) + return + } + + case .Slash: + /* + Close tag. + */ + ident := expect(t, .Ident) or_return + _ = expect(t, .Gt) or_return + + if element.ident != ident.text { + error(t, t.offset, "Mismatched Closing Tag: %v\n", ident.text) + return doc, .Mismatched_Closing_Tag + } + parent = element.parent + element = parent + + case: + error(t, t.offset, "Invalid Token after <: %#v\n", open) + return + } + + case .EOF: + break loop + + case: + /* + This should be a tag's body text. + */ + element.value = scan_string(t, tok.pos.offset) or_return + } + } + + if .Must_Have_Prolog in opts.flags && len(doc.prolog) == 0 { + return doc, .No_Prolog + } + + if .Must_Have_DocType in opts.flags && len(doc.doctype.ident) == 0 { + return doc, .No_DocType + } + + return doc, .None +} + +parse_from_file :: proc(filename: string, options := DEFAULT_Options, error_handler := default_error_handler, allocator := context.allocator) -> (doc: ^Document, err: Error) { + context.allocator = allocator + + data, data_ok := os.read_entire_file(filename) + defer delete(data) + + if !data_ok { return {}, .File_Error } + + return parse_from_slice(data, options, filename, error_handler, allocator) +} + +parse :: proc { parse_from_file, parse_from_slice } + +free_element :: proc(element: ^Element) { + if element == nil { return } + + for child in element.children { + /* + NOTE: Recursive. + + Could be rewritten so it adds them to a list of pointers to free. + */ + free_element(child) + } + delete(element.attribs) + delete(element.children) + free(element) +} + +destroy :: proc(doc: ^Document) { + if doc == nil { return } + + free_element(doc.root) + strings.intern_destroy(&doc.intern) + + delete(doc.prolog) + free(doc) +} + +/* + Helpers. +*/ + +validate_options :: proc(options: Options) -> (validated: Options, err: Error) { + validated = options + + if .Error_on_Unsupported in validated.flags && .Ignore_Unsupported in validated.flags { + return options, .Conflicting_Options + } + + if .Unbox_CDATA in validated.flags { + return options, .Unhandled_CDATA_Unboxing + } + + if .Decode_SGML_Entities in validated.flags { + return options, .Unhandled_SGML_Entity_Decoding + } + + return validated, .None +} + +expect :: proc(t: ^Tokenizer, kind: Token_Kind) -> (tok: Token, err: Error) { + tok = scan(t) + if tok.kind == kind { return tok, .None } + + error(t, t.offset, "Expected \"%v\", got \"%v\".", kind, tok.kind) + return tok, .Unexpected_Token +} + +parse_attribute :: proc(doc: ^Document) -> (attr: Attr, offset: int, err: Error) { + assert(doc != nil) + context.allocator = doc.allocator + t := doc.tokenizer + + key := expect(t, .Ident) or_return + offset = t.offset - len(key.text) + + _ = expect(t, .Eq) or_return + value := expect(t, .String) or_return + + attr.key = strings.intern_get(&doc.intern, key.text) + attr.val = strings.intern_get(&doc.intern, value.text) + + err = .None + return +} + +check_duplicate_attributes :: proc(t: ^Tokenizer, attribs: Attributes, attr: Attr, offset: int) -> (err: Error) { + for a in attribs { + if attr.key == a.key { + error(t, offset, "Duplicate attribute: %v\n", attr.key) + return .Duplicate_Attribute + } + } + return .None +} + +parse_attributes :: proc(doc: ^Document, attribs: ^Attributes) -> (err: Error) { + assert(doc != nil) + context.allocator = doc.allocator + t := doc.tokenizer + + for peek(t).kind == .Ident { + attr, offset := parse_attribute(doc) or_return + check_duplicate_attributes(t, attribs^, attr, offset) or_return + append(attribs, attr) + } + skip_whitespace(t) + return .None +} + +parse_prolog :: proc(doc: ^Document) -> (err: Error) { + assert(doc != nil) + context.allocator = doc.allocator + t := doc.tokenizer + + offset := t.offset + parse_attributes(doc, &doc.prolog) or_return + + for attr in doc.prolog { + switch attr.key { + case "version": + switch attr.val { + case "1.0", "1.1": + case: + error(t, offset, "[parse_prolog] Warning: Unhandled XML version: %v\n", attr.val) + } + + case "encoding": + switch strings.to_lower(attr.val, context.temp_allocator) { + case "utf-8", "utf8": + doc.encoding = .UTF_8 + + case "latin-1", "latin1", "iso-8859-1": + doc.encoding = .LATIN_1 + + case: + /* + Unrecognized encoding, assume UTF-8. + */ + error(t, offset, "[parse_prolog] Warning: Unrecognized encoding: %v\n", attr.val) + } + + case: + // Ignored. + } + } + + _ = expect(t, .Question) or_return + _ = expect(t, .Gt) or_return + + return .None +} + +skip_element :: proc(t: ^Tokenizer) -> (err: Error) { + close := 1 + + loop: for { + tok := scan(t) + #partial switch tok.kind { + case .EOF: + error(t, t.offset, "[skip_element] Premature EOF\n") + return .Premature_EOF + + case .Lt: + close += 1 + + case .Gt: + close -= 1 + if close == 0 { + break loop + } + + case: + + } + } + return .None +} + +parse_doctype :: proc(doc: ^Document) -> (err: Error) { + /* + + + + ]> + */ + assert(doc != nil) + context.allocator = doc.allocator + t := doc.tokenizer + + tok := expect(t, .Ident) or_return + doc.doctype.ident = strings.intern_get(&doc.intern, tok.text) + + skip_whitespace(t) + offset := t.offset + skip_element(t) or_return + + /* + -1 because the current offset is that of the closing tag, so the rest of the DOCTYPE tag ends just before it. + */ + doc.doctype.rest = strings.intern_get(&doc.intern, string(t.src[offset : t.offset - 1])) + return .None +} \ No newline at end of file diff --git a/tests/core/Makefile b/tests/core/Makefile index 0f0ffe4d6..e17dede90 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -1,22 +1,29 @@ ODIN=../../odin PYTHON=$(shell which python3) -all: download_test_assets image_test compress_test strings_test hash_test crypto_test +all: download_test_assets image_test compress_test strings_test hash_test crypto_test encoding_test download_test_assets: $(PYTHON) download_assets.py image_test: - $(ODIN) run image/test_core_image.odin + $(ODIN) run image/test_core_image.odin -out=test_image -o:speed -no-bounds-check compress_test: - $(ODIN) run compress/test_core_compress.odin + $(ODIN) run compress/test_core_compress.odin -out=test_compress -o:speed -no-bounds-check strings_test: - $(ODIN) run strings/test_core_strings.odin + $(ODIN) run strings/test_core_strings.odin -out=test_strings -o:speed -no-bounds-check + +odin_test: + $(ODIN) run odin -out=test_odin -o:speed -no-bounds-check hash_test: $(ODIN) run hash -out=test_hash -o:speed -no-bounds-check crypto_test: - $(ODIN) run crypto -out=crypto_hash -o:speed -no-bounds-check \ No newline at end of file + $(ODIN) run crypto -out=crypto_hash -o:speed -no-bounds-check + +encoding_test: + $(ODIN) run encoding/json -out=test_encoding_json -o:speed -no-bounds-check + $(ODIN) run encoding/xml -out=test_encoding_xml -o:speed -no-bounds-check diff --git a/tests/core/assets/xml/nl_NL-qt-ts.ts b/tests/core/assets/xml/nl_NL-qt-ts.ts new file mode 100644 index 000000000..6ec3f2f47 --- /dev/null +++ b/tests/core/assets/xml/nl_NL-qt-ts.ts @@ -0,0 +1,35 @@ + + + + + Page + + Text for translation + commenting + Tekst om te vertalen + + + Also text to translate + some text + Ook tekst om te vertalen + + + + installscript + + 99 bottles of beer on the wall + some new comments here + 99 flessen bier op de muur + + + + apple_count + + %d apple(s) + + %d appel + %d appels + + + + diff --git a/tests/core/assets/xml/nl_NL-xliff-1.0.xliff b/tests/core/assets/xml/nl_NL-xliff-1.0.xliff new file mode 100644 index 000000000..7a1abcd66 --- /dev/null +++ b/tests/core/assets/xml/nl_NL-xliff-1.0.xliff @@ -0,0 +1,38 @@ + + + + + + text + tekst + Context + + + text 1 + tekst 1 + Context 1 + + + text 2 + + Context of the segment 2 + + + text 3 + translation 3 + Context 3 + + + Plurals + + %d month + %d maand + + + %d months + %d maanden + + + + + diff --git a/tests/core/assets/xml/nl_NL-xliff-2.0.xliff b/tests/core/assets/xml/nl_NL-xliff-2.0.xliff new file mode 100644 index 000000000..611ac80c4 --- /dev/null +++ b/tests/core/assets/xml/nl_NL-xliff-2.0.xliff @@ -0,0 +1,52 @@ + + + + + Note for file + + + + Note for unit + + + text + + + + + + Note for unit 2 + + + text 2 + translation 2 + + + + + Note for unit 3 + + + text 3 + approved translation 3 + + + + + + Plurals + + + %d month + %d maand + + + + + %d months + %d maanden + + + + + \ No newline at end of file diff --git a/tests/core/assets/xml/utf8.xml b/tests/core/assets/xml/utf8.xml new file mode 100644 index 000000000..c9ed3bf69 --- /dev/null +++ b/tests/core/assets/xml/utf8.xml @@ -0,0 +1,8 @@ + + +<恥ずべきフクロウ 올빼미_id="Foozle Hello, world!"]]>Barzle"> +<부끄러운:barzle> + ရှက်စရာ ဇီးကွက် + Owl of Shame + More CDATA Hello, world! Nonsense. + \ No newline at end of file diff --git a/tests/core/build.bat b/tests/core/build.bat index 176b7f175..7a214acc9 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -5,34 +5,35 @@ python3 download_assets.py echo --- echo Running core:image tests echo --- -%PATH_TO_ODIN% run image %COMMON% +%PATH_TO_ODIN% run image %COMMON% -out:test_image.exe echo --- echo Running core:compress tests echo --- -%PATH_TO_ODIN% run compress %COMMON% +%PATH_TO_ODIN% run compress %COMMON% -out:test_compress.exe echo --- echo Running core:strings tests echo --- -%PATH_TO_ODIN% run strings %COMMON% +%PATH_TO_ODIN% run strings %COMMON% -out:test_strings.exe echo --- echo Running core:hash tests echo --- -%PATH_TO_ODIN% run hash %COMMON% -o:size +%PATH_TO_ODIN% run hash %COMMON% -o:size -out:test_hash.exe echo --- echo Running core:odin tests echo --- -%PATH_TO_ODIN% run odin %COMMON% -o:size +%PATH_TO_ODIN% run odin %COMMON% -o:size -out:test_odin.exe echo --- echo Running core:crypto hash tests echo --- -%PATH_TO_ODIN% run crypto %COMMON% +%PATH_TO_ODIN% run crypto %COMMON% -o:speed -out:test_crypto.exe echo --- echo Running core:encoding tests echo --- -%PATH_TO_ODIN% run encoding %COMMON% \ No newline at end of file +%PATH_TO_ODIN% run encoding\json %COMMON% -out:test_json.exe +%PATH_TO_ODIN% run encoding\xml %COMMON% -out:test_xml.exe \ No newline at end of file diff --git a/tests/core/encoding/test_core_json.odin b/tests/core/encoding/json/test_core_json.odin similarity index 63% rename from tests/core/encoding/test_core_json.odin rename to tests/core/encoding/json/test_core_json.odin index f536eb4c6..4f415c008 100644 --- a/tests/core/encoding/test_core_json.odin +++ b/tests/core/encoding/json/test_core_json.odin @@ -8,32 +8,32 @@ TEST_count := 0 TEST_fail := 0 when ODIN_TEST { - expect :: testing.expect - log :: testing.log + expect :: testing.expect + log :: testing.log } else { - expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) - TEST_count += 1 - if !condition { - TEST_fail += 1 - fmt.println(message) - return - } - fmt.println(" PASS") - } - log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) - } + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + fmt.printf("[%v] ", loc) + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.println(message) + return + } + fmt.println(" PASS") + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] ", loc) + fmt.printf("log: %v\n", v) + } } main :: proc() { - t := testing.T{} + t := testing.T{} parse_json(&t) marshal_json(&t) - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) } @test diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin new file mode 100644 index 000000000..7eefac212 --- /dev/null +++ b/tests/core/encoding/xml/test_core_xml.odin @@ -0,0 +1,264 @@ +package test_core_xml + +import "core:encoding/xml" +import "core:testing" +import "core:mem" +import "core:fmt" + +Silent :: proc(pos: xml.Pos, fmt: string, args: ..any) { + // Custom (silent) error handler. +} + +OPTIONS :: xml.Options{ + flags = { + .Ignore_Unsupported, .Intern_Comments, + }, + expected_doctype = "", +} + +TEST_count := 0 +TEST_fail := 0 + +TEST :: struct { + filename: string, + options: xml.Options, + expected: struct { + error: xml.Error, + xml_version: string, + xml_encoding: string, + doctype: string, + }, +} + +TESTS :: []TEST{ + /* + First we test that certain files parse without error. + */ + { + filename = "assets/xml/utf8.xml", + options = OPTIONS, + expected = { + error = .None, + xml_version = "1.0", + xml_encoding = "utf-8", + doctype = "恥ずべきフクロウ", + }, + }, + { + filename = "assets/xml/nl_NL-qt-ts.ts", + options = OPTIONS, + expected = { + error = .None, + xml_version = "1.0", + xml_encoding = "utf-8", + doctype = "TS", + }, + }, + { + filename = "assets/xml/nl_NL-xliff-1.0.xliff", + options = OPTIONS, + expected = { + error = .None, + xml_version = "1.0", + xml_encoding = "UTF-8", + doctype = "", + }, + }, + { + filename = "assets/xml/nl_NL-xliff-2.0.xliff", + options = OPTIONS, + expected = { + error = .None, + xml_version = "1.0", + xml_encoding = "utf-8", + doctype = "", + }, + }, + + /* + Then we test that certain errors are returned as expected. + */ + { + filename = "assets/xml/utf8.xml", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, + }, + expected_doctype = "Odin", + }, + expected = { + error = .Invalid_DocType, + xml_version = "1.0", + xml_encoding = "utf-8", + doctype = "恥ずべきフクロウ", + }, + }, +} + +when ODIN_TEST { + expect :: testing.expect + log :: testing.log +} else { + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + fmt.printf("[%v] ", loc) + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.println(message) + return + } + fmt.println(" PASS") + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] ", loc) + fmt.printf("log: %v\n", v) + } +} + +main :: proc() { + t := testing.T{} + + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + run_tests(&t) + + if len(track.allocation_map) > 0 { + for _, v in track.allocation_map { + err_msg := fmt.tprintf("%v Leaked %v bytes.", v.location, v.size) + expect(&t, false, err_msg) + } + } + + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) +} + +@test +run_tests :: proc(t: ^testing.T) { + using fmt + + count := 0 + + for test in TESTS { + printf("Trying to parse %v\n\n", test.filename) + + doc, err := xml.parse(test.filename, test.options, Silent) + defer xml.destroy(doc) + + err_msg := tprintf("Expected return value %v, got %v", test.expected.error, err) + expect(t, err == test.expected.error, err_msg) + + if len(test.expected.xml_version) > 0 { + xml_version := "" + for attr in doc.prolog { + if attr.key == "version" { + xml_version = attr.val + } + } + + err_msg = tprintf("Expected XML version %v, got %v", test.expected.xml_version, xml_version) + expect(t, xml_version == test.expected.xml_version, err_msg) + } + + if len(test.expected.xml_encoding) > 0 { + xml_encoding := "" + for attr in doc.prolog { + if attr.key == "encoding" { + xml_encoding = attr.val + } + } + + err_msg = tprintf("Expected XML encoding %v, got %v", test.expected.xml_encoding, xml_encoding) + expect(t, xml_encoding == test.expected.xml_encoding, err_msg) + } + + err_msg = tprintf("Expected DOCTYPE %v, got %v", test.expected.doctype, doc.doctype.ident) + expect(t, doc.doctype.ident == test.expected.doctype, err_msg) + + /* + File-specific tests. + */ + switch count { + case 0: + expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.") + attr := doc.root.attribs[0] + + attr_key_expected := "올빼미_id" + attr_val_expected := "Foozle Hello, world!\"]]>Barzle" + + attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) + expect(t, attr.key == attr_key_expected, attr_err) + + attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) + expect(t, attr.val == attr_val_expected, attr_err) + + expect(t, len(doc.root.children) > 0, "Expected the root tag to have children.") + child := doc.root.children[0] + + first_child_ident := "부끄러운:barzle" + attr_err = tprintf("Expected first child tag's ident to be %v, got %v", first_child_ident, child.ident) + expect(t, child.ident == first_child_ident, attr_err) + + case 2: + expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.") + + { + attr := doc.root.attribs[0] + + attr_key_expected := "version" + attr_val_expected := "1.2" + + attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) + expect(t, attr.key == attr_key_expected, attr_err) + + attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) + expect(t, attr.val == attr_val_expected, attr_err) + } + + { + attr := doc.root.attribs[1] + + attr_key_expected := "xmlns" + attr_val_expected := "urn:oasis:names:tc:xliff:document:1.2" + + attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) + expect(t, attr.key == attr_key_expected, attr_err) + + attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) + expect(t, attr.val == attr_val_expected, attr_err) + } + + case 3: + expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.") + + { + attr := doc.root.attribs[0] + + attr_key_expected := "xmlns" + attr_val_expected := "urn:oasis:names:tc:xliff:document:2.0" + + attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) + expect(t, attr.key == attr_key_expected, attr_err) + + attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) + expect(t, attr.val == attr_val_expected, attr_err) + } + + { + attr := doc.root.attribs[1] + + attr_key_expected := "version" + attr_val_expected := "2.0" + + attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) + expect(t, attr.key == attr_key_expected, attr_err) + + attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) + expect(t, attr.val == attr_val_expected, attr_err) + } + } + + count += 1 + } +} \ No newline at end of file From 9b2e67df67b5f801687f3c48c6701daa00c38088 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 30 Nov 2021 23:43:51 +0100 Subject: [PATCH 02/79] [xml] Small cleanup. --- core/encoding/xml/tokenizer.odin | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin index a63dca5bd..3dcffb0d6 100644 --- a/core/encoding/xml/tokenizer.odin +++ b/core/encoding/xml/tokenizer.odin @@ -180,7 +180,7 @@ is_valid_identifier_rune :: proc(r: rune) -> bool { } } - if unicode.is_digit(r) || unicode.is_letter(r) { + if unicode.is_letter(r) || unicode.is_digit(r) { return true } return false @@ -317,17 +317,7 @@ scan :: proc(t: ^Tokenizer) -> Token { case '\n': lit = "\n" - case '\\': - token := scan(t) - if token.pos.line == pos.line { - error(t, token.pos.offset, "expected a newline after \\") - } - return token - case: - if ch != utf8.RUNE_BOM { - // error(t, t.offset, "illegal character '%r': %d", ch, ch) - } kind = .Invalid } } From 46a4927acad674b3265969bd5bde591b480d0c73 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 1 Dec 2021 00:32:35 +0100 Subject: [PATCH 03/79] [xml] Use `io.Writer` for `xml.print(doc)`. --- core/encoding/xml/debug_print.odin | 51 ++++++++++++---------- core/encoding/xml/example/xml_example.odin | 8 +++- core/encoding/xml/xml_reader.odin | 2 +- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/core/encoding/xml/debug_print.odin b/core/encoding/xml/debug_print.odin index 0b7ffa822..be1175cbc 100644 --- a/core/encoding/xml/debug_print.odin +++ b/core/encoding/xml/debug_print.odin @@ -10,64 +10,69 @@ package xml List of contributors: Jeroen van Rijn: Initial implementation. */ +import "core:io" import "core:fmt" /* Just for debug purposes. */ -print :: proc(doc: ^Document) { - assert(doc != nil) - +print :: proc(writer: io.Writer, doc: ^Document) -> (written: int, err: io.Error) { + if doc == nil { return } using fmt - println("[XML Prolog]") + + written += wprintf(writer, "[XML Prolog]\n") for attr in doc.prolog { - printf("\t%v: %v\n", attr.key, attr.val) + written += wprintf(writer, "\t%v: %v\n", attr.key, attr.val) } - printf("[Encoding] %v\n", doc.encoding) - printf("[DOCTYPE] %v\n", doc.doctype.ident) + written += wprintf(writer, "[Encoding] %v\n", doc.encoding) + written += wprintf(writer, "[DOCTYPE] %v\n", doc.doctype.ident) if len(doc.doctype.rest) > 0 { - printf("\t%v\n", doc.doctype.rest) + wprintf(writer, "\t%v\n", doc.doctype.rest) } if doc.root != nil { - println(" --- ") - print_element(0, doc.root) - println(" --- ") - } + wprintln(writer, " --- ") + print_element(writer, doc.root) + wprintln(writer, " --- ") + } + + return written, .None } -print_element :: proc(indent: int, element: ^Element) { +print_element :: proc(writer: io.Writer, element: ^Element, indent := 0) -> (written: int, err: io.Error) { if element == nil { return } using fmt - tab :: proc(indent: int) { + tab :: proc(writer: io.Writer, indent: int) { tabs := "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" i := max(0, min(indent, len(tabs))) - printf("%v", tabs[:i]) + wprintf(writer, "%v", tabs[:i]) } - tab(indent) + tab(writer, indent) if element.kind == .Element { - printf("<%v>\n", element.ident) + wprintf(writer, "<%v>\n", element.ident) if len(element.value) > 0 { - tab(indent + 1) - printf("[Value] %v\n", element.value) + tab(writer, indent + 1) + wprintf(writer, "[Value] %v\n", element.value) } for attr in element.attribs { - tab(indent + 1) - printf("[Attr] %v: %v\n", attr.key, attr.val) + tab(writer, indent + 1) + wprintf(writer, "[Attr] %v: %v\n", attr.key, attr.val) } for child in element.children { - print_element(indent + 1, child) + print_element(writer, child, indent + 1) } } else if element.kind == .Comment { - printf("[COMMENT] %v\n", element.value) + wprintf(writer, "[COMMENT] %v\n", element.value) } + + return written, .None } \ No newline at end of file diff --git a/core/encoding/xml/example/xml_example.odin b/core/encoding/xml/example/xml_example.odin index 24a277de6..82938c223 100644 --- a/core/encoding/xml/example/xml_example.odin +++ b/core/encoding/xml/example/xml_example.odin @@ -2,6 +2,7 @@ package xml_example import "core:encoding/xml" import "core:mem" +import "core:strings" import "core:fmt" Error_Handler :: proc(pos: xml.Pos, fmt: string, args: ..any) { @@ -28,7 +29,12 @@ _main :: proc() { doc, err := xml.parse(DOC, OPTIONS, FILENAME, Error_Handler) defer xml.destroy(doc) - xml.print(doc) + buf: strings.Builder + defer strings.destroy_builder(&buf) + w := strings.to_writer(&buf) + + xml.print(w, doc) + println(strings.to_string(buf)) if err != .None { printf("Parse error: %v\n", err) diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 526be5856..34f6e65d0 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -75,6 +75,7 @@ Option_Flag :: enum { */ Decode_SGML_Entities, } +Option_Flags :: bit_set[Option_Flag; u8] Document :: struct { root: ^Element, @@ -122,7 +123,6 @@ Options :: struct { flags: Option_Flags, expected_doctype: string, } -Option_Flags :: bit_set[Option_Flag] Encoding :: enum { Unknown, From 682783a2aabad34e838493bb1e4c2437fd13058a Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 1 Dec 2021 00:43:22 +0100 Subject: [PATCH 04/79] [xml] Tab indentation in debug printer. --- core/encoding/xml/debug_print.odin | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/encoding/xml/debug_print.odin b/core/encoding/xml/debug_print.odin index be1175cbc..c4d6875cc 100644 --- a/core/encoding/xml/debug_print.odin +++ b/core/encoding/xml/debug_print.odin @@ -47,10 +47,9 @@ print_element :: proc(writer: io.Writer, element: ^Element, indent := 0) -> (wri using fmt tab :: proc(writer: io.Writer, indent: int) { - tabs := "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t" - - i := max(0, min(indent, len(tabs))) - wprintf(writer, "%v", tabs[:i]) + for _ in 0..=indent { + wprintf(writer, "\t") + } } tab(writer, indent) From 32eab04d662b0c1128e64a4b91fb81f5f2be5a95 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 1 Dec 2021 03:15:44 +0100 Subject: [PATCH 05/79] [xml] Allow multi-line bodies w/o CDATA. Strip trailing whitespace. --- core/encoding/xml/debug_print.odin | 9 ++++++--- core/encoding/xml/tokenizer.odin | 21 ++++++++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/core/encoding/xml/debug_print.odin b/core/encoding/xml/debug_print.odin index c4d6875cc..65b71e30b 100644 --- a/core/encoding/xml/debug_print.odin +++ b/core/encoding/xml/debug_print.odin @@ -27,10 +27,13 @@ print :: proc(writer: io.Writer, doc: ^Document) -> (written: int, err: io.Error } written += wprintf(writer, "[Encoding] %v\n", doc.encoding) - written += wprintf(writer, "[DOCTYPE] %v\n", doc.doctype.ident) - if len(doc.doctype.rest) > 0 { - wprintf(writer, "\t%v\n", doc.doctype.rest) + if len(doc.doctype.ident) > 0 { + written += wprintf(writer, "[DOCTYPE] %v\n", doc.doctype.ident) + + if len(doc.doctype.rest) > 0 { + wprintf(writer, "\t%v\n", doc.doctype.rest) + } } if doc.root != nil { diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin index 3dcffb0d6..e453552b8 100644 --- a/core/encoding/xml/tokenizer.odin +++ b/core/encoding/xml/tokenizer.odin @@ -205,7 +205,7 @@ scan_identifier :: proc(t: ^Tokenizer) -> string { return string(t.src[offset : t.offset]) } -scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false) -> (value: string, err: Error) { +scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false, multiline := true) -> (value: string, err: Error) { err = .None in_cdata := false @@ -238,7 +238,7 @@ scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close } case '\n': - if !in_cdata { + if !(multiline || in_cdata) { error(t, offset, string(t.src[offset : t.offset])) error(t, offset, "[scan_string] Not terminated\n") err = .Invalid_Tag_Value @@ -256,7 +256,22 @@ scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close advance_rune(t) } + /* + Strip trailing whitespace. + */ lit := string(t.src[offset : t.offset]) + + end := len(lit) + eat: for ; end > 0; end -= 1 { + ch := lit[end - 1] + switch ch { + case ' ', '\t', '\r', '\n': + case: + break eat + } + } + lit = lit[:end] + if consume_close { advance_rune(t) } @@ -307,7 +322,7 @@ scan :: proc(t: ^Tokenizer) -> Token { case ':': kind = .Colon case '"', '\'': - lit, err = scan_string(t, t.offset, ch, true) + lit, err = scan_string(t, t.offset, ch, true, false) if err == .None { kind = .String } else { From ec63d0bbd21aa3d3f33cd762bd656ea8eb0af4a6 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 1 Dec 2021 15:30:36 +0100 Subject: [PATCH 06/79] [xml] Robustness improvement. Can now parse https://www.w3.org/2003/entities/2007xml/unicode.xml no problem. --- core/encoding/xml/debug_print.odin | 4 ++ core/encoding/xml/xml_reader.odin | 73 ++++++++++++++++++++---------- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/core/encoding/xml/debug_print.odin b/core/encoding/xml/debug_print.odin index 65b71e30b..e6a8c9433 100644 --- a/core/encoding/xml/debug_print.odin +++ b/core/encoding/xml/debug_print.odin @@ -36,6 +36,10 @@ print :: proc(writer: io.Writer, doc: ^Document) -> (written: int, err: io.Error } } + for comment in doc.comments { + written += wprintf(writer, "[Pre-root comment] %v\n", comment) + } + if doc.root != nil { wprintln(writer, " --- ") print_element(writer, doc.root) diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 34f6e65d0..b2226e6b9 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -86,10 +86,16 @@ Document :: struct { /* We only scan the . The grammar does not allow a comment to end in ---> */ - if doc.root == nil { - return doc, .Comment_Before_Root_Element - } - expect(t, .Dash) offset := t.offset @@ -329,12 +339,17 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err } if .Intern_Comments in opts.flags { - el := new(Element) + comment := strings.intern_get(&doc.intern, string(t.src[offset : t.offset - 1])) - el.parent = element - el.kind = .Comment - el.value = strings.intern_get(&doc.intern, string(t.src[offset : t.offset - 1])) - append(&element.children, el) + if doc.root == nil { + append(&doc.comments, comment) + } else { + el := new(Element) + el.parent = element + el.kind = .Comment + el.value = comment + append(&element.children, el) + } } expect(t, .Dash) @@ -350,6 +365,7 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err e.g. `, which means this is an 'empty' or self-closing tag. */ end_token := scan(t) - #partial switch end_token.kind { case .Gt: /* @@ -394,9 +409,12 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err case .Slash: /* - Empty tag? + Empty tag. Close it. */ expect(t, .Gt) or_return + parent = element.parent + element = parent + tag_is_open = false case: error(t, t.offset, "Expected close tag, got: %#v\n", end_token) @@ -411,25 +429,33 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err _ = expect(t, .Gt) or_return if element.ident != ident.text { - error(t, t.offset, "Mismatched Closing Tag: %v\n", ident.text) + error(t, t.offset, "Mismatched Closing Tag. Expected %v, got %v\n", element.ident, ident.text) return doc, .Mismatched_Closing_Tag } - parent = element.parent - element = parent + parent = element.parent + element = parent + tag_is_open = false case: error(t, t.offset, "Invalid Token after <: %#v\n", open) return } - case .EOF: + case -1: + /* + End of file. + */ + if tag_is_open { + return doc, .Premature_EOF + } break loop case: /* This should be a tag's body text. */ - element.value = scan_string(t, tok.pos.offset) or_return + body_text := scan_string(t, t.offset) or_return + element.value = strings.intern_get(&doc.intern, body_text) } } @@ -480,6 +506,7 @@ destroy :: proc(doc: ^Document) { strings.intern_destroy(&doc.intern) delete(doc.prolog) + delete(doc.comments) free(doc) } From beff90e1d12391e63cd1119023f8565eda97593e Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 1 Dec 2021 18:02:48 +0100 Subject: [PATCH 07/79] [xml] Slight optimization. About a 5% speed bump. More rigorous optimization later. --- core/encoding/xml/tokenizer.odin | 62 ++++++++++++++++++------------- core/encoding/xml/xml_reader.odin | 4 +- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin index e453552b8..9247d2531 100644 --- a/core/encoding/xml/tokenizer.odin +++ b/core/encoding/xml/tokenizer.odin @@ -110,44 +110,51 @@ error :: proc(t: ^Tokenizer, offset: int, msg: string, args: ..any) { t.error_count += 1 } +@(optimization_mode="speed") advance_rune :: proc(using t: ^Tokenizer) { - if read_offset < len(src) { - offset = read_offset - if ch == '\n' { - line_offset = offset - line_count += 1 - } - r, w := rune(src[read_offset]), 1 - switch { - case r == 0: - error(t, t.offset, "illegal character NUL") - case r >= utf8.RUNE_SELF: - r, w = utf8.decode_rune_in_string(src[read_offset:]) - if r == utf8.RUNE_ERROR && w == 1 { - error(t, t.offset, "illegal UTF-8 encoding") - } else if r == utf8.RUNE_BOM && offset > 0 { - error(t, t.offset, "illegal byte order mark") + #no_bounds_check { + /* + Already bounds-checked here. + */ + if read_offset < len(src) { + offset = read_offset + if ch == '\n' { + line_offset = offset + line_count += 1 } + r, w := rune(src[read_offset]), 1 + switch { + case r == 0: + error(t, t.offset, "illegal character NUL") + case r >= utf8.RUNE_SELF: + r, w = #force_inline utf8.decode_rune_in_string(src[read_offset:]) + if r == utf8.RUNE_ERROR && w == 1 { + error(t, t.offset, "illegal UTF-8 encoding") + } else if r == utf8.RUNE_BOM && offset > 0 { + error(t, t.offset, "illegal byte order mark") + } + } + read_offset += w + ch = r + } else { + offset = len(src) + if ch == '\n' { + line_offset = offset + line_count += 1 + } + ch = -1 } - read_offset += w - ch = r - } else { - offset = len(src) - if ch == '\n' { - line_offset = offset - line_count += 1 - } - ch = -1 } } peek_byte :: proc(t: ^Tokenizer, offset := 0) -> byte { if t.read_offset+offset < len(t.src) { - return t.src[t.read_offset+offset] + #no_bounds_check return t.src[t.read_offset+offset] } return 0 } +@(optimization_mode="speed") skip_whitespace :: proc(t: ^Tokenizer) { for { switch t.ch { @@ -159,6 +166,7 @@ skip_whitespace :: proc(t: ^Tokenizer) { } } +@(optimization_mode="speed") is_letter :: proc(r: rune) -> bool { if r < utf8.RUNE_SELF { switch r { @@ -177,6 +185,7 @@ is_valid_identifier_rune :: proc(r: rune) -> bool { case '_', '-', ':': return true case 'A'..='Z', 'a'..='z': return true case '0'..'9': return true + case -1: return false } } @@ -205,6 +214,7 @@ scan_identifier :: proc(t: ^Tokenizer) -> string { return string(t.src[offset : t.offset]) } +@(optimization_mode="speed") scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false, multiline := true) -> (value: string, err: Error) { err = .None in_cdata := false diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index b2226e6b9..35dd83b3f 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -239,7 +239,7 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err /* Consume peeked `<` */ - tok := scan(t) + advance_rune(t) open := scan(t) #partial switch open.kind { @@ -265,7 +265,7 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err skip_element(t) or_return } case: - error(t, t.offset, "Expected \" Date: Wed, 1 Dec 2021 22:01:19 +0100 Subject: [PATCH 08/79] [xml] Improve CDATA + comment handling in tag body. --- core/encoding/xml/tokenizer.odin | 117 ++++++++++++++++++++++++------ core/encoding/xml/xml_reader.odin | 36 +-------- 2 files changed, 96 insertions(+), 57 deletions(-) diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin index 9247d2531..95024518d 100644 --- a/core/encoding/xml/tokenizer.odin +++ b/core/encoding/xml/tokenizer.odin @@ -46,8 +46,11 @@ Token_Kind :: enum { EOF, } -CDATA_START :: "" +CDATA_START :: "" + +COMMENT_START :: "" Tokenizer :: struct { // Immutable data @@ -214,10 +217,83 @@ scan_identifier :: proc(t: ^Tokenizer) -> string { return string(t.src[offset : t.offset]) } +/* + A comment ends when we see -->, preceded by a character that's not a dash. + "For compatibility, the string "--" (double-hyphen) must not occur within comments." + + See: https://www.w3.org/TR/2006/REC-xml11-20060816/#dt-comment + + Thanks to the length (4) of the comment start, we also have enough lookback, + and the peek at the next byte asserts that there's at least one more character + that's a `>`. +*/ +scan_comment :: proc(t: ^Tokenizer) -> (comment: string, err: Error) { + offset := t.offset + + for { + advance_rune(t) + ch := t.ch + + if ch < 0 { + error(t, offset, "[parse] Comment was not terminated\n") + return "", .Unclosed_Comment + } + + if string(t.src[t.offset - 1:][:2]) == "--" { + if peek_byte(t) == '>' { + break + } else { + error(t, t.offset - 1, "Invalid -- sequence in comment.\n") + return "", .Invalid_Sequence_In_Comment + } + } + } + + expect(t, .Dash) + expect(t, .Gt) + + return string(t.src[offset : t.offset - 1]), .None +} + +/* + Skip CDATA +*/ +skip_cdata :: proc(t: ^Tokenizer) -> (err: Error) { + if t.read_offset + len(CDATA_START) >= len(t.src) { + /* + Can't be the start of a CDATA tag. + */ + return .None + } + + if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START { + t.read_offset += len(CDATA_START) + offset := t.offset + + cdata_scan: for { + advance_rune(t) + if t.ch < 0 { + error(t, offset, "[scan_string] CDATA was not terminated\n") + return .Premature_EOF + } + + /* + Scan until the end of a CDATA tag. + */ + if t.read_offset + len(CDATA_END) < len(t.src) { + if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END { + t.read_offset += len(CDATA_END) + break cdata_scan + } + } + } + } + return +} + @(optimization_mode="speed") scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close := false, multiline := true) -> (value: string, err: Error) { err = .None - in_cdata := false loop: for { ch := t.ch @@ -228,27 +304,23 @@ scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close return "", .Premature_EOF case '<': - /* - Might be the start of a CDATA tag. - */ - if t.read_offset + len(CDATA_START) < len(t.src) { - if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START { - in_cdata = true - } - } - - case ']': - /* - Might be the end of a CDATA tag. - */ - if t.read_offset + len(CDATA_END) < len(t.src) { - if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END { - in_cdata = false + if peek_byte(t) == '!' { + if peek_byte(t, 1) == '[' { + /* + Might be the start of a CDATA tag. + */ + skip_cdata(t) or_return + } else if peek_byte(t, 1) == '-' && peek_byte(t, 2) == '-' { + /* + Comment start. Eat comment. + */ + t.read_offset += 3 + _ = scan_comment(t) or_return } } case '\n': - if !(multiline || in_cdata) { + if !multiline { error(t, offset, string(t.src[offset : t.offset])) error(t, offset, "[scan_string] Not terminated\n") err = .Invalid_Tag_Value @@ -256,13 +328,12 @@ scan_string :: proc(t: ^Tokenizer, offset: int, close: rune = '<', consume_close } } - if ch == close && !in_cdata { + if t.ch == close { /* - If it's not a CDATA tag, it's the end of this body. + If it's not a CDATA or comment, it's the end of this body. */ break loop } - advance_rune(t) } diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 35dd83b3f..146c278cb 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -307,39 +307,10 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err The grammar does not allow a comment to end in ---> */ expect(t, .Dash) - offset := t.offset - - for { - advance_rune(t) - ch := t.ch - - /* - A comment ends when we see -->, preceded by a character that's not a dash. - "For compatibility, the string "--" (double-hyphen) must not occur within comments." - - See: https://www.w3.org/TR/2006/REC-xml11-20060816/#dt-comment - - Thanks to the length (4) of the comment start, we also have enough lookback, - and the peek at the next byte asserts that there's at least one more character - that's a `>`. - */ - if ch < 0 { - error(t, offset, "[parse] Comment was not terminated\n") - return doc, .Unclosed_Comment - } - - if string(t.src[t.offset - 1:][:2]) == "--" { - if peek_byte(t) == '>' { - break - } else { - error(t, t.offset - 1, "Invalid -- sequence in comment.\n") - return doc, .Invalid_Sequence_In_Comment - } - } - } + comment := scan_comment(t) or_return if .Intern_Comments in opts.flags { - comment := strings.intern_get(&doc.intern, string(t.src[offset : t.offset - 1])) + comment = strings.intern_get(&doc.intern, comment) if doc.root == nil { append(&doc.comments, comment) @@ -352,9 +323,6 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err } } - expect(t, .Dash) - expect(t, .Gt) - case: error(t, t.offset, "Invalid Token after Date: Thu, 2 Dec 2021 18:00:29 +0100 Subject: [PATCH 09/79] [xml] Improvements. --- core/encoding/xml/example/xml_example.odin | 71 +++++++++++-------- core/encoding/xml/helpers.odin | 49 +++++++++++++ core/encoding/xml/tokenizer.odin | 6 +- core/encoding/xml/xml_reader.odin | 2 + tests/core/assets/XML/.gitignore | 2 + tests/core/assets/{xml => XML}/nl_NL-qt-ts.ts | 0 .../assets/{xml => XML}/nl_NL-xliff-1.0.xliff | 0 .../assets/{xml => XML}/nl_NL-xliff-2.0.xliff | 0 tests/core/assets/{xml => XML}/utf8.xml | 0 tests/core/download_assets.py | 43 ++++++++--- tests/core/encoding/xml/test_core_xml.odin | 10 +-- 11 files changed, 137 insertions(+), 46 deletions(-) create mode 100644 core/encoding/xml/helpers.odin create mode 100644 tests/core/assets/XML/.gitignore rename tests/core/assets/{xml => XML}/nl_NL-qt-ts.ts (100%) rename tests/core/assets/{xml => XML}/nl_NL-xliff-1.0.xliff (100%) rename tests/core/assets/{xml => XML}/nl_NL-xliff-2.0.xliff (100%) rename tests/core/assets/{xml => XML}/utf8.xml (100%) diff --git a/core/encoding/xml/example/xml_example.odin b/core/encoding/xml/example/xml_example.odin index 82938c223..085252e92 100644 --- a/core/encoding/xml/example/xml_example.odin +++ b/core/encoding/xml/example/xml_example.odin @@ -1,45 +1,55 @@ package xml_example import "core:encoding/xml" +import "core:os" +import "core:path" import "core:mem" -import "core:strings" import "core:fmt" -Error_Handler :: proc(pos: xml.Pos, fmt: string, args: ..any) { +/* + Silent error handler for the parser. +*/ +Error_Handler :: proc(pos: xml.Pos, fmt: string, args: ..any) {} -} +OPTIONS :: xml.Options{ flags = { .Ignore_Unsupported, }, expected_doctype = "unicode", } -FILENAME :: "../../../../tests/core/assets/xml/nl_NL-xliff-1.0.xliff" -DOC :: #load(FILENAME) - -OPTIONS :: xml.Options{ - flags = { - .Ignore_Unsupported, .Intern_Comments, - }, - expected_doctype = "", -} - -_main :: proc() { +example :: proc() { using fmt - println("--- DOCUMENT TO PARSE ---") - println(string(DOC)) - println("--- /DOCUMENT TO PARSE ---\n") + filename := path.join(ODIN_ROOT, "tests", "core", "assets", "XML", "unicode.xml") + defer delete(filename) - doc, err := xml.parse(DOC, OPTIONS, FILENAME, Error_Handler) + doc, err := xml.parse(filename, OPTIONS, Error_Handler) defer xml.destroy(doc) - buf: strings.Builder - defer strings.destroy_builder(&buf) - w := strings.to_writer(&buf) - - xml.print(w, doc) - println(strings.to_string(buf)) - if err != .None { - printf("Parse error: %v\n", err) - } else { - println("DONE!") + printf("Load/Parse error: %v\n", err) + if err == .File_Error { + printf("\"%v\" not found. Did you run \"tests\\download_assets.py\"?", filename) + } + os.exit(1) + } + + printf("\"%v\" loaded and parsed.\n", filename) + + charlist, charlist_ok := xml.find_child_by_ident(doc.root, "charlist") + if !charlist_ok { + eprintln("Could not locate top-level `` tag.") + os.exit(1) + } + + printf("Found `` with %v children.\n", len(charlist.children)) + + for char in charlist.children { + if char.ident != "character" { + eprintf("Expected ``, got `<%v>`\n", char.ident) + os.exit(1) + } + + if _, ok := xml.find_attribute_val_by_key(char, "dec"); !ok { + eprintln("`` attribute not found.") + os.exit(1) + } } } @@ -50,12 +60,13 @@ main :: proc() { mem.tracking_allocator_init(&track, context.allocator) context.allocator = mem.tracking_allocator(&track) - _main() + example() if len(track.allocation_map) > 0 { println() for _, v in track.allocation_map { printf("%v Leaked %v bytes.\n", v.location, v.size) } - } + } + println("Done and cleaned up!") } \ No newline at end of file diff --git a/core/encoding/xml/helpers.odin b/core/encoding/xml/helpers.odin new file mode 100644 index 000000000..14597ddbd --- /dev/null +++ b/core/encoding/xml/helpers.odin @@ -0,0 +1,49 @@ +package xml +/* + An XML 1.0 / 1.1 parser + + Copyright 2021 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + This file contains helper functions. +*/ + + +/* + Find `tag`'s nth child with a given ident. +*/ +find_child_by_ident :: proc(tag: ^Element, ident: string, nth := 0) -> (res: ^Element, found: bool) { + if tag == nil { return nil, false } + + count := 0 + for child in tag.children { + /* + Skip commments. They have no name. + */ + if child.kind != .Element { continue } + + /* + If the ident matches and it's the nth such child, return it. + */ + if child.ident == ident { + if count == nth { return child, true } + count += 1 + } + } + return nil, false +} + +/* + Find an attribute by key. +*/ +find_attribute_val_by_key :: proc(tag: ^Element, key: string) -> (val: string, found: bool) { + if tag == nil { return "", false } + + for attr in tag.attribs { + /* + If the ident matches, we're done. There can only ever be one attribute with the same name. + */ + if attr.key == key { return attr.val, true } + } + return "", false +} \ No newline at end of file diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin index 95024518d..2da3b7683 100644 --- a/core/encoding/xml/tokenizer.odin +++ b/core/encoding/xml/tokenizer.odin @@ -403,11 +403,11 @@ scan :: proc(t: ^Tokenizer) -> Token { case ':': kind = .Colon case '"', '\'': + kind = .Invalid + lit, err = scan_string(t, t.offset, ch, true, false) if err == .None { kind = .String - } else { - kind = .Invalid } case '\n': @@ -418,7 +418,7 @@ scan :: proc(t: ^Tokenizer) -> Token { } } - if lit == "" { + if kind != .String && lit == "" { lit = string(t.src[offset : t.offset]) } return Token{kind, lit, pos} diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 146c278cb..563294309 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -519,6 +519,8 @@ parse_attribute :: proc(doc: ^Document) -> (attr: Attr, offset: int, err: Error) _ = expect(t, .Eq) or_return value := expect(t, .String) or_return + error(t, t.offset, "String: %v\n", value) + attr.key = strings.intern_get(&doc.intern, key.text) attr.val = strings.intern_get(&doc.intern, value.text) diff --git a/tests/core/assets/XML/.gitignore b/tests/core/assets/XML/.gitignore new file mode 100644 index 000000000..32dc58b57 --- /dev/null +++ b/tests/core/assets/XML/.gitignore @@ -0,0 +1,2 @@ +# This file will be downloaded by download_assets.py +unicode.xml \ No newline at end of file diff --git a/tests/core/assets/xml/nl_NL-qt-ts.ts b/tests/core/assets/XML/nl_NL-qt-ts.ts similarity index 100% rename from tests/core/assets/xml/nl_NL-qt-ts.ts rename to tests/core/assets/XML/nl_NL-qt-ts.ts diff --git a/tests/core/assets/xml/nl_NL-xliff-1.0.xliff b/tests/core/assets/XML/nl_NL-xliff-1.0.xliff similarity index 100% rename from tests/core/assets/xml/nl_NL-xliff-1.0.xliff rename to tests/core/assets/XML/nl_NL-xliff-1.0.xliff diff --git a/tests/core/assets/xml/nl_NL-xliff-2.0.xliff b/tests/core/assets/XML/nl_NL-xliff-2.0.xliff similarity index 100% rename from tests/core/assets/xml/nl_NL-xliff-2.0.xliff rename to tests/core/assets/XML/nl_NL-xliff-2.0.xliff diff --git a/tests/core/assets/xml/utf8.xml b/tests/core/assets/XML/utf8.xml similarity index 100% rename from tests/core/assets/xml/utf8.xml rename to tests/core/assets/XML/utf8.xml diff --git a/tests/core/download_assets.py b/tests/core/download_assets.py index d86f7f1e7..831b5b13a 100644 --- a/tests/core/download_assets.py +++ b/tests/core/download_assets.py @@ -50,10 +50,7 @@ def try_download_file(url, out_file): print("Could not download", url) return 1 -def try_download_and_unpack_zip(suite): - url = ASSETS_BASE_URL.format(suite, "{}.zip".format(suite)) - out_file = DOWNLOAD_BASE_PATH.format(suite) + "/{}.zip".format(suite) - +def try_download_and_unpack_zip(url, out_file, extract_path): print("\tDownloading {} to {}.".format(url, out_file)) if try_download_file(url, out_file) is not None: @@ -65,7 +62,6 @@ def try_download_and_unpack_zip(suite): with zipfile.ZipFile(out_file) as z: for file in z.filelist: filename = file.filename - extract_path = DOWNLOAD_BASE_PATH.format(suite) print("\t\tExtracting: {}".format(filename)) z.extract(file, extract_path) @@ -73,25 +69,56 @@ def try_download_and_unpack_zip(suite): print("Could not extract ZIP file") return 2 +def download_png_assets(): + suite = "PNG" + url = ASSETS_BASE_URL.format(suite, "{}.zip".format(suite)) + out_file = DOWNLOAD_BASE_PATH.format(suite) + "/{}.zip".format(suite) + extract_path = DOWNLOAD_BASE_PATH.format(suite) -def main(): print("Downloading PNG assets") # Make PNG assets path try: - path = DOWNLOAD_BASE_PATH.format("PNG") + path = DOWNLOAD_BASE_PATH.format(suite) os.makedirs(path) except FileExistsError: pass # Try downloading and unpacking the PNG assets - r = try_download_and_unpack_zip("PNG") + r = try_download_and_unpack_zip(url, out_file, extract_path) if r is not None: return r # We could fall back on downloading the PNG files individually, but it's slow print("Done downloading PNG assets") + +def download_unicode_assets(): + suite = "XML" + url = "https://www.w3.org/2003/entities/2007xml/unicode.xml.zip" + out_file = DOWNLOAD_BASE_PATH.format(suite) + "/{}.zip".format(suite) + extract_path = DOWNLOAD_BASE_PATH.format(suite) + + print("Downloading {}.".format(url)) + + # Make XML assets path + try: + path = DOWNLOAD_BASE_PATH.format(suite) + os.makedirs(path) + except FileExistsError: + pass + + # Try downloading and unpacking the assets + r = try_download_and_unpack_zip(url, out_file, extract_path) + if r is not None: + return r + + print("Done downloading Unicode/XML assets") + +def main(): + download_png_assets() + download_unicode_assets() + return 0 if __name__ == '__main__': diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin index 7eefac212..c2e0aa172 100644 --- a/tests/core/encoding/xml/test_core_xml.odin +++ b/tests/core/encoding/xml/test_core_xml.odin @@ -35,7 +35,7 @@ TESTS :: []TEST{ First we test that certain files parse without error. */ { - filename = "assets/xml/utf8.xml", + filename = "assets/XML/utf8.xml", options = OPTIONS, expected = { error = .None, @@ -45,7 +45,7 @@ TESTS :: []TEST{ }, }, { - filename = "assets/xml/nl_NL-qt-ts.ts", + filename = "assets/XML/nl_NL-qt-ts.ts", options = OPTIONS, expected = { error = .None, @@ -55,7 +55,7 @@ TESTS :: []TEST{ }, }, { - filename = "assets/xml/nl_NL-xliff-1.0.xliff", + filename = "assets/XML/nl_NL-xliff-1.0.xliff", options = OPTIONS, expected = { error = .None, @@ -65,7 +65,7 @@ TESTS :: []TEST{ }, }, { - filename = "assets/xml/nl_NL-xliff-2.0.xliff", + filename = "assets/XML/nl_NL-xliff-2.0.xliff", options = OPTIONS, expected = { error = .None, @@ -79,7 +79,7 @@ TESTS :: []TEST{ Then we test that certain errors are returned as expected. */ { - filename = "assets/xml/utf8.xml", + filename = "assets/XML/utf8.xml", options = { flags = { .Ignore_Unsupported, .Intern_Comments, From 2dd67dba89732b89adb0199bc0a99de4cbc34e8f Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 2 Dec 2021 20:12:12 +0100 Subject: [PATCH 10/79] [core:encoding/entity] Add new package to decode &; entities. Includes generator to generate a lookup for named entitiess. --- core/encoding/entity/LICENSE_table.md | 21 + core/encoding/entity/entity.odin | 358 + .../entity/example/entity_example.odin | 122 + core/encoding/entity/example/test.html | 26 + core/encoding/entity/generated.odin | 7493 +++++++++++++++++ core/encoding/xml/xml_reader.odin | 2 - core/unicode/tools/generate_entity_table.odin | 287 + 7 files changed, 8307 insertions(+), 2 deletions(-) create mode 100644 core/encoding/entity/LICENSE_table.md create mode 100644 core/encoding/entity/entity.odin create mode 100644 core/encoding/entity/example/entity_example.odin create mode 100644 core/encoding/entity/example/test.html create mode 100644 core/encoding/entity/generated.odin create mode 100644 core/unicode/tools/generate_entity_table.odin diff --git a/core/encoding/entity/LICENSE_table.md b/core/encoding/entity/LICENSE_table.md new file mode 100644 index 000000000..51e3f34b5 --- /dev/null +++ b/core/encoding/entity/LICENSE_table.md @@ -0,0 +1,21 @@ +# License + +By obtaining, using and/or copying this work, you (the licensee) agree that you have read, understood, and will comply with the following terms and conditions. + +Permission to copy, modify, and distribute this software and its documentation, with or without modification, for any purpose and without fee or royalty is hereby granted, provided that you include the following on ALL copies of the software and documentation or portions thereof, including modifications: + +The full text of this NOTICE in a location viewable to users of the redistributed or derivative work. +Any pre-existing intellectual property disclaimers, notices, or terms and conditions. If none exist, the W3C Software Short Notice should be included (hypertext is preferred, text is permitted) within the body of any redistributed or derivative code. + +Notice of any changes or modifications to the files, including the date changes were made. (We recommend you provide URIs to the location from which the code is derived.) + +# Disclaimers + +THIS SOFTWARE AND DOCUMENTATION IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE OR DOCUMENTATION WILL NOT INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS. + +COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR DOCUMENTATION. + +The name and trademarks of copyright holders may NOT be used in advertising or publicity pertaining to the software without specific, written prior permission. Title to copyright in this software and any associated documentation will at all times remain with copyright holders. + +# Notes +This version: http://www.w3.org/Consortium/Legal/2002/copyright-software-20021231 \ No newline at end of file diff --git a/core/encoding/entity/entity.odin b/core/encoding/entity/entity.odin new file mode 100644 index 000000000..e40896819 --- /dev/null +++ b/core/encoding/entity/entity.odin @@ -0,0 +1,358 @@ +package unicode_entity +/* + A unicode entity encoder/decoder + + Copyright 2021 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + This code has several procedures to map unicode runes to/from different textual encodings. + - SGML/XML/HTML entity + -- &#; + -- &#x; + -- &; (If the lookup tables are compiled in). + Reference: https://www.w3.org/2003/entities/2007xml/unicode.xml + + - URL encode / decode %hex entity + Reference: https://datatracker.ietf.org/doc/html/rfc3986/#section-2.1 + + List of contributors: + Jeroen van Rijn: Initial implementation. +*/ + +import "core:unicode/utf8" +import "core:unicode" +import "core:strings" + +MAX_RUNE_CODEPOINT :: int(unicode.MAX_RUNE) + +write_rune :: strings.write_rune_builder +write_string :: strings.write_string_builder + +Error :: enum u8 { + None = 0, + Tokenizer_Is_Nil, + + Illegal_NUL_Character, + Illegal_UTF_Encoding, + Illegal_BOM, + + CDATA_Not_Terminated, + Comment_Not_Terminated, + Invalid_Entity_Encoding, +} + +Tokenizer :: struct { + r: rune, + w: int, + + src: string, + offset: int, + read_offset: int, +} + +CDATA_START :: "" + +COMMENT_START :: "" + +/* + Default: CDATA and comments are passed through unchanged. +*/ +XML_Decode_Option :: enum u8 { + /* + CDATA is unboxed. + */ + CDATA_Unbox, + + /* + Unboxed CDATA is decoded as well. + Ignored if `.CDATA_Unbox` is not given. + */ + CDATA_Decode, + + /* + Comments are stripped. + */ + Comment_Strip, +} +XML_Decode_Options :: bit_set[XML_Decode_Option; u8] + +/* + Decode a string that may include SGML/XML/HTML entities. + The caller has to free the result. +*/ +decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator := context.allocator) -> (decoded: string, err: Error) { + context.allocator = allocator + + l := len(input) + if l == 0 { return "", .None } + + builder := strings.make_builder() + defer strings.destroy_builder(&builder) + + t := Tokenizer{src=input} + in_data := false + + loop: for { + advance(&t) or_return + if t.r < 0 { break loop } + + /* + Below here we're never inside a CDATA tag. + At most we'll see the start of one, but that doesn't affect the logic. + */ + switch t.r { + case '<': + /* + Might be the start of a CDATA tag or comment. + + We don't need to check if we need to write a `<`, because if it isn't CDATA or a comment, + it couldn't have been part of an XML tag body to be decoded here. + */ + in_data = _handle_xml_special(&t, &builder, options) or_return + + case ']': + /* + If we're unboxing _and_ decoding CDATA, we'll have to check for the end tag. + */ + if in_data { + if t.read_offset + len(CDATA_END) < len(t.src) { + if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END { + in_data = false + t.read_offset += len(CDATA_END) - 1 + } + } + continue + } else { + write_rune(&builder, ']') + } + + case: + if in_data && .CDATA_Decode not_in options { + /* + Unboxed, but undecoded. + */ + write_rune(&builder, t.r) + continue + } + + if t.r == '&' { + if entity, entity_err := _extract_xml_entity(&t); entity_err != .None { + /* + We read to the end of the string without closing the entity. + Pass through as-is. + */ + write_string(&builder, entity) + } else { + if decoded, ok := xml_decode_entity(entity); ok { + write_rune(&builder, decoded) + } else { + /* + Decode failed. Pass through original. + */ + write_string(&builder, "&") + write_string(&builder, entity) + write_string(&builder, ";") + } + + } + } else { + write_rune(&builder, t.r) + } + } + } + + return strings.clone(strings.to_string(builder), allocator), err +} + +advance :: proc(t: ^Tokenizer) -> (err: Error) { + if t == nil { return .Tokenizer_Is_Nil } + using t + + #no_bounds_check { + if read_offset < len(src) { + offset = read_offset + r, w = rune(src[read_offset]), 1 + switch { + case r == 0: + return .Illegal_NUL_Character + case r >= utf8.RUNE_SELF: + r, w = utf8.decode_rune_in_string(src[read_offset:]) + if r == utf8.RUNE_ERROR && w == 1 { + return .Illegal_UTF_Encoding + } else if r == utf8.RUNE_BOM && offset > 0 { + return .Illegal_BOM + } + } + read_offset += w + return .None + } else { + offset = len(src) + r = -1 + return + } + } +} + +xml_decode_entity :: proc(entity: string) -> (decoded: rune, ok: bool) { + entity := entity + if len(entity) == 0 { return -1, false } + + switch entity[0] { + case '#': + base := 10 + val := 0 + entity = entity[1:] + + if len(entity) == 0 { return -1, false } + + if entity[0] == 'x' || entity[0] == 'X' { + base = 16 + entity = entity[1:] + } + + for len(entity) > 0 { + r := entity[0] + switch r { + case '0'..'9': + val *= base + val += int(r - '0') + + case 'a'..'f': + if base == 10 { return -1, false } + val *= base + val += int(r - 'a' + 10) + + case 'A'..'F': + if base == 10 { return -1, false } + val *= base + val += int(r - 'A' + 10) + + case: + return -1, false + } + + if val > MAX_RUNE_CODEPOINT { return -1, false } + entity = entity[1:] + } + return rune(val), true + + case: + /* + Named entity. + */ + return named_xml_entity_to_rune(entity) + } +} + +/* + Private XML helper to extract `&;` entity. +*/ +@(private="file") +_extract_xml_entity :: proc(t: ^Tokenizer) -> (entity: string, err: Error) { + assert(t != nil && t.r == '&') + + /* + All of these would be in the ASCII range. + Even if one is not, it doesn't matter. All characters we need to compare to extract are. + */ + using t + + length := len(t.src) + found := false + + #no_bounds_check { + for read_offset < length { + if src[read_offset] == ';' { + found = true + read_offset += 1 + break + } + read_offset += 1 + } + } + + if found { + return string(src[offset + 1 : read_offset - 1]), .None + } + return string(src[offset : read_offset]), .Invalid_Entity_Encoding +} + +/* + Private XML helper for CDATA and comments. +*/ +@(private="file") +_handle_xml_special :: proc(t: ^Tokenizer, builder: ^strings.Builder, options: XML_Decode_Options) -> (in_data: bool, err: Error) { + assert(t != nil && t.r == '<') + if t.read_offset + len(CDATA_START) >= len(t.src) { return false, .None } + + if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START { + t.read_offset += len(CDATA_START) - 1 + + if .CDATA_Unbox in options && .CDATA_Decode in options { + /* + We're unboxing _and_ decoding CDATA + */ + return true, .None + } + + /* + CDATA is passed through. + */ + offset := t.offset + + /* + Scan until end of CDATA. + */ + for { + advance(t) or_return + if t.r < 0 { return true, .CDATA_Not_Terminated } + + if t.read_offset + len(CDATA_END) < len(t.src) { + if string(t.src[t.offset:][:len(CDATA_END)]) == CDATA_END { + t.read_offset += len(CDATA_END) - 1 + + cdata := string(t.src[offset : t.read_offset]) + + if .CDATA_Unbox in options { + cdata = cdata[len(CDATA_START):] + cdata = cdata[:len(cdata) - len(CDATA_END)] + } + + write_string(builder, cdata) + return false, .None + } + } + } + + } else if string(t.src[t.offset:][:len(COMMENT_START)]) == COMMENT_START { + t.read_offset += len(COMMENT_START) + /* + Comment is passed through by default. + */ + offset := t.offset + + /* + Scan until end of Comment. + */ + for { + advance(t) or_return + if t.r < 0 { return true, .Comment_Not_Terminated } + + if t.read_offset + len(COMMENT_END) < len(t.src) { + if string(t.src[t.offset:][:len(COMMENT_END)]) == COMMENT_END { + t.read_offset += len(COMMENT_END) - 1 + + if .Comment_Strip not_in options { + comment := string(t.src[offset : t.read_offset]) + write_string(builder, comment) + } + return false, .None + } + } + } + + } + return false, .None +} \ No newline at end of file diff --git a/core/encoding/entity/example/entity_example.odin b/core/encoding/entity/example/entity_example.odin new file mode 100644 index 000000000..8758d9ad9 --- /dev/null +++ b/core/encoding/entity/example/entity_example.odin @@ -0,0 +1,122 @@ +package unicode_entity_example + +import "core:encoding/xml" +import "core:encoding/entity" +import "core:strings" +import "core:mem" +import "core:fmt" +import "core:time" + +OPTIONS :: xml.Options{ + flags = { + .Ignore_Unsupported, .Intern_Comments, + }, + expected_doctype = "", +} + +doc_print :: proc(doc: ^xml.Document) { + buf: strings.Builder + defer strings.destroy_builder(&buf) + w := strings.to_writer(&buf) + + xml.print(w, doc) + fmt.println(strings.to_string(buf)) +} + +_entities :: proc() { + doc: ^xml.Document + err: xml.Error + + DOC :: #load("../../../../tests/core/assets/XML/unicode.xml") + + parse_duration: time.Duration + + { + time.SCOPED_TICK_DURATION(&parse_duration) + doc, err = xml.parse(DOC, OPTIONS) + } + defer xml.destroy(doc) + + doc_print(doc) + + ms := time.duration_milliseconds(parse_duration) + + speed := (f64(1000.0) / ms) * f64(len(DOC)) / 1_024.0 / 1_024.0 + + fmt.printf("Parse time: %.2f ms (%.2f MiB/s).\n", ms, speed) + fmt.printf("Error: %v\n", err) +} + +_main :: proc() { + using fmt + + doc, err := xml.parse(#load("test.html")) + defer xml.destroy(doc) + doc_print(doc) + + if false { + val := doc.root.children[1].children[2].value + + println() + replaced, ok := entity.decode_xml(val) + defer delete(replaced) + + printf("Before: '%v', Err: %v\n", val, err) + printf("Passthrough: '%v'\nOK: %v\n", replaced, ok) + println() + } + + if false { + val := doc.root.children[1].children[2].value + + println() + replaced, ok := entity.decode_xml(val, { .CDATA_Unbox }) + defer delete(replaced) + + printf("Before: '%v', Err: %v\n", val, err) + printf("CDATA_Unbox: '%v'\nOK: %v\n", replaced, ok) + println() + } + + if true { + val := doc.root.children[1].children[2].value + + println() + replaced, ok := entity.decode_xml(val, { .CDATA_Unbox, .CDATA_Decode }) + defer delete(replaced) + + printf("Before: '%v', Err: %v\n", val, err) + printf("CDATA_Decode: '%v'\nOK: %v\n", replaced, ok) + println() + } + + if true { + val := doc.root.children[1].children[1].value + + println() + replaced, ok := entity.decode_xml(val, { .Comment_Strip }) + defer delete(replaced) + + printf("Before: '%v', Err: %v\n", val, err) + printf("Comment_Strip: '%v'\nOK: %v\n", replaced, ok) + println() + } +} + +main :: proc() { + using fmt + + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + _main() + //_entities() + + if len(track.allocation_map) > 0 { + println() + for _, v in track.allocation_map { + printf("%v Leaked %v bytes.\n", v.location, v.size) + } + } +} \ No newline at end of file diff --git a/core/encoding/entity/example/test.html b/core/encoding/entity/example/test.html new file mode 100644 index 000000000..60e32bf03 --- /dev/null +++ b/core/encoding/entity/example/test.html @@ -0,0 +1,26 @@ + + + Entity Reference Test + + + +

Entity Reference Test

+
+ Foozle]! © 42&;1234& +
+
+ Foozle]! © 42&;1234& +
+
+ | | | fj ` \ ® ϱ ∳ +
+ + \ No newline at end of file diff --git a/core/encoding/entity/generated.odin b/core/encoding/entity/generated.odin new file mode 100644 index 000000000..9afdcae6d --- /dev/null +++ b/core/encoding/entity/generated.odin @@ -0,0 +1,7493 @@ +package unicode_entity + +/* + ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ +*/ + +/* + This file is generated from "https://www.w3.org/2003/entities/2007xml/unicode.xml". + + UPDATE: + - Ensure the XML file was downloaded using "tests\core\download_assets.py". + - Run "core/unicode/tools/generate_entity_table.odin" + + Odin unicode generated tables: https://github.com/odin-lang/Odin/tree/master/core/encoding/entity + + Copyright © 2021 World Wide Web Consortium, (Massachusetts Institute of Technology, + European Research Consortium for Informatics and Mathematics, Keio University, Beihang). + + All Rights Reserved. + + This work is distributed under the W3C® Software License [1] in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + [1] http://www.w3.org/Consortium/Legal/copyright-software + + See also: LICENSE_table.md +*/ + +// `<` +XML_NAME_TO_RUNE_MIN_LENGTH :: 2 +// `∳` +XML_NAME_TO_RUNE_MAX_LENGTH :: 31 + + +/* + Input: + entity_name - a string, like "copy" that describes a user-encoded Unicode entity as used in XML. + + Output: + "decoded" - The decoded rune if found by name, or -1 otherwise. + "ok" - true if found, false if not. + + IMPORTANT: XML processors (including browsers) treat these names as case-sensitive. So do we. +*/ +named_xml_entity_to_rune :: proc(name: string) -> (decoded: rune, ok: bool) { + /* + Early out if the name is too short or too long. + min as a precaution in case the generated table has a bogus value. + */ + if len(name) < min(1, XML_NAME_TO_RUNE_MIN_LENGTH) || len(name) > XML_NAME_TO_RUNE_MAX_LENGTH { + return -1, false + } + + switch rune(name[0]) { + + case 'A': + switch name { + case "AElig": + // LATIN CAPITAL LETTER AE + return rune(0xc6), true + case "AMP": + // AMPERSAND + return rune(0x26), true + case "Aacgr": + // GREEK CAPITAL LETTER ALPHA WITH TONOS + return rune(0x0386), true + case "Aacute": + // LATIN CAPITAL LETTER A WITH ACUTE + return rune(0xc1), true + case "Abreve": + // LATIN CAPITAL LETTER A WITH BREVE + return rune(0x0102), true + case "Acirc": + // LATIN CAPITAL LETTER A WITH CIRCUMFLEX + return rune(0xc2), true + case "Acy": + // CYRILLIC CAPITAL LETTER A + return rune(0x0410), true + case "Afr": + // MATHEMATICAL FRAKTUR CAPITAL A + return rune(0x01d504), true + case "Agrave": + // LATIN CAPITAL LETTER A WITH GRAVE + return rune(0xc0), true + case "Agr": + // GREEK CAPITAL LETTER ALPHA + return rune(0x0391), true + case "Alpha": + // GREEK CAPITAL LETTER ALPHA + return rune(0x0391), true + case "Amacr": + // LATIN CAPITAL LETTER A WITH MACRON + return rune(0x0100), true + case "And": + // DOUBLE LOGICAL AND + return rune(0x2a53), true + case "Aogon": + // LATIN CAPITAL LETTER A WITH OGONEK + return rune(0x0104), true + case "Aopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL A + return rune(0x01d538), true + case "ApplyFunction": + // FUNCTION APPLICATION + return rune(0x2061), true + case "Aring": + // LATIN CAPITAL LETTER A WITH RING ABOVE + return rune(0xc5), true + case "Ascr": + // MATHEMATICAL SCRIPT CAPITAL A + return rune(0x01d49c), true + case "Assign": + // COLON EQUALS + return rune(0x2254), true + case "Ast": + // TWO ASTERISKS ALIGNED VERTICALLY + return rune(0x2051), true + case "Atilde": + // LATIN CAPITAL LETTER A WITH TILDE + return rune(0xc3), true + case "Auml": + // LATIN CAPITAL LETTER A WITH DIAERESIS + return rune(0xc4), true + } + + case 'B': + switch name { + case "Backslash": + // SET MINUS + return rune(0x2216), true + case "Barint": + // INTEGRAL WITH DOUBLE STROKE + return rune(0x2a0e), true + case "Barv": + // SHORT DOWN TACK WITH OVERBAR + return rune(0x2ae7), true + case "Barwedl": + // LOGICAL AND WITH DOUBLE OVERBAR + return rune(0x2a5e), true + case "Barwed": + // PERSPECTIVE + return rune(0x2306), true + case "Bcy": + // CYRILLIC CAPITAL LETTER BE + return rune(0x0411), true + case "Because": + // BECAUSE + return rune(0x2235), true + case "Bernoullis": + // SCRIPT CAPITAL B + return rune(0x212c), true + case "Beta": + // GREEK CAPITAL LETTER BETA + return rune(0x0392), true + case "Bfr": + // MATHEMATICAL FRAKTUR CAPITAL B + return rune(0x01d505), true + case "Bgr": + // GREEK CAPITAL LETTER BETA + return rune(0x0392), true + case "Bopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL B + return rune(0x01d539), true + case "Breve": + // BREVE + return rune(0x02d8), true + case "Bscr": + // SCRIPT CAPITAL B + return rune(0x212c), true + case "Bumpeq": + // GEOMETRICALLY EQUIVALENT TO + return rune(0x224e), true + case "Bvert": + // BOX DRAWINGS LIGHT TRIPLE DASH VERTICAL + return rune(0x2506), true + } + + case 'C': + switch name { + case "CHcy": + // CYRILLIC CAPITAL LETTER CHE + return rune(0x0427), true + case "COPY": + // COPYRIGHT SIGN + return rune(0xa9), true + case "Cacute": + // LATIN CAPITAL LETTER C WITH ACUTE + return rune(0x0106), true + case "CapitalDifferentialD": + // DOUBLE-STRUCK ITALIC CAPITAL D + return rune(0x2145), true + case "Cap": + // DOUBLE INTERSECTION + return rune(0x22d2), true + case "Cayleys": + // BLACK-LETTER CAPITAL C + return rune(0x212d), true + case "Ccaron": + // LATIN CAPITAL LETTER C WITH CARON + return rune(0x010c), true + case "Ccedil": + // LATIN CAPITAL LETTER C WITH CEDILLA + return rune(0xc7), true + case "Ccirc": + // LATIN CAPITAL LETTER C WITH CIRCUMFLEX + return rune(0x0108), true + case "Cconint": + // VOLUME INTEGRAL + return rune(0x2230), true + case "Cdot": + // LATIN CAPITAL LETTER C WITH DOT ABOVE + return rune(0x010a), true + case "Cedilla": + // CEDILLA + return rune(0xb8), true + case "CenterDot": + // MIDDLE DOT + return rune(0xb7), true + case "Cfr": + // BLACK-LETTER CAPITAL C + return rune(0x212d), true + case "Chi": + // GREEK CAPITAL LETTER CHI + return rune(0x03a7), true + case "CircleDot": + // CIRCLED DOT OPERATOR + return rune(0x2299), true + case "CircleMinus": + // CIRCLED MINUS + return rune(0x2296), true + case "CirclePlus": + // CIRCLED PLUS + return rune(0x2295), true + case "CircleTimes": + // CIRCLED TIMES + return rune(0x2297), true + case "ClockwiseContourIntegral": + // CLOCKWISE CONTOUR INTEGRAL + return rune(0x2232), true + case "CloseCurlyDoubleQuote": + // RIGHT DOUBLE QUOTATION MARK + return rune(0x201d), true + case "CloseCurlyQuote": + // RIGHT SINGLE QUOTATION MARK + return rune(0x2019), true + case "Colon": + // PROPORTION + return rune(0x2237), true + case "Colone": + // DOUBLE COLON EQUAL + return rune(0x2a74), true + case "Congruent": + // IDENTICAL TO + return rune(0x2261), true + case "Conint": + // SURFACE INTEGRAL + return rune(0x222f), true + case "ContourIntegral": + // CONTOUR INTEGRAL + return rune(0x222e), true + case "Copf": + // DOUBLE-STRUCK CAPITAL C + return rune(0x2102), true + case "Coproduct": + // N-ARY COPRODUCT + return rune(0x2210), true + case "CounterClockwiseContourIntegral": + // ANTICLOCKWISE CONTOUR INTEGRAL + return rune(0x2233), true + case "Cross": + // VECTOR OR CROSS PRODUCT + return rune(0x2a2f), true + case "Cscr": + // MATHEMATICAL SCRIPT CAPITAL C + return rune(0x01d49e), true + case "CupCap": + // EQUIVALENT TO + return rune(0x224d), true + case "Cup": + // DOUBLE UNION + return rune(0x22d3), true + } + + case 'D': + switch name { + case "DD": + // DOUBLE-STRUCK ITALIC CAPITAL D + return rune(0x2145), true + case "DDotrahd": + // RIGHTWARDS ARROW WITH DOTTED STEM + return rune(0x2911), true + case "DJcy": + // CYRILLIC CAPITAL LETTER DJE + return rune(0x0402), true + case "DScy": + // CYRILLIC CAPITAL LETTER DZE + return rune(0x0405), true + case "DZcy": + // CYRILLIC CAPITAL LETTER DZHE + return rune(0x040f), true + case "Dagger": + // DOUBLE DAGGER + return rune(0x2021), true + case "Darr": + // DOWNWARDS TWO HEADED ARROW + return rune(0x21a1), true + case "Dashv": + // VERTICAL BAR DOUBLE LEFT TURNSTILE + return rune(0x2ae4), true + case "Dcaron": + // LATIN CAPITAL LETTER D WITH CARON + return rune(0x010e), true + case "Dcy": + // CYRILLIC CAPITAL LETTER DE + return rune(0x0414), true + case "Del": + // NABLA + return rune(0x2207), true + case "Delta": + // GREEK CAPITAL LETTER DELTA + return rune(0x0394), true + case "Dfr": + // MATHEMATICAL FRAKTUR CAPITAL D + return rune(0x01d507), true + case "Dgr": + // GREEK CAPITAL LETTER DELTA + return rune(0x0394), true + case "DiacriticalAcute": + // ACUTE ACCENT + return rune(0xb4), true + case "DiacriticalDot": + // DOT ABOVE + return rune(0x02d9), true + case "DiacriticalDoubleAcute": + // DOUBLE ACUTE ACCENT + return rune(0x02dd), true + case "DiacriticalGrave": + // GRAVE ACCENT + return rune(0x60), true + case "DiacriticalTilde": + // SMALL TILDE + return rune(0x02dc), true + case "Diamond": + // DIAMOND OPERATOR + return rune(0x22c4), true + case "DifferentialD": + // DOUBLE-STRUCK ITALIC SMALL D + return rune(0x2146), true + case "Dopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL D + return rune(0x01d53b), true + case "Dot": + // DIAERESIS + return rune(0xa8), true + case "DotDot": + // COMBINING FOUR DOTS ABOVE + return rune(0x20dc), true + case "DotEqual": + // APPROACHES THE LIMIT + return rune(0x2250), true + case "DoubleContourIntegral": + // SURFACE INTEGRAL + return rune(0x222f), true + case "DoubleDot": + // DIAERESIS + return rune(0xa8), true + case "DoubleDownArrow": + // DOWNWARDS DOUBLE ARROW + return rune(0x21d3), true + case "DoubleLeftArrow": + // LEFTWARDS DOUBLE ARROW + return rune(0x21d0), true + case "DoubleLeftRightArrow": + // LEFT RIGHT DOUBLE ARROW + return rune(0x21d4), true + case "DoubleLeftTee": + // VERTICAL BAR DOUBLE LEFT TURNSTILE + return rune(0x2ae4), true + case "DoubleLongLeftArrow": + // LONG LEFTWARDS DOUBLE ARROW + return rune(0x27f8), true + case "DoubleLongLeftRightArrow": + // LONG LEFT RIGHT DOUBLE ARROW + return rune(0x27fa), true + case "DoubleLongRightArrow": + // LONG RIGHTWARDS DOUBLE ARROW + return rune(0x27f9), true + case "DoubleRightArrow": + // RIGHTWARDS DOUBLE ARROW + return rune(0x21d2), true + case "DoubleRightTee": + // TRUE + return rune(0x22a8), true + case "DoubleUpArrow": + // UPWARDS DOUBLE ARROW + return rune(0x21d1), true + case "DoubleUpDownArrow": + // UP DOWN DOUBLE ARROW + return rune(0x21d5), true + case "DoubleVerticalBar": + // PARALLEL TO + return rune(0x2225), true + case "DownArrowUpArrow": + // DOWNWARDS ARROW LEFTWARDS OF UPWARDS ARROW + return rune(0x21f5), true + case "DownArrow": + // DOWNWARDS ARROW + return rune(0x2193), true + case "DownArrowBar": + // DOWNWARDS ARROW TO BAR + return rune(0x2913), true + case "DownBreve": + // COMBINING INVERTED BREVE + return rune(0x0311), true + case "DownLeftRightVector": + // LEFT BARB DOWN RIGHT BARB DOWN HARPOON + return rune(0x2950), true + case "DownLeftTeeVector": + // LEFTWARDS HARPOON WITH BARB DOWN FROM BAR + return rune(0x295e), true + case "DownLeftVector": + // LEFTWARDS HARPOON WITH BARB DOWNWARDS + return rune(0x21bd), true + case "DownLeftVectorBar": + // LEFTWARDS HARPOON WITH BARB DOWN TO BAR + return rune(0x2956), true + case "DownRightTeeVector": + // RIGHTWARDS HARPOON WITH BARB DOWN FROM BAR + return rune(0x295f), true + case "DownRightVector": + // RIGHTWARDS HARPOON WITH BARB DOWNWARDS + return rune(0x21c1), true + case "DownRightVectorBar": + // RIGHTWARDS HARPOON WITH BARB DOWN TO BAR + return rune(0x2957), true + case "DownTeeArrow": + // DOWNWARDS ARROW FROM BAR + return rune(0x21a7), true + case "DownTee": + // DOWN TACK + return rune(0x22a4), true + case "Downarrow": + // DOWNWARDS DOUBLE ARROW + return rune(0x21d3), true + case "Dscr": + // MATHEMATICAL SCRIPT CAPITAL D + return rune(0x01d49f), true + case "Dstrok": + // LATIN CAPITAL LETTER D WITH STROKE + return rune(0x0110), true + } + + case 'E': + switch name { + case "EEacgr": + // GREEK CAPITAL LETTER ETA WITH TONOS + return rune(0x0389), true + case "EEgr": + // GREEK CAPITAL LETTER ETA + return rune(0x0397), true + case "ENG": + // LATIN CAPITAL LETTER ENG + return rune(0x014a), true + case "ETH": + // LATIN CAPITAL LETTER ETH + return rune(0xd0), true + case "Eacgr": + // GREEK CAPITAL LETTER EPSILON WITH TONOS + return rune(0x0388), true + case "Eacute": + // LATIN CAPITAL LETTER E WITH ACUTE + return rune(0xc9), true + case "Ecaron": + // LATIN CAPITAL LETTER E WITH CARON + return rune(0x011a), true + case "Ecirc": + // LATIN CAPITAL LETTER E WITH CIRCUMFLEX + return rune(0xca), true + case "Ecy": + // CYRILLIC CAPITAL LETTER E + return rune(0x042d), true + case "Edot": + // LATIN CAPITAL LETTER E WITH DOT ABOVE + return rune(0x0116), true + case "Efr": + // MATHEMATICAL FRAKTUR CAPITAL E + return rune(0x01d508), true + case "Egrave": + // LATIN CAPITAL LETTER E WITH GRAVE + return rune(0xc8), true + case "Egr": + // GREEK CAPITAL LETTER EPSILON + return rune(0x0395), true + case "Element": + // ELEMENT OF + return rune(0x2208), true + case "Emacr": + // LATIN CAPITAL LETTER E WITH MACRON + return rune(0x0112), true + case "EmptySmallSquare": + // WHITE MEDIUM SQUARE + return rune(0x25fb), true + case "EmptyVerySmallSquare": + // WHITE SMALL SQUARE + return rune(0x25ab), true + case "Eogon": + // LATIN CAPITAL LETTER E WITH OGONEK + return rune(0x0118), true + case "Eopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL E + return rune(0x01d53c), true + case "Epsilon": + // GREEK CAPITAL LETTER EPSILON + return rune(0x0395), true + case "EqualTilde": + // MINUS TILDE + return rune(0x2242), true + case "Equal": + // TWO CONSECUTIVE EQUALS SIGNS + return rune(0x2a75), true + case "Equilibrium": + // RIGHTWARDS HARPOON OVER LEFTWARDS HARPOON + return rune(0x21cc), true + case "Escr": + // SCRIPT CAPITAL E + return rune(0x2130), true + case "Esim": + // EQUALS SIGN ABOVE TILDE OPERATOR + return rune(0x2a73), true + case "Eta": + // GREEK CAPITAL LETTER ETA + return rune(0x0397), true + case "Euml": + // LATIN CAPITAL LETTER E WITH DIAERESIS + return rune(0xcb), true + case "Exists": + // THERE EXISTS + return rune(0x2203), true + case "ExponentialE": + // DOUBLE-STRUCK ITALIC SMALL E + return rune(0x2147), true + } + + case 'F': + switch name { + case "Fcy": + // CYRILLIC CAPITAL LETTER EF + return rune(0x0424), true + case "Ffr": + // MATHEMATICAL FRAKTUR CAPITAL F + return rune(0x01d509), true + case "FilledSmallSquare": + // BLACK MEDIUM SQUARE + return rune(0x25fc), true + case "FilledVerySmallSquare": + // BLACK SMALL SQUARE + return rune(0x25aa), true + case "Fopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL F + return rune(0x01d53d), true + case "ForAll": + // FOR ALL + return rune(0x2200), true + case "Fouriertrf": + // SCRIPT CAPITAL F + return rune(0x2131), true + case "Fscr": + // SCRIPT CAPITAL F + return rune(0x2131), true + } + + case 'G': + switch name { + case "GJcy": + // CYRILLIC CAPITAL LETTER GJE + return rune(0x0403), true + case "GT": + // GREATER-THAN SIGN + return rune(0x3e), true + case "Game": + // TURNED SANS-SERIF CAPITAL G + return rune(0x2141), true + case "Gamma": + // GREEK CAPITAL LETTER GAMMA + return rune(0x0393), true + case "Gammad": + // GREEK LETTER DIGAMMA + return rune(0x03dc), true + case "Gbreve": + // LATIN CAPITAL LETTER G WITH BREVE + return rune(0x011e), true + case "Gcedil": + // LATIN CAPITAL LETTER G WITH CEDILLA + return rune(0x0122), true + case "Gcirc": + // LATIN CAPITAL LETTER G WITH CIRCUMFLEX + return rune(0x011c), true + case "Gcy": + // CYRILLIC CAPITAL LETTER GHE + return rune(0x0413), true + case "Gdot": + // LATIN CAPITAL LETTER G WITH DOT ABOVE + return rune(0x0120), true + case "Gfr": + // MATHEMATICAL FRAKTUR CAPITAL G + return rune(0x01d50a), true + case "Ggr": + // GREEK CAPITAL LETTER GAMMA + return rune(0x0393), true + case "Gg": + // VERY MUCH GREATER-THAN + return rune(0x22d9), true + case "Gopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL G + return rune(0x01d53e), true + case "GreaterEqual": + // GREATER-THAN OR EQUAL TO + return rune(0x2265), true + case "GreaterEqualLess": + // GREATER-THAN EQUAL TO OR LESS-THAN + return rune(0x22db), true + case "GreaterFullEqual": + // GREATER-THAN OVER EQUAL TO + return rune(0x2267), true + case "GreaterGreater": + // DOUBLE NESTED GREATER-THAN + return rune(0x2aa2), true + case "GreaterLess": + // GREATER-THAN OR LESS-THAN + return rune(0x2277), true + case "GreaterSlantEqual": + // GREATER-THAN OR SLANTED EQUAL TO + return rune(0x2a7e), true + case "GreaterTilde": + // GREATER-THAN OR EQUIVALENT TO + return rune(0x2273), true + case "Gscr": + // MATHEMATICAL SCRIPT CAPITAL G + return rune(0x01d4a2), true + case "Gt": + // MUCH GREATER-THAN + return rune(0x226b), true + } + + case 'H': + switch name { + case "HARDcy": + // CYRILLIC CAPITAL LETTER HARD SIGN + return rune(0x042a), true + case "Hacek": + // CARON + return rune(0x02c7), true + case "Hat": + // CIRCUMFLEX ACCENT + return rune(0x5e), true + case "Hcirc": + // LATIN CAPITAL LETTER H WITH CIRCUMFLEX + return rune(0x0124), true + case "Hfr": + // BLACK-LETTER CAPITAL H + return rune(0x210c), true + case "HilbertSpace": + // SCRIPT CAPITAL H + return rune(0x210b), true + case "Hopf": + // DOUBLE-STRUCK CAPITAL H + return rune(0x210d), true + case "HorizontalLine": + // BOX DRAWINGS LIGHT HORIZONTAL + return rune(0x2500), true + case "Hscr": + // SCRIPT CAPITAL H + return rune(0x210b), true + case "Hstrok": + // LATIN CAPITAL LETTER H WITH STROKE + return rune(0x0126), true + case "HumpDownHump": + // GEOMETRICALLY EQUIVALENT TO + return rune(0x224e), true + case "HumpEqual": + // DIFFERENCE BETWEEN + return rune(0x224f), true + } + + case 'I': + switch name { + case "IEcy": + // CYRILLIC CAPITAL LETTER IE + return rune(0x0415), true + case "IJlig": + // LATIN CAPITAL LIGATURE IJ + return rune(0x0132), true + case "IOcy": + // CYRILLIC CAPITAL LETTER IO + return rune(0x0401), true + case "Iacgr": + // GREEK CAPITAL LETTER IOTA WITH TONOS + return rune(0x038a), true + case "Iacute": + // LATIN CAPITAL LETTER I WITH ACUTE + return rune(0xcd), true + case "Icirc": + // LATIN CAPITAL LETTER I WITH CIRCUMFLEX + return rune(0xce), true + case "Icy": + // CYRILLIC CAPITAL LETTER I + return rune(0x0418), true + case "Idigr": + // GREEK CAPITAL LETTER IOTA WITH DIALYTIKA + return rune(0x03aa), true + case "Idot": + // LATIN CAPITAL LETTER I WITH DOT ABOVE + return rune(0x0130), true + case "Ifr": + // BLACK-LETTER CAPITAL I + return rune(0x2111), true + case "Igrave": + // LATIN CAPITAL LETTER I WITH GRAVE + return rune(0xcc), true + case "Igr": + // GREEK CAPITAL LETTER IOTA + return rune(0x0399), true + case "Imacr": + // LATIN CAPITAL LETTER I WITH MACRON + return rune(0x012a), true + case "ImaginaryI": + // DOUBLE-STRUCK ITALIC SMALL I + return rune(0x2148), true + case "Implies": + // RIGHTWARDS DOUBLE ARROW + return rune(0x21d2), true + case "Im": + // BLACK-LETTER CAPITAL I + return rune(0x2111), true + case "Integral": + // INTEGRAL + return rune(0x222b), true + case "Int": + // DOUBLE INTEGRAL + return rune(0x222c), true + case "Intersection": + // N-ARY INTERSECTION + return rune(0x22c2), true + case "InvisibleComma": + // INVISIBLE SEPARATOR + return rune(0x2063), true + case "InvisibleTimes": + // INVISIBLE TIMES + return rune(0x2062), true + case "Iogon": + // LATIN CAPITAL LETTER I WITH OGONEK + return rune(0x012e), true + case "Iopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL I + return rune(0x01d540), true + case "Iota": + // GREEK CAPITAL LETTER IOTA + return rune(0x0399), true + case "Iscr": + // SCRIPT CAPITAL I + return rune(0x2110), true + case "Itilde": + // LATIN CAPITAL LETTER I WITH TILDE + return rune(0x0128), true + case "Iukcy": + // CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I + return rune(0x0406), true + case "Iuml": + // LATIN CAPITAL LETTER I WITH DIAERESIS + return rune(0xcf), true + } + + case 'J': + switch name { + case "Jcirc": + // LATIN CAPITAL LETTER J WITH CIRCUMFLEX + return rune(0x0134), true + case "Jcy": + // CYRILLIC CAPITAL LETTER SHORT I + return rune(0x0419), true + case "Jfr": + // MATHEMATICAL FRAKTUR CAPITAL J + return rune(0x01d50d), true + case "Jopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL J + return rune(0x01d541), true + case "Jscr": + // MATHEMATICAL SCRIPT CAPITAL J + return rune(0x01d4a5), true + case "Jsercy": + // CYRILLIC CAPITAL LETTER JE + return rune(0x0408), true + case "Jukcy": + // CYRILLIC CAPITAL LETTER UKRAINIAN IE + return rune(0x0404), true + } + + case 'K': + switch name { + case "KHcy": + // CYRILLIC CAPITAL LETTER HA + return rune(0x0425), true + case "KHgr": + // GREEK CAPITAL LETTER CHI + return rune(0x03a7), true + case "KJcy": + // CYRILLIC CAPITAL LETTER KJE + return rune(0x040c), true + case "Kappa": + // GREEK CAPITAL LETTER KAPPA + return rune(0x039a), true + case "Kcedil": + // LATIN CAPITAL LETTER K WITH CEDILLA + return rune(0x0136), true + case "Kcy": + // CYRILLIC CAPITAL LETTER KA + return rune(0x041a), true + case "Kfr": + // MATHEMATICAL FRAKTUR CAPITAL K + return rune(0x01d50e), true + case "Kgr": + // GREEK CAPITAL LETTER KAPPA + return rune(0x039a), true + case "Kopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL K + return rune(0x01d542), true + case "Kscr": + // MATHEMATICAL SCRIPT CAPITAL K + return rune(0x01d4a6), true + } + + case 'L': + switch name { + case "LJcy": + // CYRILLIC CAPITAL LETTER LJE + return rune(0x0409), true + case "LT": + // LESS-THAN SIGN + return rune(0x3c), true + case "Lacute": + // LATIN CAPITAL LETTER L WITH ACUTE + return rune(0x0139), true + case "Lambda": + // GREEK CAPITAL LETTER LAMDA + return rune(0x039b), true + case "Lang": + // MATHEMATICAL LEFT DOUBLE ANGLE BRACKET + return rune(0x27ea), true + case "Laplacetrf": + // SCRIPT CAPITAL L + return rune(0x2112), true + case "Larr": + // LEFTWARDS TWO HEADED ARROW + return rune(0x219e), true + case "Lcaron": + // LATIN CAPITAL LETTER L WITH CARON + return rune(0x013d), true + case "Lcedil": + // LATIN CAPITAL LETTER L WITH CEDILLA + return rune(0x013b), true + case "Lcy": + // CYRILLIC CAPITAL LETTER EL + return rune(0x041b), true + case "LeftAngleBracket": + // MATHEMATICAL LEFT ANGLE BRACKET + return rune(0x27e8), true + case "LeftArrowBar": + // LEFTWARDS ARROW TO BAR + return rune(0x21e4), true + case "LeftArrowRightArrow": + // LEFTWARDS ARROW OVER RIGHTWARDS ARROW + return rune(0x21c6), true + case "LeftArrow": + // LEFTWARDS ARROW + return rune(0x2190), true + case "LeftCeiling": + // LEFT CEILING + return rune(0x2308), true + case "LeftDoubleBracket": + // MATHEMATICAL LEFT WHITE SQUARE BRACKET + return rune(0x27e6), true + case "LeftDownTeeVector": + // DOWNWARDS HARPOON WITH BARB LEFT FROM BAR + return rune(0x2961), true + case "LeftDownVector": + // DOWNWARDS HARPOON WITH BARB LEFTWARDS + return rune(0x21c3), true + case "LeftDownVectorBar": + // DOWNWARDS HARPOON WITH BARB LEFT TO BAR + return rune(0x2959), true + case "LeftFloor": + // LEFT FLOOR + return rune(0x230a), true + case "LeftRightArrow": + // LEFT RIGHT ARROW + return rune(0x2194), true + case "LeftRightVector": + // LEFT BARB UP RIGHT BARB UP HARPOON + return rune(0x294e), true + case "LeftTeeArrow": + // LEFTWARDS ARROW FROM BAR + return rune(0x21a4), true + case "LeftTeeVector": + // LEFTWARDS HARPOON WITH BARB UP FROM BAR + return rune(0x295a), true + case "LeftTee": + // LEFT TACK + return rune(0x22a3), true + case "LeftTriangleBar": + // LEFT TRIANGLE BESIDE VERTICAL BAR + return rune(0x29cf), true + case "LeftTriangle": + // NORMAL SUBGROUP OF + return rune(0x22b2), true + case "LeftTriangleEqual": + // NORMAL SUBGROUP OF OR EQUAL TO + return rune(0x22b4), true + case "LeftUpDownVector": + // UP BARB LEFT DOWN BARB LEFT HARPOON + return rune(0x2951), true + case "LeftUpTeeVector": + // UPWARDS HARPOON WITH BARB LEFT FROM BAR + return rune(0x2960), true + case "LeftUpVector": + // UPWARDS HARPOON WITH BARB LEFTWARDS + return rune(0x21bf), true + case "LeftUpVectorBar": + // UPWARDS HARPOON WITH BARB LEFT TO BAR + return rune(0x2958), true + case "LeftVector": + // LEFTWARDS HARPOON WITH BARB UPWARDS + return rune(0x21bc), true + case "LeftVectorBar": + // LEFTWARDS HARPOON WITH BARB UP TO BAR + return rune(0x2952), true + case "Leftarrow": + // LEFTWARDS DOUBLE ARROW + return rune(0x21d0), true + case "Leftrightarrow": + // LEFT RIGHT DOUBLE ARROW + return rune(0x21d4), true + case "LessEqualGreater": + // LESS-THAN EQUAL TO OR GREATER-THAN + return rune(0x22da), true + case "LessFullEqual": + // LESS-THAN OVER EQUAL TO + return rune(0x2266), true + case "LessGreater": + // LESS-THAN OR GREATER-THAN + return rune(0x2276), true + case "LessLess": + // DOUBLE NESTED LESS-THAN + return rune(0x2aa1), true + case "LessSlantEqual": + // LESS-THAN OR SLANTED EQUAL TO + return rune(0x2a7d), true + case "LessTilde": + // LESS-THAN OR EQUIVALENT TO + return rune(0x2272), true + case "Lfr": + // MATHEMATICAL FRAKTUR CAPITAL L + return rune(0x01d50f), true + case "Lgr": + // GREEK CAPITAL LETTER LAMDA + return rune(0x039b), true + case "Lleftarrow": + // LEFTWARDS TRIPLE ARROW + return rune(0x21da), true + case "Ll": + // VERY MUCH LESS-THAN + return rune(0x22d8), true + case "Lmidot": + // LATIN CAPITAL LETTER L WITH MIDDLE DOT + return rune(0x013f), true + case "LongLeftArrow": + // LONG LEFTWARDS ARROW + return rune(0x27f5), true + case "LongLeftRightArrow": + // LONG LEFT RIGHT ARROW + return rune(0x27f7), true + case "LongRightArrow": + // LONG RIGHTWARDS ARROW + return rune(0x27f6), true + case "Longleftarrow": + // LONG LEFTWARDS DOUBLE ARROW + return rune(0x27f8), true + case "Longleftrightarrow": + // LONG LEFT RIGHT DOUBLE ARROW + return rune(0x27fa), true + case "Longrightarrow": + // LONG RIGHTWARDS DOUBLE ARROW + return rune(0x27f9), true + case "Lopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL L + return rune(0x01d543), true + case "LowerLeftArrow": + // SOUTH WEST ARROW + return rune(0x2199), true + case "LowerRightArrow": + // SOUTH EAST ARROW + return rune(0x2198), true + case "Lscr": + // SCRIPT CAPITAL L + return rune(0x2112), true + case "Lsh": + // UPWARDS ARROW WITH TIP LEFTWARDS + return rune(0x21b0), true + case "Lstrok": + // LATIN CAPITAL LETTER L WITH STROKE + return rune(0x0141), true + case "Ltbar": + // DOUBLE NESTED LESS-THAN WITH UNDERBAR + return rune(0x2aa3), true + case "Lt": + // MUCH LESS-THAN + return rune(0x226a), true + } + + case 'M': + switch name { + case "Mapfrom": + // LEFTWARDS DOUBLE ARROW FROM BAR + return rune(0x2906), true + case "Mapto": + // RIGHTWARDS DOUBLE ARROW FROM BAR + return rune(0x2907), true + case "Map": + // RIGHTWARDS TWO-HEADED ARROW FROM BAR + return rune(0x2905), true + case "Mcy": + // CYRILLIC CAPITAL LETTER EM + return rune(0x041c), true + case "MediumSpace": + // MEDIUM MATHEMATICAL SPACE + return rune(0x205f), true + case "Mellintrf": + // SCRIPT CAPITAL M + return rune(0x2133), true + case "Mfr": + // MATHEMATICAL FRAKTUR CAPITAL M + return rune(0x01d510), true + case "Mgr": + // GREEK CAPITAL LETTER MU + return rune(0x039c), true + case "MinusPlus": + // MINUS-OR-PLUS SIGN + return rune(0x2213), true + case "Mopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL M + return rune(0x01d544), true + case "Mscr": + // SCRIPT CAPITAL M + return rune(0x2133), true + case "Mu": + // GREEK CAPITAL LETTER MU + return rune(0x039c), true + } + + case 'N': + switch name { + case "NJcy": + // CYRILLIC CAPITAL LETTER NJE + return rune(0x040a), true + case "Nacute": + // LATIN CAPITAL LETTER N WITH ACUTE + return rune(0x0143), true + case "Ncaron": + // LATIN CAPITAL LETTER N WITH CARON + return rune(0x0147), true + case "Ncedil": + // LATIN CAPITAL LETTER N WITH CEDILLA + return rune(0x0145), true + case "Ncy": + // CYRILLIC CAPITAL LETTER EN + return rune(0x041d), true + case "NegativeMediumSpace": + // ZERO WIDTH SPACE + return rune(0x200b), true + case "NegativeThickSpace": + // ZERO WIDTH SPACE + return rune(0x200b), true + case "NegativeThinSpace": + // ZERO WIDTH SPACE + return rune(0x200b), true + case "NegativeVeryThinSpace": + // ZERO WIDTH SPACE + return rune(0x200b), true + case "NestedGreaterGreater": + // MUCH GREATER-THAN + return rune(0x226b), true + case "NestedLessLess": + // MUCH LESS-THAN + return rune(0x226a), true + case "NewLine": + // LINE FEED (LF) + return rune(0x0a), true + case "Nfr": + // MATHEMATICAL FRAKTUR CAPITAL N + return rune(0x01d511), true + case "Ngr": + // GREEK CAPITAL LETTER NU + return rune(0x039d), true + case "NoBreak": + // WORD JOINER + return rune(0x2060), true + case "NonBreakingSpace": + // NO-BREAK SPACE + return rune(0xa0), true + case "Nopf": + // DOUBLE-STRUCK CAPITAL N + return rune(0x2115), true + case "NotDoubleVerticalBar": + // NOT PARALLEL TO + return rune(0x2226), true + case "NotElement": + // NOT AN ELEMENT OF + return rune(0x2209), true + case "NotEqualTilde": + // MINUS TILDE with slash + return rune(0x2242), true + case "NotEqual": + // NOT EQUAL TO + return rune(0x2260), true + case "NotExists": + // THERE DOES NOT EXIST + return rune(0x2204), true + case "NotHumpDownHump": + // GEOMETRICALLY EQUIVALENT TO with slash + return rune(0x224e), true + case "NotHumpEqual": + // DIFFERENCE BETWEEN with slash + return rune(0x224f), true + case "NotLessGreater": + // NEITHER LESS-THAN NOR GREATER-THAN + return rune(0x2278), true + case "NotReverseElement": + // DOES NOT CONTAIN AS MEMBER + return rune(0x220c), true + case "NotTilde": + // NOT TILDE + return rune(0x2241), true + case "NotTildeEqual": + // NOT ASYMPTOTICALLY EQUAL TO + return rune(0x2244), true + case "NotTildeFullEqual": + // NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO + return rune(0x2247), true + case "NotTildeTilde": + // NOT ALMOST EQUAL TO + return rune(0x2249), true + case "NotVerticalBar": + // DOES NOT DIVIDE + return rune(0x2224), true + case "Not": + // DOUBLE STROKE NOT SIGN + return rune(0x2aec), true + case "NotCongruent": + // NOT IDENTICAL TO + return rune(0x2262), true + case "NotCupCap": + // NOT EQUIVALENT TO + return rune(0x226d), true + case "NotGreaterFullEqual": + // GREATER-THAN OVER EQUAL TO with slash + return rune(0x2267), true + case "NotGreaterGreater": + // MUCH GREATER THAN with slash + return rune(0x226b), true + case "NotGreaterSlantEqual": + // GREATER-THAN OR SLANTED EQUAL TO with slash + return rune(0x2a7e), true + case "NotGreater": + // NOT GREATER-THAN + return rune(0x226f), true + case "NotGreaterEqual": + // NEITHER GREATER-THAN NOR EQUAL TO + return rune(0x2271), true + case "NotGreaterLess": + // NEITHER GREATER-THAN NOR LESS-THAN + return rune(0x2279), true + case "NotGreaterTilde": + // NEITHER GREATER-THAN NOR EQUIVALENT TO + return rune(0x2275), true + case "NotLeftTriangleBar": + // LEFT TRIANGLE BESIDE VERTICAL BAR with slash + return rune(0x29cf), true + case "NotLeftTriangle": + // NOT NORMAL SUBGROUP OF + return rune(0x22ea), true + case "NotLeftTriangleEqual": + // NOT NORMAL SUBGROUP OF OR EQUAL TO + return rune(0x22ec), true + case "NotLessLess": + // MUCH LESS THAN with slash + return rune(0x226a), true + case "NotLessSlantEqual": + // LESS-THAN OR SLANTED EQUAL TO with slash + return rune(0x2a7d), true + case "NotLess": + // NOT LESS-THAN + return rune(0x226e), true + case "NotLessEqual": + // NEITHER LESS-THAN NOR EQUAL TO + return rune(0x2270), true + case "NotLessTilde": + // NEITHER LESS-THAN NOR EQUIVALENT TO + return rune(0x2274), true + case "NotNestedGreaterGreater": + // DOUBLE NESTED GREATER-THAN with slash + return rune(0x2aa2), true + case "NotNestedLessLess": + // DOUBLE NESTED LESS-THAN with slash + return rune(0x2aa1), true + case "NotPrecedesEqual": + // PRECEDES ABOVE SINGLE-LINE EQUALS SIGN with slash + return rune(0x2aaf), true + case "NotPrecedes": + // DOES NOT PRECEDE + return rune(0x2280), true + case "NotPrecedesSlantEqual": + // DOES NOT PRECEDE OR EQUAL + return rune(0x22e0), true + case "NotRightTriangleBar": + // VERTICAL BAR BESIDE RIGHT TRIANGLE with slash + return rune(0x29d0), true + case "NotRightTriangle": + // DOES NOT CONTAIN AS NORMAL SUBGROUP + return rune(0x22eb), true + case "NotRightTriangleEqual": + // DOES NOT CONTAIN AS NORMAL SUBGROUP OR EQUAL + return rune(0x22ed), true + case "NotSquareSubset": + // SQUARE IMAGE OF with slash + return rune(0x228f), true + case "NotSquareSubsetEqual": + // NOT SQUARE IMAGE OF OR EQUAL TO + return rune(0x22e2), true + case "NotSquareSuperset": + // SQUARE ORIGINAL OF with slash + return rune(0x2290), true + case "NotSquareSupersetEqual": + // NOT SQUARE ORIGINAL OF OR EQUAL TO + return rune(0x22e3), true + case "NotSubset": + // SUBSET OF with vertical line + return rune(0x2282), true + case "NotSubsetEqual": + // NEITHER A SUBSET OF NOR EQUAL TO + return rune(0x2288), true + case "NotSucceedsEqual": + // SUCCEEDS ABOVE SINGLE-LINE EQUALS SIGN with slash + return rune(0x2ab0), true + case "NotSucceedsTilde": + // SUCCEEDS OR EQUIVALENT TO with slash + return rune(0x227f), true + case "NotSucceeds": + // DOES NOT SUCCEED + return rune(0x2281), true + case "NotSucceedsSlantEqual": + // DOES NOT SUCCEED OR EQUAL + return rune(0x22e1), true + case "NotSuperset": + // SUPERSET OF with vertical line + return rune(0x2283), true + case "NotSupersetEqual": + // NEITHER A SUPERSET OF NOR EQUAL TO + return rune(0x2289), true + case "Nscr": + // MATHEMATICAL SCRIPT CAPITAL N + return rune(0x01d4a9), true + case "Ntilde": + // LATIN CAPITAL LETTER N WITH TILDE + return rune(0xd1), true + case "Nu": + // GREEK CAPITAL LETTER NU + return rune(0x039d), true + } + + case 'O': + switch name { + case "OElig": + // LATIN CAPITAL LIGATURE OE + return rune(0x0152), true + case "OHacgr": + // GREEK CAPITAL LETTER OMEGA WITH TONOS + return rune(0x038f), true + case "OHgr": + // GREEK CAPITAL LETTER OMEGA + return rune(0x03a9), true + case "Oacgr": + // GREEK CAPITAL LETTER OMICRON WITH TONOS + return rune(0x038c), true + case "Oacute": + // LATIN CAPITAL LETTER O WITH ACUTE + return rune(0xd3), true + case "Ocirc": + // LATIN CAPITAL LETTER O WITH CIRCUMFLEX + return rune(0xd4), true + case "Ocy": + // CYRILLIC CAPITAL LETTER O + return rune(0x041e), true + case "Odblac": + // LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + return rune(0x0150), true + case "Ofr": + // MATHEMATICAL FRAKTUR CAPITAL O + return rune(0x01d512), true + case "Ograve": + // LATIN CAPITAL LETTER O WITH GRAVE + return rune(0xd2), true + case "Ogr": + // GREEK CAPITAL LETTER OMICRON + return rune(0x039f), true + case "Omacr": + // LATIN CAPITAL LETTER O WITH MACRON + return rune(0x014c), true + case "Omega": + // GREEK CAPITAL LETTER OMEGA + return rune(0x03a9), true + case "Omicron": + // GREEK CAPITAL LETTER OMICRON + return rune(0x039f), true + case "Oopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL O + return rune(0x01d546), true + case "OpenCurlyDoubleQuote": + // LEFT DOUBLE QUOTATION MARK + return rune(0x201c), true + case "OpenCurlyQuote": + // LEFT SINGLE QUOTATION MARK + return rune(0x2018), true + case "Or": + // DOUBLE LOGICAL OR + return rune(0x2a54), true + case "Oscr": + // MATHEMATICAL SCRIPT CAPITAL O + return rune(0x01d4aa), true + case "Oslash": + // LATIN CAPITAL LETTER O WITH STROKE + return rune(0xd8), true + case "Otilde": + // LATIN CAPITAL LETTER O WITH TILDE + return rune(0xd5), true + case "Otimes": + // MULTIPLICATION SIGN IN DOUBLE CIRCLE + return rune(0x2a37), true + case "Ouml": + // LATIN CAPITAL LETTER O WITH DIAERESIS + return rune(0xd6), true + case "OverBar": + // OVERLINE + return rune(0x203e), true + case "OverBrace": + // TOP CURLY BRACKET + return rune(0x23de), true + case "OverBracket": + // TOP SQUARE BRACKET + return rune(0x23b4), true + case "OverParenthesis": + // TOP PARENTHESIS + return rune(0x23dc), true + } + + case 'P': + switch name { + case "PHgr": + // GREEK CAPITAL LETTER PHI + return rune(0x03a6), true + case "PSgr": + // GREEK CAPITAL LETTER PSI + return rune(0x03a8), true + case "PartialD": + // PARTIAL DIFFERENTIAL + return rune(0x2202), true + case "Pcy": + // CYRILLIC CAPITAL LETTER PE + return rune(0x041f), true + case "Pfr": + // MATHEMATICAL FRAKTUR CAPITAL P + return rune(0x01d513), true + case "Pgr": + // GREEK CAPITAL LETTER PI + return rune(0x03a0), true + case "Phi": + // GREEK CAPITAL LETTER PHI + return rune(0x03a6), true + case "Pi": + // GREEK CAPITAL LETTER PI + return rune(0x03a0), true + case "PlusMinus": + // PLUS-MINUS SIGN + return rune(0xb1), true + case "Poincareplane": + // BLACK-LETTER CAPITAL H + return rune(0x210c), true + case "Popf": + // DOUBLE-STRUCK CAPITAL P + return rune(0x2119), true + case "Product": + // N-ARY PRODUCT + return rune(0x220f), true + case "Proportional": + // PROPORTIONAL TO + return rune(0x221d), true + case "Proportion": + // PROPORTION + return rune(0x2237), true + case "Pr": + // DOUBLE PRECEDES + return rune(0x2abb), true + case "PrecedesEqual": + // PRECEDES ABOVE SINGLE-LINE EQUALS SIGN + return rune(0x2aaf), true + case "Precedes": + // PRECEDES + return rune(0x227a), true + case "PrecedesSlantEqual": + // PRECEDES OR EQUAL TO + return rune(0x227c), true + case "PrecedesTilde": + // PRECEDES OR EQUIVALENT TO + return rune(0x227e), true + case "Prime": + // DOUBLE PRIME + return rune(0x2033), true + case "Pscr": + // MATHEMATICAL SCRIPT CAPITAL P + return rune(0x01d4ab), true + case "Psi": + // GREEK CAPITAL LETTER PSI + return rune(0x03a8), true + } + + case 'Q': + switch name { + case "QUOT": + // QUOTATION MARK + return rune(0x22), true + case "Qfr": + // MATHEMATICAL FRAKTUR CAPITAL Q + return rune(0x01d514), true + case "Qopf": + // DOUBLE-STRUCK CAPITAL Q + return rune(0x211a), true + case "Qscr": + // MATHEMATICAL SCRIPT CAPITAL Q + return rune(0x01d4ac), true + } + + case 'R': + switch name { + case "RBarr": + // RIGHTWARDS TWO-HEADED TRIPLE DASH ARROW + return rune(0x2910), true + case "REG": + // REGISTERED SIGN + return rune(0xae), true + case "Racute": + // LATIN CAPITAL LETTER R WITH ACUTE + return rune(0x0154), true + case "Rang": + // MATHEMATICAL RIGHT DOUBLE ANGLE BRACKET + return rune(0x27eb), true + case "Rarr": + // RIGHTWARDS TWO HEADED ARROW + return rune(0x21a0), true + case "Rarrtl": + // RIGHTWARDS TWO-HEADED ARROW WITH TAIL + return rune(0x2916), true + case "Rcaron": + // LATIN CAPITAL LETTER R WITH CARON + return rune(0x0158), true + case "Rcedil": + // LATIN CAPITAL LETTER R WITH CEDILLA + return rune(0x0156), true + case "Rcy": + // CYRILLIC CAPITAL LETTER ER + return rune(0x0420), true + case "ReverseElement": + // CONTAINS AS MEMBER + return rune(0x220b), true + case "ReverseEquilibrium": + // LEFTWARDS HARPOON OVER RIGHTWARDS HARPOON + return rune(0x21cb), true + case "Re": + // BLACK-LETTER CAPITAL R + return rune(0x211c), true + case "ReverseUpEquilibrium": + // DOWNWARDS HARPOON WITH BARB LEFT BESIDE UPWARDS HARPOON WITH BARB RIGHT + return rune(0x296f), true + case "Rfr": + // BLACK-LETTER CAPITAL R + return rune(0x211c), true + case "Rgr": + // GREEK CAPITAL LETTER RHO + return rune(0x03a1), true + case "Rho": + // GREEK CAPITAL LETTER RHO + return rune(0x03a1), true + case "RightAngleBracket": + // MATHEMATICAL RIGHT ANGLE BRACKET + return rune(0x27e9), true + case "RightArrowBar": + // RIGHTWARDS ARROW TO BAR + return rune(0x21e5), true + case "RightArrow": + // RIGHTWARDS ARROW + return rune(0x2192), true + case "RightArrowLeftArrow": + // RIGHTWARDS ARROW OVER LEFTWARDS ARROW + return rune(0x21c4), true + case "RightCeiling": + // RIGHT CEILING + return rune(0x2309), true + case "RightDoubleBracket": + // MATHEMATICAL RIGHT WHITE SQUARE BRACKET + return rune(0x27e7), true + case "RightDownTeeVector": + // DOWNWARDS HARPOON WITH BARB RIGHT FROM BAR + return rune(0x295d), true + case "RightDownVector": + // DOWNWARDS HARPOON WITH BARB RIGHTWARDS + return rune(0x21c2), true + case "RightDownVectorBar": + // DOWNWARDS HARPOON WITH BARB RIGHT TO BAR + return rune(0x2955), true + case "RightFloor": + // RIGHT FLOOR + return rune(0x230b), true + case "RightTeeArrow": + // RIGHTWARDS ARROW FROM BAR + return rune(0x21a6), true + case "RightTeeVector": + // RIGHTWARDS HARPOON WITH BARB UP FROM BAR + return rune(0x295b), true + case "RightTee": + // RIGHT TACK + return rune(0x22a2), true + case "RightTriangleBar": + // VERTICAL BAR BESIDE RIGHT TRIANGLE + return rune(0x29d0), true + case "RightTriangle": + // CONTAINS AS NORMAL SUBGROUP + return rune(0x22b3), true + case "RightTriangleEqual": + // CONTAINS AS NORMAL SUBGROUP OR EQUAL TO + return rune(0x22b5), true + case "RightUpDownVector": + // UP BARB RIGHT DOWN BARB RIGHT HARPOON + return rune(0x294f), true + case "RightUpTeeVector": + // UPWARDS HARPOON WITH BARB RIGHT FROM BAR + return rune(0x295c), true + case "RightUpVector": + // UPWARDS HARPOON WITH BARB RIGHTWARDS + return rune(0x21be), true + case "RightUpVectorBar": + // UPWARDS HARPOON WITH BARB RIGHT TO BAR + return rune(0x2954), true + case "RightVector": + // RIGHTWARDS HARPOON WITH BARB UPWARDS + return rune(0x21c0), true + case "RightVectorBar": + // RIGHTWARDS HARPOON WITH BARB UP TO BAR + return rune(0x2953), true + case "Rightarrow": + // RIGHTWARDS DOUBLE ARROW + return rune(0x21d2), true + case "Ropf": + // DOUBLE-STRUCK CAPITAL R + return rune(0x211d), true + case "RoundImplies": + // RIGHT DOUBLE ARROW WITH ROUNDED HEAD + return rune(0x2970), true + case "Rrightarrow": + // RIGHTWARDS TRIPLE ARROW + return rune(0x21db), true + case "Rscr": + // SCRIPT CAPITAL R + return rune(0x211b), true + case "Rsh": + // UPWARDS ARROW WITH TIP RIGHTWARDS + return rune(0x21b1), true + case "RuleDelayed": + // RULE-DELAYED + return rune(0x29f4), true + } + + case 'S': + switch name { + case "SHCHcy": + // CYRILLIC CAPITAL LETTER SHCHA + return rune(0x0429), true + case "SHcy": + // CYRILLIC CAPITAL LETTER SHA + return rune(0x0428), true + case "SOFTcy": + // CYRILLIC CAPITAL LETTER SOFT SIGN + return rune(0x042c), true + case "Sacute": + // LATIN CAPITAL LETTER S WITH ACUTE + return rune(0x015a), true + case "Sc": + // DOUBLE SUCCEEDS + return rune(0x2abc), true + case "Scaron": + // LATIN CAPITAL LETTER S WITH CARON + return rune(0x0160), true + case "Scedil": + // LATIN CAPITAL LETTER S WITH CEDILLA + return rune(0x015e), true + case "Scirc": + // LATIN CAPITAL LETTER S WITH CIRCUMFLEX + return rune(0x015c), true + case "Scy": + // CYRILLIC CAPITAL LETTER ES + return rune(0x0421), true + case "Sfr": + // MATHEMATICAL FRAKTUR CAPITAL S + return rune(0x01d516), true + case "Sgr": + // GREEK CAPITAL LETTER SIGMA + return rune(0x03a3), true + case "ShortDownArrow": + // DOWNWARDS ARROW + return rune(0x2193), true + case "ShortLeftArrow": + // LEFTWARDS ARROW + return rune(0x2190), true + case "ShortRightArrow": + // RIGHTWARDS ARROW + return rune(0x2192), true + case "ShortUpArrow": + // UPWARDS ARROW + return rune(0x2191), true + case "Sigma": + // GREEK CAPITAL LETTER SIGMA + return rune(0x03a3), true + case "SmallCircle": + // RING OPERATOR + return rune(0x2218), true + case "Sopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL S + return rune(0x01d54a), true + case "Sqrt": + // SQUARE ROOT + return rune(0x221a), true + case "SquareIntersection": + // SQUARE CAP + return rune(0x2293), true + case "SquareSubset": + // SQUARE IMAGE OF + return rune(0x228f), true + case "SquareSubsetEqual": + // SQUARE IMAGE OF OR EQUAL TO + return rune(0x2291), true + case "Square": + // WHITE SQUARE + return rune(0x25a1), true + case "SquareSuperset": + // SQUARE ORIGINAL OF + return rune(0x2290), true + case "SquareSupersetEqual": + // SQUARE ORIGINAL OF OR EQUAL TO + return rune(0x2292), true + case "SquareUnion": + // SQUARE CUP + return rune(0x2294), true + case "Sscr": + // MATHEMATICAL SCRIPT CAPITAL S + return rune(0x01d4ae), true + case "Star": + // STAR OPERATOR + return rune(0x22c6), true + case "Sub": + // DOUBLE SUBSET + return rune(0x22d0), true + case "Subset": + // DOUBLE SUBSET + return rune(0x22d0), true + case "SubsetEqual": + // SUBSET OF OR EQUAL TO + return rune(0x2286), true + case "Succeeds": + // SUCCEEDS + return rune(0x227b), true + case "SucceedsEqual": + // SUCCEEDS ABOVE SINGLE-LINE EQUALS SIGN + return rune(0x2ab0), true + case "SucceedsSlantEqual": + // SUCCEEDS OR EQUAL TO + return rune(0x227d), true + case "SucceedsTilde": + // SUCCEEDS OR EQUIVALENT TO + return rune(0x227f), true + case "SuchThat": + // CONTAINS AS MEMBER + return rune(0x220b), true + case "Sum": + // N-ARY SUMMATION + return rune(0x2211), true + case "SupersetEqual": + // SUPERSET OF OR EQUAL TO + return rune(0x2287), true + case "Sup": + // DOUBLE SUPERSET + return rune(0x22d1), true + case "Superset": + // SUPERSET OF + return rune(0x2283), true + case "Supset": + // DOUBLE SUPERSET + return rune(0x22d1), true + } + + case 'T': + switch name { + case "THORN": + // LATIN CAPITAL LETTER THORN + return rune(0xde), true + case "THgr": + // GREEK CAPITAL LETTER THETA + return rune(0x0398), true + case "TRADE": + // TRADE MARK SIGN + return rune(0x2122), true + case "TSHcy": + // CYRILLIC CAPITAL LETTER TSHE + return rune(0x040b), true + case "TScy": + // CYRILLIC CAPITAL LETTER TSE + return rune(0x0426), true + case "Tab": + // CHARACTER TABULATION + return rune(0x09), true + case "Tau": + // GREEK CAPITAL LETTER TAU + return rune(0x03a4), true + case "Tcaron": + // LATIN CAPITAL LETTER T WITH CARON + return rune(0x0164), true + case "Tcedil": + // LATIN CAPITAL LETTER T WITH CEDILLA + return rune(0x0162), true + case "Tcy": + // CYRILLIC CAPITAL LETTER TE + return rune(0x0422), true + case "Tfr": + // MATHEMATICAL FRAKTUR CAPITAL T + return rune(0x01d517), true + case "Tgr": + // GREEK CAPITAL LETTER TAU + return rune(0x03a4), true + case "Therefore": + // THEREFORE + return rune(0x2234), true + case "Theta": + // GREEK CAPITAL LETTER THETA + return rune(0x0398), true + case "Thetav": + // GREEK CAPITAL THETA SYMBOL + return rune(0x03f4), true + case "ThickSpace": + // space of width 5/18 em + return rune(0x205f), true + case "ThinSpace": + // THIN SPACE + return rune(0x2009), true + case "Tilde": + // TILDE OPERATOR + return rune(0x223c), true + case "TildeEqual": + // ASYMPTOTICALLY EQUAL TO + return rune(0x2243), true + case "TildeFullEqual": + // APPROXIMATELY EQUAL TO + return rune(0x2245), true + case "TildeTilde": + // ALMOST EQUAL TO + return rune(0x2248), true + case "Topf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL T + return rune(0x01d54b), true + case "TripleDot": + // COMBINING THREE DOTS ABOVE + return rune(0x20db), true + case "Tscr": + // MATHEMATICAL SCRIPT CAPITAL T + return rune(0x01d4af), true + case "Tstrok": + // LATIN CAPITAL LETTER T WITH STROKE + return rune(0x0166), true + } + + case 'U': + switch name { + case "Uacgr": + // GREEK CAPITAL LETTER UPSILON WITH TONOS + return rune(0x038e), true + case "Uacute": + // LATIN CAPITAL LETTER U WITH ACUTE + return rune(0xda), true + case "Uarrocir": + // UPWARDS TWO-HEADED ARROW FROM SMALL CIRCLE + return rune(0x2949), true + case "Uarr": + // UPWARDS TWO HEADED ARROW + return rune(0x219f), true + case "Ubrcy": + // CYRILLIC CAPITAL LETTER SHORT U + return rune(0x040e), true + case "Ubreve": + // LATIN CAPITAL LETTER U WITH BREVE + return rune(0x016c), true + case "Ucirc": + // LATIN CAPITAL LETTER U WITH CIRCUMFLEX + return rune(0xdb), true + case "Ucy": + // CYRILLIC CAPITAL LETTER U + return rune(0x0423), true + case "Udblac": + // LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + return rune(0x0170), true + case "Udigr": + // GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA + return rune(0x03ab), true + case "Ufr": + // MATHEMATICAL FRAKTUR CAPITAL U + return rune(0x01d518), true + case "Ugrave": + // LATIN CAPITAL LETTER U WITH GRAVE + return rune(0xd9), true + case "Ugr": + // GREEK CAPITAL LETTER UPSILON + return rune(0x03a5), true + case "Umacr": + // LATIN CAPITAL LETTER U WITH MACRON + return rune(0x016a), true + case "UnderBar": + // LOW LINE + return rune(0x5f), true + case "UnderBrace": + // BOTTOM CURLY BRACKET + return rune(0x23df), true + case "UnderBracket": + // BOTTOM SQUARE BRACKET + return rune(0x23b5), true + case "UnderParenthesis": + // BOTTOM PARENTHESIS + return rune(0x23dd), true + case "Union": + // N-ARY UNION + return rune(0x22c3), true + case "UnionPlus": + // MULTISET UNION + return rune(0x228e), true + case "Uogon": + // LATIN CAPITAL LETTER U WITH OGONEK + return rune(0x0172), true + case "Uopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL U + return rune(0x01d54c), true + case "UpArrow": + // UPWARDS ARROW + return rune(0x2191), true + case "UpArrowBar": + // UPWARDS ARROW TO BAR + return rune(0x2912), true + case "UpArrowDownArrow": + // UPWARDS ARROW LEFTWARDS OF DOWNWARDS ARROW + return rune(0x21c5), true + case "UpDownArrow": + // UP DOWN ARROW + return rune(0x2195), true + case "UpEquilibrium": + // UPWARDS HARPOON WITH BARB LEFT BESIDE DOWNWARDS HARPOON WITH BARB RIGHT + return rune(0x296e), true + case "UpTee": + // UP TACK + return rune(0x22a5), true + case "UpTeeArrow": + // UPWARDS ARROW FROM BAR + return rune(0x21a5), true + case "Uparrow": + // UPWARDS DOUBLE ARROW + return rune(0x21d1), true + case "Updownarrow": + // UP DOWN DOUBLE ARROW + return rune(0x21d5), true + case "UpperLeftArrow": + // NORTH WEST ARROW + return rune(0x2196), true + case "UpperRightArrow": + // NORTH EAST ARROW + return rune(0x2197), true + case "Upsilon": + // GREEK CAPITAL LETTER UPSILON + return rune(0x03a5), true + case "Upsi": + // GREEK UPSILON WITH HOOK SYMBOL + return rune(0x03d2), true + case "Uring": + // LATIN CAPITAL LETTER U WITH RING ABOVE + return rune(0x016e), true + case "Uscr": + // MATHEMATICAL SCRIPT CAPITAL U + return rune(0x01d4b0), true + case "Utilde": + // LATIN CAPITAL LETTER U WITH TILDE + return rune(0x0168), true + case "Uuml": + // LATIN CAPITAL LETTER U WITH DIAERESIS + return rune(0xdc), true + } + + case 'V': + switch name { + case "VDash": + // DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE + return rune(0x22ab), true + case "Vbar": + // DOUBLE UP TACK + return rune(0x2aeb), true + case "Vcy": + // CYRILLIC CAPITAL LETTER VE + return rune(0x0412), true + case "Vdashl": + // LONG DASH FROM LEFT MEMBER OF DOUBLE VERTICAL + return rune(0x2ae6), true + case "Vdash": + // FORCES + return rune(0x22a9), true + case "Vee": + // N-ARY LOGICAL OR + return rune(0x22c1), true + case "Verbar": + // DOUBLE VERTICAL LINE + return rune(0x2016), true + case "Vert": + // DOUBLE VERTICAL LINE + return rune(0x2016), true + case "VerticalBar": + // DIVIDES + return rune(0x2223), true + case "VerticalLine": + // VERTICAL LINE + return rune(0x7c), true + case "VerticalSeparator": + // LIGHT VERTICAL BAR + return rune(0x2758), true + case "VerticalTilde": + // WREATH PRODUCT + return rune(0x2240), true + case "VeryThinSpace": + // HAIR SPACE + return rune(0x200a), true + case "Vfr": + // MATHEMATICAL FRAKTUR CAPITAL V + return rune(0x01d519), true + case "Vopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL V + return rune(0x01d54d), true + case "Vscr": + // MATHEMATICAL SCRIPT CAPITAL V + return rune(0x01d4b1), true + case "Vvdash": + // TRIPLE VERTICAL BAR RIGHT TURNSTILE + return rune(0x22aa), true + } + + case 'W': + switch name { + case "Wcirc": + // LATIN CAPITAL LETTER W WITH CIRCUMFLEX + return rune(0x0174), true + case "Wedge": + // N-ARY LOGICAL AND + return rune(0x22c0), true + case "Wfr": + // MATHEMATICAL FRAKTUR CAPITAL W + return rune(0x01d51a), true + case "Wopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL W + return rune(0x01d54e), true + case "Wscr": + // MATHEMATICAL SCRIPT CAPITAL W + return rune(0x01d4b2), true + } + + case 'X': + switch name { + case "Xfr": + // MATHEMATICAL FRAKTUR CAPITAL X + return rune(0x01d51b), true + case "Xgr": + // GREEK CAPITAL LETTER XI + return rune(0x039e), true + case "Xi": + // GREEK CAPITAL LETTER XI + return rune(0x039e), true + case "Xopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL X + return rune(0x01d54f), true + case "Xscr": + // MATHEMATICAL SCRIPT CAPITAL X + return rune(0x01d4b3), true + } + + case 'Y': + switch name { + case "YAcy": + // CYRILLIC CAPITAL LETTER YA + return rune(0x042f), true + case "YIcy": + // CYRILLIC CAPITAL LETTER YI + return rune(0x0407), true + case "YUcy": + // CYRILLIC CAPITAL LETTER YU + return rune(0x042e), true + case "Yacute": + // LATIN CAPITAL LETTER Y WITH ACUTE + return rune(0xdd), true + case "Ycirc": + // LATIN CAPITAL LETTER Y WITH CIRCUMFLEX + return rune(0x0176), true + case "Ycy": + // CYRILLIC CAPITAL LETTER YERU + return rune(0x042b), true + case "Yfr": + // MATHEMATICAL FRAKTUR CAPITAL Y + return rune(0x01d51c), true + case "Yopf": + // MATHEMATICAL DOUBLE-STRUCK CAPITAL Y + return rune(0x01d550), true + case "Yscr": + // MATHEMATICAL SCRIPT CAPITAL Y + return rune(0x01d4b4), true + case "Yuml": + // LATIN CAPITAL LETTER Y WITH DIAERESIS + return rune(0x0178), true + } + + case 'Z': + switch name { + case "ZHcy": + // CYRILLIC CAPITAL LETTER ZHE + return rune(0x0416), true + case "Zacute": + // LATIN CAPITAL LETTER Z WITH ACUTE + return rune(0x0179), true + case "Zcaron": + // LATIN CAPITAL LETTER Z WITH CARON + return rune(0x017d), true + case "Zcy": + // CYRILLIC CAPITAL LETTER ZE + return rune(0x0417), true + case "Zdot": + // LATIN CAPITAL LETTER Z WITH DOT ABOVE + return rune(0x017b), true + case "ZeroWidthSpace": + // ZERO WIDTH SPACE + return rune(0x200b), true + case "Zeta": + // GREEK CAPITAL LETTER ZETA + return rune(0x0396), true + case "Zfr": + // BLACK-LETTER CAPITAL Z + return rune(0x2128), true + case "Zgr": + // GREEK CAPITAL LETTER ZETA + return rune(0x0396), true + case "Zopf": + // DOUBLE-STRUCK CAPITAL Z + return rune(0x2124), true + case "Zscr": + // MATHEMATICAL SCRIPT CAPITAL Z + return rune(0x01d4b5), true + } + + case 'a': + switch name { + case "aacgr": + // GREEK SMALL LETTER ALPHA WITH TONOS + return rune(0x03ac), true + case "aacute": + // LATIN SMALL LETTER A WITH ACUTE + return rune(0xe1), true + case "abreve": + // LATIN SMALL LETTER A WITH BREVE + return rune(0x0103), true + case "acE": + // INVERTED LAZY S with double underline + return rune(0x223e), true + case "acd": + // SINE WAVE + return rune(0x223f), true + case "acute": + // ACUTE ACCENT + return rune(0xb4), true + case "ac": + // INVERTED LAZY S + return rune(0x223e), true + case "acirc": + // LATIN SMALL LETTER A WITH CIRCUMFLEX + return rune(0xe2), true + case "actuary": + // COMBINING ANNUITY SYMBOL + return rune(0x20e7), true + case "acy": + // CYRILLIC SMALL LETTER A + return rune(0x0430), true + case "aelig": + // LATIN SMALL LETTER AE + return rune(0xe6), true + case "af": + // FUNCTION APPLICATION + return rune(0x2061), true + case "afr": + // MATHEMATICAL FRAKTUR SMALL A + return rune(0x01d51e), true + case "agr": + // GREEK SMALL LETTER ALPHA + return rune(0x03b1), true + case "agrave": + // LATIN SMALL LETTER A WITH GRAVE + return rune(0xe0), true + case "alefsym": + // ALEF SYMBOL + return rune(0x2135), true + case "aleph": + // ALEF SYMBOL + return rune(0x2135), true + case "alpha": + // GREEK SMALL LETTER ALPHA + return rune(0x03b1), true + case "amacr": + // LATIN SMALL LETTER A WITH MACRON + return rune(0x0101), true + case "amalg": + // AMALGAMATION OR COPRODUCT + return rune(0x2a3f), true + case "amp": + // AMPERSAND + return rune(0x26), true + case "andand": + // TWO INTERSECTING LOGICAL AND + return rune(0x2a55), true + case "andd": + // LOGICAL AND WITH HORIZONTAL DASH + return rune(0x2a5c), true + case "andslope": + // SLOPING LARGE AND + return rune(0x2a58), true + case "andv": + // LOGICAL AND WITH MIDDLE STEM + return rune(0x2a5a), true + case "and": + // LOGICAL AND + return rune(0x2227), true + case "angdnl": + // TURNED ANGLE + return rune(0x29a2), true + case "angdnr": + // ACUTE ANGLE + return rune(0x299f), true + case "ange": + // ANGLE WITH UNDERBAR + return rune(0x29a4), true + case "angles": + // ANGLE WITH S INSIDE + return rune(0x299e), true + case "angle": + // ANGLE + return rune(0x2220), true + case "angmsdaa": + // MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING UP AND RIGHT + return rune(0x29a8), true + case "angmsdab": + // MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING UP AND LEFT + return rune(0x29a9), true + case "angmsdac": + // MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING DOWN AND RIGHT + return rune(0x29aa), true + case "angmsdad": + // MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING DOWN AND LEFT + return rune(0x29ab), true + case "angmsdae": + // MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING RIGHT AND UP + return rune(0x29ac), true + case "angmsdaf": + // MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING LEFT AND UP + return rune(0x29ad), true + case "angmsdag": + // MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING RIGHT AND DOWN + return rune(0x29ae), true + case "angmsdah": + // MEASURED ANGLE WITH OPEN ARM ENDING IN ARROW POINTING LEFT AND DOWN + return rune(0x29af), true + case "angmsd": + // MEASURED ANGLE + return rune(0x2221), true + case "angrtvbd": + // MEASURED RIGHT ANGLE WITH DOT + return rune(0x299d), true + case "angrtvb": + // RIGHT ANGLE WITH ARC + return rune(0x22be), true + case "angsph": + // SPHERICAL ANGLE + return rune(0x2222), true + case "angst": + // LATIN CAPITAL LETTER A WITH RING ABOVE + return rune(0xc5), true + case "angupl": + // REVERSED ANGLE + return rune(0x29a3), true + case "angzarr": + // RIGHT ANGLE WITH DOWNWARDS ZIGZAG ARROW + return rune(0x237c), true + case "ang": + // ANGLE + return rune(0x2220), true + case "ang90": + // RIGHT ANGLE + return rune(0x221f), true + case "angrt": + // RIGHT ANGLE + return rune(0x221f), true + case "aogon": + // LATIN SMALL LETTER A WITH OGONEK + return rune(0x0105), true + case "aopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL A + return rune(0x01d552), true + case "apE": + // APPROXIMATELY EQUAL OR EQUAL TO + return rune(0x2a70), true + case "apacir": + // ALMOST EQUAL TO WITH CIRCUMFLEX ACCENT + return rune(0x2a6f), true + case "ape": + // ALMOST EQUAL OR EQUAL TO + return rune(0x224a), true + case "apid": + // TRIPLE TILDE + return rune(0x224b), true + case "approxeq": + // ALMOST EQUAL OR EQUAL TO + return rune(0x224a), true + case "approx": + // ALMOST EQUAL TO + return rune(0x2248), true + case "ap": + // ALMOST EQUAL TO + return rune(0x2248), true + case "apos": + // APOSTROPHE + return rune(0x27), true + case "aring": + // LATIN SMALL LETTER A WITH RING ABOVE + return rune(0xe5), true + case "arrllsr": + // LEFTWARDS ARROW ABOVE SHORT RIGHTWARDS ARROW + return rune(0x2943), true + case "arrlrsl": + // RIGHTWARDS ARROW ABOVE SHORT LEFTWARDS ARROW + return rune(0x2942), true + case "arrsrll": + // SHORT RIGHTWARDS ARROW ABOVE LEFTWARDS ARROW + return rune(0x2944), true + case "ascr": + // MATHEMATICAL SCRIPT SMALL A + return rune(0x01d4b6), true + case "astb": + // SQUARED ASTERISK + return rune(0x29c6), true + case "ast": + // ASTERISK + return rune(0x2a), true + case "asympeq": + // EQUIVALENT TO + return rune(0x224d), true + case "asymp": + // ALMOST EQUAL TO + return rune(0x2248), true + case "atilde": + // LATIN SMALL LETTER A WITH TILDE + return rune(0xe3), true + case "auml": + // LATIN SMALL LETTER A WITH DIAERESIS + return rune(0xe4), true + case "awconint": + // ANTICLOCKWISE CONTOUR INTEGRAL + return rune(0x2233), true + case "awint": + // ANTICLOCKWISE INTEGRATION + return rune(0x2a11), true + } + + case 'b': + switch name { + case "b.Delta": + // MATHEMATICAL BOLD CAPITAL DELTA + return rune(0x01d6ab), true + case "b.Gamma": + // MATHEMATICAL BOLD CAPITAL GAMMA + return rune(0x01d6aa), true + case "b.Gammad": + // MATHEMATICAL BOLD CAPITAL DIGAMMA + return rune(0x01d7ca), true + case "b.Lambda": + // MATHEMATICAL BOLD CAPITAL LAMDA + return rune(0x01d6b2), true + case "b.Omega": + // MATHEMATICAL BOLD CAPITAL OMEGA + return rune(0x01d6c0), true + case "b.Phi": + // MATHEMATICAL BOLD CAPITAL PHI + return rune(0x01d6bd), true + case "b.Pi": + // MATHEMATICAL BOLD CAPITAL PI + return rune(0x01d6b7), true + case "b.Psi": + // MATHEMATICAL BOLD CAPITAL PSI + return rune(0x01d6bf), true + case "b.Sigma": + // MATHEMATICAL BOLD CAPITAL SIGMA + return rune(0x01d6ba), true + case "b.Theta": + // MATHEMATICAL BOLD CAPITAL THETA + return rune(0x01d6af), true + case "b.Upsi": + // MATHEMATICAL BOLD CAPITAL UPSILON + return rune(0x01d6bc), true + case "b.Xi": + // MATHEMATICAL BOLD CAPITAL XI + return rune(0x01d6b5), true + case "b.alpha": + // MATHEMATICAL BOLD SMALL ALPHA + return rune(0x01d6c2), true + case "b.beta": + // MATHEMATICAL BOLD SMALL BETA + return rune(0x01d6c3), true + case "b.chi": + // MATHEMATICAL BOLD SMALL CHI + return rune(0x01d6d8), true + case "b.delta": + // MATHEMATICAL BOLD SMALL DELTA + return rune(0x01d6c5), true + case "b.epsi": + // MATHEMATICAL BOLD SMALL EPSILON + return rune(0x01d6c6), true + case "b.epsiv": + // MATHEMATICAL BOLD EPSILON SYMBOL + return rune(0x01d6dc), true + case "b.eta": + // MATHEMATICAL BOLD SMALL ETA + return rune(0x01d6c8), true + case "b.gammad": + // MATHEMATICAL BOLD SMALL DIGAMMA + return rune(0x01d7cb), true + case "b.gamma": + // MATHEMATICAL BOLD SMALL GAMMA + return rune(0x01d6c4), true + case "b.iota": + // MATHEMATICAL BOLD SMALL IOTA + return rune(0x01d6ca), true + case "b.kappa": + // MATHEMATICAL BOLD SMALL KAPPA + return rune(0x01d6cb), true + case "b.kappav": + // MATHEMATICAL BOLD KAPPA SYMBOL + return rune(0x01d6de), true + case "b.lambda": + // MATHEMATICAL BOLD SMALL LAMDA + return rune(0x01d6cc), true + case "b.mu": + // MATHEMATICAL BOLD SMALL MU + return rune(0x01d6cd), true + case "b.nu": + // MATHEMATICAL BOLD SMALL NU + return rune(0x01d6ce), true + case "b.omega": + // MATHEMATICAL BOLD SMALL OMEGA + return rune(0x01d6da), true + case "b.phi": + // MATHEMATICAL BOLD SMALL PHI + return rune(0x01d6d7), true + case "b.phiv": + // MATHEMATICAL BOLD PHI SYMBOL + return rune(0x01d6df), true + case "b.pi": + // MATHEMATICAL BOLD SMALL PI + return rune(0x01d6d1), true + case "b.piv": + // MATHEMATICAL BOLD PI SYMBOL + return rune(0x01d6e1), true + case "b.psi": + // MATHEMATICAL BOLD SMALL PSI + return rune(0x01d6d9), true + case "b.rho": + // MATHEMATICAL BOLD SMALL RHO + return rune(0x01d6d2), true + case "b.rhov": + // MATHEMATICAL BOLD RHO SYMBOL + return rune(0x01d6e0), true + case "b.sigmav": + // MATHEMATICAL BOLD SMALL FINAL SIGMA + return rune(0x01d6d3), true + case "b.sigma": + // MATHEMATICAL BOLD SMALL SIGMA + return rune(0x01d6d4), true + case "b.tau": + // MATHEMATICAL BOLD SMALL TAU + return rune(0x01d6d5), true + case "b.thetas": + // MATHEMATICAL BOLD SMALL THETA + return rune(0x01d6c9), true + case "b.thetav": + // MATHEMATICAL BOLD THETA SYMBOL + return rune(0x01d6dd), true + case "b.upsi": + // MATHEMATICAL BOLD SMALL UPSILON + return rune(0x01d6d6), true + case "b.xi": + // MATHEMATICAL BOLD SMALL XI + return rune(0x01d6cf), true + case "b.zeta": + // MATHEMATICAL BOLD SMALL ZETA + return rune(0x01d6c7), true + case "bNot": + // REVERSED DOUBLE STROKE NOT SIGN + return rune(0x2aed), true + case "backcong": + // ALL EQUAL TO + return rune(0x224c), true + case "backepsilon": + // GREEK REVERSED LUNATE EPSILON SYMBOL + return rune(0x03f6), true + case "backprime": + // REVERSED PRIME + return rune(0x2035), true + case "backsimeq": + // REVERSED TILDE EQUALS + return rune(0x22cd), true + case "backsim": + // REVERSED TILDE + return rune(0x223d), true + case "barV": + // DOUBLE DOWN TACK + return rune(0x2aea), true + case "barvee": + // NOR + return rune(0x22bd), true + case "barwed": + // PROJECTIVE + return rune(0x2305), true + case "barwedge": + // PROJECTIVE + return rune(0x2305), true + case "bbrk": + // BOTTOM SQUARE BRACKET + return rune(0x23b5), true + case "bbrktbrk": + // BOTTOM SQUARE BRACKET OVER TOP SQUARE BRACKET + return rune(0x23b6), true + case "bcong": + // ALL EQUAL TO + return rune(0x224c), true + case "bcy": + // CYRILLIC SMALL LETTER BE + return rune(0x0431), true + case "bdlhar": + // DOWNWARDS HARPOON WITH BARB LEFT FROM BAR + return rune(0x2961), true + case "bdquo": + // DOUBLE LOW-9 QUOTATION MARK + return rune(0x201e), true + case "bdrhar": + // DOWNWARDS HARPOON WITH BARB RIGHT FROM BAR + return rune(0x295d), true + case "because": + // BECAUSE + return rune(0x2235), true + case "becaus": + // BECAUSE + return rune(0x2235), true + case "bemptyv": + // REVERSED EMPTY SET + return rune(0x29b0), true + case "bepsi": + // GREEK REVERSED LUNATE EPSILON SYMBOL + return rune(0x03f6), true + case "bernou": + // SCRIPT CAPITAL B + return rune(0x212c), true + case "beta": + // GREEK SMALL LETTER BETA + return rune(0x03b2), true + case "beth": + // BET SYMBOL + return rune(0x2136), true + case "between": + // BETWEEN + return rune(0x226c), true + case "bfr": + // MATHEMATICAL FRAKTUR SMALL B + return rune(0x01d51f), true + case "bgr": + // GREEK SMALL LETTER BETA + return rune(0x03b2), true + case "bigcap": + // N-ARY INTERSECTION + return rune(0x22c2), true + case "bigcirc": + // LARGE CIRCLE + return rune(0x25ef), true + case "bigcup": + // N-ARY UNION + return rune(0x22c3), true + case "bigodot": + // N-ARY CIRCLED DOT OPERATOR + return rune(0x2a00), true + case "bigoplus": + // N-ARY CIRCLED PLUS OPERATOR + return rune(0x2a01), true + case "bigotimes": + // N-ARY CIRCLED TIMES OPERATOR + return rune(0x2a02), true + case "bigsqcup": + // N-ARY SQUARE UNION OPERATOR + return rune(0x2a06), true + case "bigstar": + // BLACK STAR + return rune(0x2605), true + case "bigtriangledown": + // WHITE DOWN-POINTING TRIANGLE + return rune(0x25bd), true + case "bigtriangleup": + // WHITE UP-POINTING TRIANGLE + return rune(0x25b3), true + case "biguplus": + // N-ARY UNION OPERATOR WITH PLUS + return rune(0x2a04), true + case "bigvee": + // N-ARY LOGICAL OR + return rune(0x22c1), true + case "bigwedge": + // N-ARY LOGICAL AND + return rune(0x22c0), true + case "bkarow": + // RIGHTWARDS DOUBLE DASH ARROW + return rune(0x290d), true + case "blacklozenge": + // BLACK LOZENGE + return rune(0x29eb), true + case "blacksquare": + // BLACK SMALL SQUARE + return rune(0x25aa), true + case "blacktriangledown": + // BLACK DOWN-POINTING SMALL TRIANGLE + return rune(0x25be), true + case "blacktriangleleft": + // BLACK LEFT-POINTING SMALL TRIANGLE + return rune(0x25c2), true + case "blacktriangleright": + // BLACK RIGHT-POINTING SMALL TRIANGLE + return rune(0x25b8), true + case "blacktriangle": + // BLACK UP-POINTING SMALL TRIANGLE + return rune(0x25b4), true + case "blank": + // BLANK SYMBOL + return rune(0x2422), true + case "bldhar": + // LEFTWARDS HARPOON WITH BARB DOWN FROM BAR + return rune(0x295e), true + case "blk12": + // MEDIUM SHADE + return rune(0x2592), true + case "blk14": + // LIGHT SHADE + return rune(0x2591), true + case "blk34": + // DARK SHADE + return rune(0x2593), true + case "block": + // FULL BLOCK + return rune(0x2588), true + case "bluhar": + // LEFTWARDS HARPOON WITH BARB UP FROM BAR + return rune(0x295a), true + case "bnequiv": + // IDENTICAL TO with reverse slash + return rune(0x2261), true + case "bne": + // EQUALS SIGN with reverse slash + return rune(0x3d), true + case "bnot": + // REVERSED NOT SIGN + return rune(0x2310), true + case "bopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL B + return rune(0x01d553), true + case "bot": + // UP TACK + return rune(0x22a5), true + case "bottom": + // UP TACK + return rune(0x22a5), true + case "bowtie": + // BOWTIE + return rune(0x22c8), true + case "boxDL": + // BOX DRAWINGS DOUBLE DOWN AND LEFT + return rune(0x2557), true + case "boxDR": + // BOX DRAWINGS DOUBLE DOWN AND RIGHT + return rune(0x2554), true + case "boxDl": + // BOX DRAWINGS DOWN DOUBLE AND LEFT SINGLE + return rune(0x2556), true + case "boxDr": + // BOX DRAWINGS DOWN DOUBLE AND RIGHT SINGLE + return rune(0x2553), true + case "boxHD": + // BOX DRAWINGS DOUBLE DOWN AND HORIZONTAL + return rune(0x2566), true + case "boxHU": + // BOX DRAWINGS DOUBLE UP AND HORIZONTAL + return rune(0x2569), true + case "boxHd": + // BOX DRAWINGS DOWN SINGLE AND HORIZONTAL DOUBLE + return rune(0x2564), true + case "boxHu": + // BOX DRAWINGS UP SINGLE AND HORIZONTAL DOUBLE + return rune(0x2567), true + case "boxH": + // BOX DRAWINGS DOUBLE HORIZONTAL + return rune(0x2550), true + case "boxUL": + // BOX DRAWINGS DOUBLE UP AND LEFT + return rune(0x255d), true + case "boxUR": + // BOX DRAWINGS DOUBLE UP AND RIGHT + return rune(0x255a), true + case "boxUl": + // BOX DRAWINGS UP DOUBLE AND LEFT SINGLE + return rune(0x255c), true + case "boxUr": + // BOX DRAWINGS UP DOUBLE AND RIGHT SINGLE + return rune(0x2559), true + case "boxVH": + // BOX DRAWINGS DOUBLE VERTICAL AND HORIZONTAL + return rune(0x256c), true + case "boxVL": + // BOX DRAWINGS DOUBLE VERTICAL AND LEFT + return rune(0x2563), true + case "boxVR": + // BOX DRAWINGS DOUBLE VERTICAL AND RIGHT + return rune(0x2560), true + case "boxVh": + // BOX DRAWINGS VERTICAL DOUBLE AND HORIZONTAL SINGLE + return rune(0x256b), true + case "boxVl": + // BOX DRAWINGS VERTICAL DOUBLE AND LEFT SINGLE + return rune(0x2562), true + case "boxVr": + // BOX DRAWINGS VERTICAL DOUBLE AND RIGHT SINGLE + return rune(0x255f), true + case "boxV": + // BOX DRAWINGS DOUBLE VERTICAL + return rune(0x2551), true + case "boxbox": + // TWO JOINED SQUARES + return rune(0x29c9), true + case "boxdL": + // BOX DRAWINGS DOWN SINGLE AND LEFT DOUBLE + return rune(0x2555), true + case "boxdR": + // BOX DRAWINGS DOWN SINGLE AND RIGHT DOUBLE + return rune(0x2552), true + case "boxdl": + // BOX DRAWINGS LIGHT DOWN AND LEFT + return rune(0x2510), true + case "boxdr": + // BOX DRAWINGS LIGHT DOWN AND RIGHT + return rune(0x250c), true + case "boxhU": + // BOX DRAWINGS UP DOUBLE AND HORIZONTAL SINGLE + return rune(0x2568), true + case "boxh": + // BOX DRAWINGS LIGHT HORIZONTAL + return rune(0x2500), true + case "boxhD": + // BOX DRAWINGS DOWN DOUBLE AND HORIZONTAL SINGLE + return rune(0x2565), true + case "boxhd": + // BOX DRAWINGS LIGHT DOWN AND HORIZONTAL + return rune(0x252c), true + case "boxhu": + // BOX DRAWINGS LIGHT UP AND HORIZONTAL + return rune(0x2534), true + case "boxminus": + // SQUARED MINUS + return rune(0x229f), true + case "boxplus": + // SQUARED PLUS + return rune(0x229e), true + case "boxtimes": + // SQUARED TIMES + return rune(0x22a0), true + case "boxuL": + // BOX DRAWINGS UP SINGLE AND LEFT DOUBLE + return rune(0x255b), true + case "boxuR": + // BOX DRAWINGS UP SINGLE AND RIGHT DOUBLE + return rune(0x2558), true + case "boxul": + // BOX DRAWINGS LIGHT UP AND LEFT + return rune(0x2518), true + case "boxur": + // BOX DRAWINGS LIGHT UP AND RIGHT + return rune(0x2514), true + case "boxvL": + // BOX DRAWINGS VERTICAL SINGLE AND LEFT DOUBLE + return rune(0x2561), true + case "boxvR": + // BOX DRAWINGS VERTICAL SINGLE AND RIGHT DOUBLE + return rune(0x255e), true + case "boxvl": + // BOX DRAWINGS LIGHT VERTICAL AND LEFT + return rune(0x2524), true + case "boxvr": + // BOX DRAWINGS LIGHT VERTICAL AND RIGHT + return rune(0x251c), true + case "boxv": + // BOX DRAWINGS LIGHT VERTICAL + return rune(0x2502), true + case "boxvH": + // BOX DRAWINGS VERTICAL SINGLE AND HORIZONTAL DOUBLE + return rune(0x256a), true + case "boxvh": + // BOX DRAWINGS LIGHT VERTICAL AND HORIZONTAL + return rune(0x253c), true + case "bprime": + // REVERSED PRIME + return rune(0x2035), true + case "brdhar": + // RIGHTWARDS HARPOON WITH BARB DOWN FROM BAR + return rune(0x295f), true + case "breve": + // BREVE + return rune(0x02d8), true + case "bruhar": + // RIGHTWARDS HARPOON WITH BARB UP FROM BAR + return rune(0x295b), true + case "brvbar": + // BROKEN BAR + return rune(0xa6), true + case "bscr": + // MATHEMATICAL SCRIPT SMALL B + return rune(0x01d4b7), true + case "bsemi": + // REVERSED SEMICOLON + return rune(0x204f), true + case "bsim": + // REVERSED TILDE + return rune(0x223d), true + case "bsime": + // REVERSED TILDE EQUALS + return rune(0x22cd), true + case "bsolb": + // SQUARED FALLING DIAGONAL SLASH + return rune(0x29c5), true + case "bsolhsub": + // REVERSE SOLIDUS PRECEDING SUBSET + return rune(0x27c8), true + case "bsol": + // REVERSE SOLIDUS + return rune(0x5c), true + case "btimes": + // SEMIDIRECT PRODUCT WITH BOTTOM CLOSED + return rune(0x2a32), true + case "bulhar": + // UPWARDS HARPOON WITH BARB LEFT FROM BAR + return rune(0x2960), true + case "bullet": + // BULLET + return rune(0x2022), true + case "bull": + // BULLET + return rune(0x2022), true + case "bump": + // GEOMETRICALLY EQUIVALENT TO + return rune(0x224e), true + case "bumpE": + // EQUALS SIGN WITH BUMPY ABOVE + return rune(0x2aae), true + case "bumpe": + // DIFFERENCE BETWEEN + return rune(0x224f), true + case "bumpeq": + // DIFFERENCE BETWEEN + return rune(0x224f), true + case "burhar": + // UPWARDS HARPOON WITH BARB RIGHT FROM BAR + return rune(0x295c), true + } + + case 'c': + switch name { + case "cacute": + // LATIN SMALL LETTER C WITH ACUTE + return rune(0x0107), true + case "cap": + // INTERSECTION + return rune(0x2229), true + case "capand": + // INTERSECTION WITH LOGICAL AND + return rune(0x2a44), true + case "capbrcup": + // INTERSECTION ABOVE BAR ABOVE UNION + return rune(0x2a49), true + case "capcap": + // INTERSECTION BESIDE AND JOINED WITH INTERSECTION + return rune(0x2a4b), true + case "capcup": + // INTERSECTION ABOVE UNION + return rune(0x2a47), true + case "capdot": + // INTERSECTION WITH DOT + return rune(0x2a40), true + case "capint": + // INTEGRAL WITH INTERSECTION + return rune(0x2a19), true + case "caps": + // INTERSECTION with serifs + return rune(0x2229), true + case "caret": + // CARET INSERTION POINT + return rune(0x2041), true + case "caron": + // CARON + return rune(0x02c7), true + case "ccaps": + // CLOSED INTERSECTION WITH SERIFS + return rune(0x2a4d), true + case "ccaron": + // LATIN SMALL LETTER C WITH CARON + return rune(0x010d), true + case "ccedil": + // LATIN SMALL LETTER C WITH CEDILLA + return rune(0xe7), true + case "ccirc": + // LATIN SMALL LETTER C WITH CIRCUMFLEX + return rune(0x0109), true + case "ccups": + // CLOSED UNION WITH SERIFS + return rune(0x2a4c), true + case "ccupssm": + // CLOSED UNION WITH SERIFS AND SMASH PRODUCT + return rune(0x2a50), true + case "cdot": + // LATIN SMALL LETTER C WITH DOT ABOVE + return rune(0x010b), true + case "cedil": + // CEDILLA + return rune(0xb8), true + case "cemptyv": + // EMPTY SET WITH SMALL CIRCLE ABOVE + return rune(0x29b2), true + case "centerdot": + // MIDDLE DOT + return rune(0xb7), true + case "cent": + // CENT SIGN + return rune(0xa2), true + case "cfr": + // MATHEMATICAL FRAKTUR SMALL C + return rune(0x01d520), true + case "chcy": + // CYRILLIC SMALL LETTER CHE + return rune(0x0447), true + case "check": + // CHECK MARK + return rune(0x2713), true + case "checkmark": + // CHECK MARK + return rune(0x2713), true + case "chi": + // GREEK SMALL LETTER CHI + return rune(0x03c7), true + case "circeq": + // RING EQUAL TO + return rune(0x2257), true + case "circlearrowleft": + // ANTICLOCKWISE OPEN CIRCLE ARROW + return rune(0x21ba), true + case "circlearrowright": + // CLOCKWISE OPEN CIRCLE ARROW + return rune(0x21bb), true + case "circledS": + // CIRCLED LATIN CAPITAL LETTER S + return rune(0x24c8), true + case "circledast": + // CIRCLED ASTERISK OPERATOR + return rune(0x229b), true + case "circledcirc": + // CIRCLED RING OPERATOR + return rune(0x229a), true + case "circleddash": + // CIRCLED DASH + return rune(0x229d), true + case "cire": + // RING EQUAL TO + return rune(0x2257), true + case "cir": + // WHITE CIRCLE + return rune(0x25cb), true + case "cirE": + // CIRCLE WITH TWO HORIZONTAL STROKES TO THE RIGHT + return rune(0x29c3), true + case "cirb": + // SQUARED SMALL CIRCLE + return rune(0x29c7), true + case "circ": + // MODIFIER LETTER CIRCUMFLEX ACCENT + return rune(0x02c6), true + case "circledR": + // REGISTERED SIGN + return rune(0xae), true + case "cirdarr": + // WHITE CIRCLE WITH DOWN ARROW + return rune(0x29ec), true + case "cirerr": + // ERROR-BARRED WHITE CIRCLE + return rune(0x29f2), true + case "cirfdarr": + // BLACK CIRCLE WITH DOWN ARROW + return rune(0x29ed), true + case "cirferr": + // ERROR-BARRED BLACK CIRCLE + return rune(0x29f3), true + case "cirfnint": + // CIRCULATION FUNCTION + return rune(0x2a10), true + case "cirmid": + // VERTICAL LINE WITH CIRCLE ABOVE + return rune(0x2aef), true + case "cirscir": + // CIRCLE WITH SMALL CIRCLE TO THE RIGHT + return rune(0x29c2), true + case "closur": + // CLOSE UP + return rune(0x2050), true + case "clubs": + // BLACK CLUB SUIT + return rune(0x2663), true + case "clubsuit": + // BLACK CLUB SUIT + return rune(0x2663), true + case "colone": + // COLON EQUALS + return rune(0x2254), true + case "coloneq": + // COLON EQUALS + return rune(0x2254), true + case "colon": + // COLON + return rune(0x3a), true + case "commat": + // COMMERCIAL AT + return rune(0x40), true + case "comma": + // COMMA + return rune(0x2c), true + case "comp": + // COMPLEMENT + return rune(0x2201), true + case "compfn": + // RING OPERATOR + return rune(0x2218), true + case "complement": + // COMPLEMENT + return rune(0x2201), true + case "complexes": + // DOUBLE-STRUCK CAPITAL C + return rune(0x2102), true + case "cong": + // APPROXIMATELY EQUAL TO + return rune(0x2245), true + case "congdot": + // CONGRUENT WITH DOT ABOVE + return rune(0x2a6d), true + case "conint": + // CONTOUR INTEGRAL + return rune(0x222e), true + case "copf": + // MATHEMATICAL DOUBLE-STRUCK SMALL C + return rune(0x01d554), true + case "coprod": + // N-ARY COPRODUCT + return rune(0x2210), true + case "copysr": + // SOUND RECORDING COPYRIGHT + return rune(0x2117), true + case "copy": + // COPYRIGHT SIGN + return rune(0xa9), true + case "crarr": + // DOWNWARDS ARROW WITH CORNER LEFTWARDS + return rune(0x21b5), true + case "cross": + // BALLOT X + return rune(0x2717), true + case "cscr": + // MATHEMATICAL SCRIPT SMALL C + return rune(0x01d4b8), true + case "csub": + // CLOSED SUBSET + return rune(0x2acf), true + case "csube": + // CLOSED SUBSET OR EQUAL TO + return rune(0x2ad1), true + case "csup": + // CLOSED SUPERSET + return rune(0x2ad0), true + case "csupe": + // CLOSED SUPERSET OR EQUAL TO + return rune(0x2ad2), true + case "ctdot": + // MIDLINE HORIZONTAL ELLIPSIS + return rune(0x22ef), true + case "cudarrl": + // RIGHT-SIDE ARC CLOCKWISE ARROW + return rune(0x2938), true + case "cudarrr": + // ARROW POINTING RIGHTWARDS THEN CURVING DOWNWARDS + return rune(0x2935), true + case "cuepr": + // EQUAL TO OR PRECEDES + return rune(0x22de), true + case "cuesc": + // EQUAL TO OR SUCCEEDS + return rune(0x22df), true + case "cularr": + // ANTICLOCKWISE TOP SEMICIRCLE ARROW + return rune(0x21b6), true + case "cularrp": + // TOP ARC ANTICLOCKWISE ARROW WITH PLUS + return rune(0x293d), true + case "cup": + // UNION + return rune(0x222a), true + case "cupbrcap": + // UNION ABOVE BAR ABOVE INTERSECTION + return rune(0x2a48), true + case "cupcap": + // UNION ABOVE INTERSECTION + return rune(0x2a46), true + case "cupcup": + // UNION BESIDE AND JOINED WITH UNION + return rune(0x2a4a), true + case "cupdot": + // MULTISET MULTIPLICATION + return rune(0x228d), true + case "cupint": + // INTEGRAL WITH UNION + return rune(0x2a1a), true + case "cupor": + // UNION WITH LOGICAL OR + return rune(0x2a45), true + case "cupre": + // PRECEDES OR EQUAL TO + return rune(0x227c), true + case "cups": + // UNION with serifs + return rune(0x222a), true + case "curarr": + // CLOCKWISE TOP SEMICIRCLE ARROW + return rune(0x21b7), true + case "curarrm": + // TOP ARC CLOCKWISE ARROW WITH MINUS + return rune(0x293c), true + case "curlyeqprec": + // EQUAL TO OR PRECEDES + return rune(0x22de), true + case "curlyeqsucc": + // EQUAL TO OR SUCCEEDS + return rune(0x22df), true + case "curlyvee": + // CURLY LOGICAL OR + return rune(0x22ce), true + case "curlywedge": + // CURLY LOGICAL AND + return rune(0x22cf), true + case "curren": + // CURRENCY SIGN + return rune(0xa4), true + case "curvearrowleft": + // ANTICLOCKWISE TOP SEMICIRCLE ARROW + return rune(0x21b6), true + case "curvearrowright": + // CLOCKWISE TOP SEMICIRCLE ARROW + return rune(0x21b7), true + case "cuvee": + // CURLY LOGICAL OR + return rune(0x22ce), true + case "cuwed": + // CURLY LOGICAL AND + return rune(0x22cf), true + case "cwconint": + // CLOCKWISE CONTOUR INTEGRAL + return rune(0x2232), true + case "cwint": + // CLOCKWISE INTEGRAL + return rune(0x2231), true + case "cylcty": + // CYLINDRICITY + return rune(0x232d), true + } + + case 'd': + switch name { + case "dAarr": + // DOWNWARDS TRIPLE ARROW + return rune(0x290b), true + case "dArr": + // DOWNWARDS DOUBLE ARROW + return rune(0x21d3), true + case "dHar": + // DOWNWARDS HARPOON WITH BARB LEFT BESIDE DOWNWARDS HARPOON WITH BARB RIGHT + return rune(0x2965), true + case "dagger": + // DAGGER + return rune(0x2020), true + case "dalembrt": + // SQUARE WITH CONTOURED OUTLINE + return rune(0x29e0), true + case "daleth": + // DALET SYMBOL + return rune(0x2138), true + case "darr2": + // DOWNWARDS PAIRED ARROWS + return rune(0x21ca), true + case "darr": + // DOWNWARDS ARROW + return rune(0x2193), true + case "darrb": + // DOWNWARDS ARROW TO BAR + return rune(0x2913), true + case "darrln": + // DOWNWARDS ARROW WITH HORIZONTAL STROKE + return rune(0x2908), true + case "dashv": + // LEFT TACK + return rune(0x22a3), true + case "dash": + // HYPHEN + return rune(0x2010), true + case "dashV": + // DOUBLE VERTICAL BAR LEFT TURNSTILE + return rune(0x2ae3), true + case "dbkarow": + // RIGHTWARDS TRIPLE DASH ARROW + return rune(0x290f), true + case "dblac": + // DOUBLE ACUTE ACCENT + return rune(0x02dd), true + case "dcaron": + // LATIN SMALL LETTER D WITH CARON + return rune(0x010f), true + case "dcy": + // CYRILLIC SMALL LETTER DE + return rune(0x0434), true + case "ddarr": + // DOWNWARDS PAIRED ARROWS + return rune(0x21ca), true + case "dd": + // DOUBLE-STRUCK ITALIC SMALL D + return rune(0x2146), true + case "ddagger": + // DOUBLE DAGGER + return rune(0x2021), true + case "ddotseq": + // EQUALS SIGN WITH TWO DOTS ABOVE AND TWO DOTS BELOW + return rune(0x2a77), true + case "deg": + // DEGREE SIGN + return rune(0xb0), true + case "delta": + // GREEK SMALL LETTER DELTA + return rune(0x03b4), true + case "demptyv": + // EMPTY SET WITH OVERBAR + return rune(0x29b1), true + case "dfisht": + // DOWN FISH TAIL + return rune(0x297f), true + case "dfr": + // MATHEMATICAL FRAKTUR SMALL D + return rune(0x01d521), true + case "dgr": + // GREEK SMALL LETTER DELTA + return rune(0x03b4), true + case "dharl": + // DOWNWARDS HARPOON WITH BARB LEFTWARDS + return rune(0x21c3), true + case "dharr": + // DOWNWARDS HARPOON WITH BARB RIGHTWARDS + return rune(0x21c2), true + case "diam": + // DIAMOND OPERATOR + return rune(0x22c4), true + case "diamdarr": + // BLACK DIAMOND WITH DOWN ARROW + return rune(0x29ea), true + case "diamerr": + // ERROR-BARRED WHITE DIAMOND + return rune(0x29f0), true + case "diamerrf": + // ERROR-BARRED BLACK DIAMOND + return rune(0x29f1), true + case "diamond": + // DIAMOND OPERATOR + return rune(0x22c4), true + case "diamondsuit": + // BLACK DIAMOND SUIT + return rune(0x2666), true + case "diams": + // BLACK DIAMOND SUIT + return rune(0x2666), true + case "die": + // DIAERESIS + return rune(0xa8), true + case "digamma": + // GREEK SMALL LETTER DIGAMMA + return rune(0x03dd), true + case "disin": + // ELEMENT OF WITH LONG HORIZONTAL STROKE + return rune(0x22f2), true + case "divideontimes": + // DIVISION TIMES + return rune(0x22c7), true + case "divonx": + // DIVISION TIMES + return rune(0x22c7), true + case "div": + // DIVISION SIGN + return rune(0xf7), true + case "divide": + // DIVISION SIGN + return rune(0xf7), true + case "djcy": + // CYRILLIC SMALL LETTER DJE + return rune(0x0452), true + case "dlarr": + // SOUTH WEST ARROW + return rune(0x2199), true + case "dlcorn": + // BOTTOM LEFT CORNER + return rune(0x231e), true + case "dlcrop": + // BOTTOM LEFT CROP + return rune(0x230d), true + case "dlharb": + // DOWNWARDS HARPOON WITH BARB LEFT TO BAR + return rune(0x2959), true + case "dollar": + // DOLLAR SIGN + return rune(0x24), true + case "dopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL D + return rune(0x01d555), true + case "doteq": + // APPROACHES THE LIMIT + return rune(0x2250), true + case "doteqdot": + // GEOMETRICALLY EQUAL TO + return rune(0x2251), true + case "dotminus": + // DOT MINUS + return rune(0x2238), true + case "dotplus": + // DOT PLUS + return rune(0x2214), true + case "dotsquare": + // SQUARED DOT OPERATOR + return rune(0x22a1), true + case "dot": + // DOT ABOVE + return rune(0x02d9), true + case "doublebarwedge": + // PERSPECTIVE + return rune(0x2306), true + case "downarrow": + // DOWNWARDS ARROW + return rune(0x2193), true + case "downdownarrows": + // DOWNWARDS PAIRED ARROWS + return rune(0x21ca), true + case "downharpoonleft": + // DOWNWARDS HARPOON WITH BARB LEFTWARDS + return rune(0x21c3), true + case "downharpoonright": + // DOWNWARDS HARPOON WITH BARB RIGHTWARDS + return rune(0x21c2), true + case "drarr": + // SOUTH EAST ARROW + return rune(0x2198), true + case "drbkarow": + // RIGHTWARDS TWO-HEADED TRIPLE DASH ARROW + return rune(0x2910), true + case "drcorn": + // BOTTOM RIGHT CORNER + return rune(0x231f), true + case "drcrop": + // BOTTOM RIGHT CROP + return rune(0x230c), true + case "drharb": + // DOWNWARDS HARPOON WITH BARB RIGHT TO BAR + return rune(0x2955), true + case "dscr": + // MATHEMATICAL SCRIPT SMALL D + return rune(0x01d4b9), true + case "dscy": + // CYRILLIC SMALL LETTER DZE + return rune(0x0455), true + case "dsol": + // SOLIDUS WITH OVERBAR + return rune(0x29f6), true + case "dstrok": + // LATIN SMALL LETTER D WITH STROKE + return rune(0x0111), true + case "dtdot": + // DOWN RIGHT DIAGONAL ELLIPSIS + return rune(0x22f1), true + case "dtrif": + // BLACK DOWN-POINTING SMALL TRIANGLE + return rune(0x25be), true + case "dtri": + // WHITE DOWN-POINTING SMALL TRIANGLE + return rune(0x25bf), true + case "dtrilf": + // DOWN-POINTING TRIANGLE WITH LEFT HALF BLACK + return rune(0x29e8), true + case "dtrirf": + // DOWN-POINTING TRIANGLE WITH RIGHT HALF BLACK + return rune(0x29e9), true + case "duarr": + // DOWNWARDS ARROW LEFTWARDS OF UPWARDS ARROW + return rune(0x21f5), true + case "duhar": + // DOWNWARDS HARPOON WITH BARB LEFT BESIDE UPWARDS HARPOON WITH BARB RIGHT + return rune(0x296f), true + case "dumap": + // DOUBLE-ENDED MULTIMAP + return rune(0x29df), true + case "dwangle": + // OBLIQUE ANGLE OPENING UP + return rune(0x29a6), true + case "dzcy": + // CYRILLIC SMALL LETTER DZHE + return rune(0x045f), true + case "dzigrarr": + // LONG RIGHTWARDS SQUIGGLE ARROW + return rune(0x27ff), true + } + + case 'e': + switch name { + case "eDDot": + // EQUALS SIGN WITH TWO DOTS ABOVE AND TWO DOTS BELOW + return rune(0x2a77), true + case "eDot": + // GEOMETRICALLY EQUAL TO + return rune(0x2251), true + case "eacgr": + // GREEK SMALL LETTER EPSILON WITH TONOS + return rune(0x03ad), true + case "eacute": + // LATIN SMALL LETTER E WITH ACUTE + return rune(0xe9), true + case "easter": + // EQUALS WITH ASTERISK + return rune(0x2a6e), true + case "ecaron": + // LATIN SMALL LETTER E WITH CARON + return rune(0x011b), true + case "ecir": + // RING IN EQUAL TO + return rune(0x2256), true + case "ecirc": + // LATIN SMALL LETTER E WITH CIRCUMFLEX + return rune(0xea), true + case "ecolon": + // EQUALS COLON + return rune(0x2255), true + case "ecy": + // CYRILLIC SMALL LETTER E + return rune(0x044d), true + case "edot": + // LATIN SMALL LETTER E WITH DOT ABOVE + return rune(0x0117), true + case "ee": + // DOUBLE-STRUCK ITALIC SMALL E + return rune(0x2147), true + case "eeacgr": + // GREEK SMALL LETTER ETA WITH TONOS + return rune(0x03ae), true + case "eegr": + // GREEK SMALL LETTER ETA + return rune(0x03b7), true + case "efDot": + // APPROXIMATELY EQUAL TO OR THE IMAGE OF + return rune(0x2252), true + case "efr": + // MATHEMATICAL FRAKTUR SMALL E + return rune(0x01d522), true + case "egr": + // GREEK SMALL LETTER EPSILON + return rune(0x03b5), true + case "egs": + // SLANTED EQUAL TO OR GREATER-THAN + return rune(0x2a96), true + case "egsdot": + // SLANTED EQUAL TO OR GREATER-THAN WITH DOT INSIDE + return rune(0x2a98), true + case "eg": + // DOUBLE-LINE EQUAL TO OR GREATER-THAN + return rune(0x2a9a), true + case "egrave": + // LATIN SMALL LETTER E WITH GRAVE + return rune(0xe8), true + case "elinters": + // ELECTRICAL INTERSECTION + return rune(0x23e7), true + case "ell": + // SCRIPT SMALL L + return rune(0x2113), true + case "els": + // SLANTED EQUAL TO OR LESS-THAN + return rune(0x2a95), true + case "elsdot": + // SLANTED EQUAL TO OR LESS-THAN WITH DOT INSIDE + return rune(0x2a97), true + case "el": + // DOUBLE-LINE EQUAL TO OR LESS-THAN + return rune(0x2a99), true + case "emacr": + // LATIN SMALL LETTER E WITH MACRON + return rune(0x0113), true + case "empty": + // EMPTY SET + return rune(0x2205), true + case "emptyset": + // EMPTY SET + return rune(0x2205), true + case "emptyv": + // EMPTY SET + return rune(0x2205), true + case "emsp13": + // THREE-PER-EM SPACE + return rune(0x2004), true + case "emsp14": + // FOUR-PER-EM SPACE + return rune(0x2005), true + case "emsp": + // EM SPACE + return rune(0x2003), true + case "eng": + // LATIN SMALL LETTER ENG + return rune(0x014b), true + case "ensp": + // EN SPACE + return rune(0x2002), true + case "eogon": + // LATIN SMALL LETTER E WITH OGONEK + return rune(0x0119), true + case "eopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL E + return rune(0x01d556), true + case "epar": + // EQUAL AND PARALLEL TO + return rune(0x22d5), true + case "eparsl": + // EQUALS SIGN AND SLANTED PARALLEL + return rune(0x29e3), true + case "eplus": + // EQUALS SIGN ABOVE PLUS SIGN + return rune(0x2a71), true + case "epsilon": + // GREEK SMALL LETTER EPSILON + return rune(0x03b5), true + case "epsis": + // GREEK LUNATE EPSILON SYMBOL + return rune(0x03f5), true + case "epsiv": + // GREEK LUNATE EPSILON SYMBOL + return rune(0x03f5), true + case "epsi": + // GREEK SMALL LETTER EPSILON + return rune(0x03b5), true + case "eqcirc": + // RING IN EQUAL TO + return rune(0x2256), true + case "eqcolon": + // EQUALS COLON + return rune(0x2255), true + case "eqeq": + // TWO CONSECUTIVE EQUALS SIGNS + return rune(0x2a75), true + case "eqsim": + // MINUS TILDE + return rune(0x2242), true + case "eqslantgtr": + // SLANTED EQUAL TO OR GREATER-THAN + return rune(0x2a96), true + case "eqslantless": + // SLANTED EQUAL TO OR LESS-THAN + return rune(0x2a95), true + case "equals": + // EQUALS SIGN + return rune(0x3d), true + case "equest": + // QUESTIONED EQUAL TO + return rune(0x225f), true + case "equiv": + // IDENTICAL TO + return rune(0x2261), true + case "equivDD": + // EQUIVALENT WITH FOUR DOTS ABOVE + return rune(0x2a78), true + case "eqvparsl": + // IDENTICAL TO AND SLANTED PARALLEL + return rune(0x29e5), true + case "erDot": + // IMAGE OF OR APPROXIMATELY EQUAL TO + return rune(0x2253), true + case "erarr": + // EQUALS SIGN ABOVE RIGHTWARDS ARROW + return rune(0x2971), true + case "escr": + // SCRIPT SMALL E + return rune(0x212f), true + case "esdot": + // APPROACHES THE LIMIT + return rune(0x2250), true + case "esim": + // MINUS TILDE + return rune(0x2242), true + case "eta": + // GREEK SMALL LETTER ETA + return rune(0x03b7), true + case "eth": + // LATIN SMALL LETTER ETH + return rune(0xf0), true + case "euml": + // LATIN SMALL LETTER E WITH DIAERESIS + return rune(0xeb), true + case "euro": + // EURO SIGN + return rune(0x20ac), true + case "excl": + // EXCLAMATION MARK + return rune(0x21), true + case "exist": + // THERE EXISTS + return rune(0x2203), true + case "expectation": + // SCRIPT CAPITAL E + return rune(0x2130), true + case "exponentiale": + // DOUBLE-STRUCK ITALIC SMALL E + return rune(0x2147), true + } + + case 'f': + switch name { + case "fallingdotseq": + // APPROXIMATELY EQUAL TO OR THE IMAGE OF + return rune(0x2252), true + case "fbowtie": + // BLACK BOWTIE + return rune(0x29d3), true + case "fcy": + // CYRILLIC SMALL LETTER EF + return rune(0x0444), true + case "fdiag": + // BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT + return rune(0x2572), true + case "fdiordi": + // FALLING DIAGONAL CROSSING RISING DIAGONAL + return rune(0x292c), true + case "fdonearr": + // FALLING DIAGONAL CROSSING NORTH EAST ARROW + return rune(0x292f), true + case "female": + // FEMALE SIGN + return rune(0x2640), true + case "ffilig": + // LATIN SMALL LIGATURE FFI + return rune(0xfb03), true + case "fflig": + // LATIN SMALL LIGATURE FF + return rune(0xfb00), true + case "ffllig": + // LATIN SMALL LIGATURE FFL + return rune(0xfb04), true + case "ffr": + // MATHEMATICAL FRAKTUR SMALL F + return rune(0x01d523), true + case "fhrglass": + // BLACK HOURGLASS + return rune(0x29d7), true + case "filig": + // LATIN SMALL LIGATURE FI + return rune(0xfb01), true + case "fjlig": + // fj ligature + return rune(0x66), true + case "flat": + // MUSIC FLAT SIGN + return rune(0x266d), true + case "fllig": + // LATIN SMALL LIGATURE FL + return rune(0xfb02), true + case "fltns": + // WHITE PARALLELOGRAM + return rune(0x25b1), true + case "fnof": + // LATIN SMALL LETTER F WITH HOOK + return rune(0x0192), true + case "fopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL F + return rune(0x01d557), true + case "forall": + // FOR ALL + return rune(0x2200), true + case "fork": + // PITCHFORK + return rune(0x22d4), true + case "forkv": + // ELEMENT OF OPENING DOWNWARDS + return rune(0x2ad9), true + case "fpartint": + // FINITE PART INTEGRAL + return rune(0x2a0d), true + case "frac12": + // VULGAR FRACTION ONE HALF + return rune(0xbd), true + case "frac13": + // VULGAR FRACTION ONE THIRD + return rune(0x2153), true + case "frac14": + // VULGAR FRACTION ONE QUARTER + return rune(0xbc), true + case "frac15": + // VULGAR FRACTION ONE FIFTH + return rune(0x2155), true + case "frac16": + // VULGAR FRACTION ONE SIXTH + return rune(0x2159), true + case "frac18": + // VULGAR FRACTION ONE EIGHTH + return rune(0x215b), true + case "frac23": + // VULGAR FRACTION TWO THIRDS + return rune(0x2154), true + case "frac25": + // VULGAR FRACTION TWO FIFTHS + return rune(0x2156), true + case "frac34": + // VULGAR FRACTION THREE QUARTERS + return rune(0xbe), true + case "frac35": + // VULGAR FRACTION THREE FIFTHS + return rune(0x2157), true + case "frac38": + // VULGAR FRACTION THREE EIGHTHS + return rune(0x215c), true + case "frac45": + // VULGAR FRACTION FOUR FIFTHS + return rune(0x2158), true + case "frac56": + // VULGAR FRACTION FIVE SIXTHS + return rune(0x215a), true + case "frac58": + // VULGAR FRACTION FIVE EIGHTHS + return rune(0x215d), true + case "frac78": + // VULGAR FRACTION SEVEN EIGHTHS + return rune(0x215e), true + case "frasl": + // FRACTION SLASH + return rune(0x2044), true + case "frown": + // FROWN + return rune(0x2322), true + case "fscr": + // MATHEMATICAL SCRIPT SMALL F + return rune(0x01d4bb), true + } + + case 'g': + switch name { + case "gE": + // GREATER-THAN OVER EQUAL TO + return rune(0x2267), true + case "gEl": + // GREATER-THAN ABOVE DOUBLE-LINE EQUAL ABOVE LESS-THAN + return rune(0x2a8c), true + case "gacute": + // LATIN SMALL LETTER G WITH ACUTE + return rune(0x01f5), true + case "gammad": + // GREEK SMALL LETTER DIGAMMA + return rune(0x03dd), true + case "gamma": + // GREEK SMALL LETTER GAMMA + return rune(0x03b3), true + case "gap": + // GREATER-THAN OR APPROXIMATE + return rune(0x2a86), true + case "gbreve": + // LATIN SMALL LETTER G WITH BREVE + return rune(0x011f), true + case "gcedil": + // LATIN SMALL LETTER G WITH CEDILLA + return rune(0x0123), true + case "gcirc": + // LATIN SMALL LETTER G WITH CIRCUMFLEX + return rune(0x011d), true + case "gcy": + // CYRILLIC SMALL LETTER GHE + return rune(0x0433), true + case "gdot": + // LATIN SMALL LETTER G WITH DOT ABOVE + return rune(0x0121), true + case "ge": + // GREATER-THAN OR EQUAL TO + return rune(0x2265), true + case "gel": + // GREATER-THAN EQUAL TO OR LESS-THAN + return rune(0x22db), true + case "geq": + // GREATER-THAN OR EQUAL TO + return rune(0x2265), true + case "geqq": + // GREATER-THAN OVER EQUAL TO + return rune(0x2267), true + case "geqslant": + // GREATER-THAN OR SLANTED EQUAL TO + return rune(0x2a7e), true + case "gesl": + // GREATER-THAN slanted EQUAL TO OR LESS-THAN + return rune(0x22db), true + case "ges": + // GREATER-THAN OR SLANTED EQUAL TO + return rune(0x2a7e), true + case "gescc": + // GREATER-THAN CLOSED BY CURVE ABOVE SLANTED EQUAL + return rune(0x2aa9), true + case "gesdot": + // GREATER-THAN OR SLANTED EQUAL TO WITH DOT INSIDE + return rune(0x2a80), true + case "gesdoto": + // GREATER-THAN OR SLANTED EQUAL TO WITH DOT ABOVE + return rune(0x2a82), true + case "gesdotol": + // GREATER-THAN OR SLANTED EQUAL TO WITH DOT ABOVE LEFT + return rune(0x2a84), true + case "gesles": + // GREATER-THAN ABOVE SLANTED EQUAL ABOVE LESS-THAN ABOVE SLANTED EQUAL + return rune(0x2a94), true + case "gfr": + // MATHEMATICAL FRAKTUR SMALL G + return rune(0x01d524), true + case "gg": + // MUCH GREATER-THAN + return rune(0x226b), true + case "ggg": + // VERY MUCH GREATER-THAN + return rune(0x22d9), true + case "ggr": + // GREEK SMALL LETTER GAMMA + return rune(0x03b3), true + case "gimel": + // GIMEL SYMBOL + return rune(0x2137), true + case "gjcy": + // CYRILLIC SMALL LETTER GJE + return rune(0x0453), true + case "gl": + // GREATER-THAN OR LESS-THAN + return rune(0x2277), true + case "glE": + // GREATER-THAN ABOVE LESS-THAN ABOVE DOUBLE-LINE EQUAL + return rune(0x2a92), true + case "gla": + // GREATER-THAN BESIDE LESS-THAN + return rune(0x2aa5), true + case "glj": + // GREATER-THAN OVERLAPPING LESS-THAN + return rune(0x2aa4), true + case "gnE": + // GREATER-THAN BUT NOT EQUAL TO + return rune(0x2269), true + case "gnap": + // GREATER-THAN AND NOT APPROXIMATE + return rune(0x2a8a), true + case "gnapprox": + // GREATER-THAN AND NOT APPROXIMATE + return rune(0x2a8a), true + case "gneqq": + // GREATER-THAN BUT NOT EQUAL TO + return rune(0x2269), true + case "gne": + // GREATER-THAN AND SINGLE-LINE NOT EQUAL TO + return rune(0x2a88), true + case "gneq": + // GREATER-THAN AND SINGLE-LINE NOT EQUAL TO + return rune(0x2a88), true + case "gnsim": + // GREATER-THAN BUT NOT EQUIVALENT TO + return rune(0x22e7), true + case "gopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL G + return rune(0x01d558), true + case "grave": + // GRAVE ACCENT + return rune(0x60), true + case "gscr": + // SCRIPT SMALL G + return rune(0x210a), true + case "gsdot": + // GREATER-THAN WITH DOT + return rune(0x22d7), true + case "gsim": + // GREATER-THAN OR EQUIVALENT TO + return rune(0x2273), true + case "gsime": + // GREATER-THAN ABOVE SIMILAR OR EQUAL + return rune(0x2a8e), true + case "gsiml": + // GREATER-THAN ABOVE SIMILAR ABOVE LESS-THAN + return rune(0x2a90), true + case "gtcc": + // GREATER-THAN CLOSED BY CURVE + return rune(0x2aa7), true + case "gtcir": + // GREATER-THAN WITH CIRCLE INSIDE + return rune(0x2a7a), true + case "gtdot": + // GREATER-THAN WITH DOT + return rune(0x22d7), true + case "gtlPar": + // DOUBLE LEFT ARC GREATER-THAN BRACKET + return rune(0x2995), true + case "gtquest": + // GREATER-THAN WITH QUESTION MARK ABOVE + return rune(0x2a7c), true + case "gtrapprox": + // GREATER-THAN OR APPROXIMATE + return rune(0x2a86), true + case "gtrarr": + // GREATER-THAN ABOVE RIGHTWARDS ARROW + return rune(0x2978), true + case "gtrdot": + // GREATER-THAN WITH DOT + return rune(0x22d7), true + case "gtreqless": + // GREATER-THAN EQUAL TO OR LESS-THAN + return rune(0x22db), true + case "gtreqqless": + // GREATER-THAN ABOVE DOUBLE-LINE EQUAL ABOVE LESS-THAN + return rune(0x2a8c), true + case "gtrless": + // GREATER-THAN OR LESS-THAN + return rune(0x2277), true + case "gtrpar": + // SPHERICAL ANGLE OPENING LEFT + return rune(0x29a0), true + case "gtrsim": + // GREATER-THAN OR EQUIVALENT TO + return rune(0x2273), true + case "gt": + // GREATER-THAN SIGN + return rune(0x3e), true + case "gvertneqq": + // GREATER-THAN BUT NOT EQUAL TO - with vertical stroke + return rune(0x2269), true + case "gvnE": + // GREATER-THAN BUT NOT EQUAL TO - with vertical stroke + return rune(0x2269), true + } + + case 'h': + switch name { + case "hArr": + // LEFT RIGHT DOUBLE ARROW + return rune(0x21d4), true + case "hairsp": + // HAIR SPACE + return rune(0x200a), true + case "half": + // VULGAR FRACTION ONE HALF + return rune(0xbd), true + case "hamilt": + // SCRIPT CAPITAL H + return rune(0x210b), true + case "hardcy": + // CYRILLIC SMALL LETTER HARD SIGN + return rune(0x044a), true + case "harrw": + // LEFT RIGHT WAVE ARROW + return rune(0x21ad), true + case "harr": + // LEFT RIGHT ARROW + return rune(0x2194), true + case "harrcir": + // LEFT RIGHT ARROW THROUGH SMALL CIRCLE + return rune(0x2948), true + case "hbar": + // PLANCK CONSTANT OVER TWO PI + return rune(0x210f), true + case "hcirc": + // LATIN SMALL LETTER H WITH CIRCUMFLEX + return rune(0x0125), true + case "hearts": + // BLACK HEART SUIT + return rune(0x2665), true + case "heartsuit": + // BLACK HEART SUIT + return rune(0x2665), true + case "hellip": + // HORIZONTAL ELLIPSIS + return rune(0x2026), true + case "hercon": + // HERMITIAN CONJUGATE MATRIX + return rune(0x22b9), true + case "hfr": + // MATHEMATICAL FRAKTUR SMALL H + return rune(0x01d525), true + case "hksearow": + // SOUTH EAST ARROW WITH HOOK + return rune(0x2925), true + case "hkswarow": + // SOUTH WEST ARROW WITH HOOK + return rune(0x2926), true + case "hoarr": + // LEFT RIGHT OPEN-HEADED ARROW + return rune(0x21ff), true + case "homtht": + // HOMOTHETIC + return rune(0x223b), true + case "hookleftarrow": + // LEFTWARDS ARROW WITH HOOK + return rune(0x21a9), true + case "hookrightarrow": + // RIGHTWARDS ARROW WITH HOOK + return rune(0x21aa), true + case "hopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL H + return rune(0x01d559), true + case "horbar": + // HORIZONTAL BAR + return rune(0x2015), true + case "hrglass": + // WHITE HOURGLASS + return rune(0x29d6), true + case "hscr": + // MATHEMATICAL SCRIPT SMALL H + return rune(0x01d4bd), true + case "hslash": + // PLANCK CONSTANT OVER TWO PI + return rune(0x210f), true + case "hstrok": + // LATIN SMALL LETTER H WITH STROKE + return rune(0x0127), true + case "htimes": + // VECTOR OR CROSS PRODUCT + return rune(0x2a2f), true + case "hybull": + // HYPHEN BULLET + return rune(0x2043), true + case "hyphen": + // HYPHEN + return rune(0x2010), true + } + + case 'i': + switch name { + case "iacgr": + // GREEK SMALL LETTER IOTA WITH TONOS + return rune(0x03af), true + case "iacute": + // LATIN SMALL LETTER I WITH ACUTE + return rune(0xed), true + case "ic": + // INVISIBLE SEPARATOR + return rune(0x2063), true + case "icirc": + // LATIN SMALL LETTER I WITH CIRCUMFLEX + return rune(0xee), true + case "icy": + // CYRILLIC SMALL LETTER I + return rune(0x0438), true + case "idiagr": + // GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + return rune(0x0390), true + case "idigr": + // GREEK SMALL LETTER IOTA WITH DIALYTIKA + return rune(0x03ca), true + case "iecy": + // CYRILLIC SMALL LETTER IE + return rune(0x0435), true + case "iexcl": + // INVERTED EXCLAMATION MARK + return rune(0xa1), true + case "iff": + // LEFT RIGHT DOUBLE ARROW + return rune(0x21d4), true + case "ifr": + // MATHEMATICAL FRAKTUR SMALL I + return rune(0x01d526), true + case "igr": + // GREEK SMALL LETTER IOTA + return rune(0x03b9), true + case "igrave": + // LATIN SMALL LETTER I WITH GRAVE + return rune(0xec), true + case "iiint": + // TRIPLE INTEGRAL + return rune(0x222d), true + case "ii": + // DOUBLE-STRUCK ITALIC SMALL I + return rune(0x2148), true + case "iiiint": + // QUADRUPLE INTEGRAL OPERATOR + return rune(0x2a0c), true + case "iinfin": + // INCOMPLETE INFINITY + return rune(0x29dc), true + case "iiota": + // TURNED GREEK SMALL LETTER IOTA + return rune(0x2129), true + case "ijlig": + // LATIN SMALL LIGATURE IJ + return rune(0x0133), true + case "imacr": + // LATIN SMALL LETTER I WITH MACRON + return rune(0x012b), true + case "image": + // BLACK-LETTER CAPITAL I + return rune(0x2111), true + case "imagline": + // SCRIPT CAPITAL I + return rune(0x2110), true + case "imagpart": + // BLACK-LETTER CAPITAL I + return rune(0x2111), true + case "imath": + // LATIN SMALL LETTER DOTLESS I + return rune(0x0131), true + case "imof": + // IMAGE OF + return rune(0x22b7), true + case "imped": + // LATIN CAPITAL LETTER Z WITH STROKE + return rune(0x01b5), true + case "in": + // ELEMENT OF + return rune(0x2208), true + case "incare": + // CARE OF + return rune(0x2105), true + case "infin": + // INFINITY + return rune(0x221e), true + case "infintie": + // TIE OVER INFINITY + return rune(0x29dd), true + case "inodot": + // LATIN SMALL LETTER DOTLESS I + return rune(0x0131), true + case "int": + // INTEGRAL + return rune(0x222b), true + case "intcal": + // INTERCALATE + return rune(0x22ba), true + case "integers": + // DOUBLE-STRUCK CAPITAL Z + return rune(0x2124), true + case "intercal": + // INTERCALATE + return rune(0x22ba), true + case "intlarhk": + // INTEGRAL WITH LEFTWARDS ARROW WITH HOOK + return rune(0x2a17), true + case "intprod": + // INTERIOR PRODUCT + return rune(0x2a3c), true + case "iocy": + // CYRILLIC SMALL LETTER IO + return rune(0x0451), true + case "iogon": + // LATIN SMALL LETTER I WITH OGONEK + return rune(0x012f), true + case "iopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL I + return rune(0x01d55a), true + case "iota": + // GREEK SMALL LETTER IOTA + return rune(0x03b9), true + case "iprod": + // INTERIOR PRODUCT + return rune(0x2a3c), true + case "iprodr": + // RIGHTHAND INTERIOR PRODUCT + return rune(0x2a3d), true + case "iquest": + // INVERTED QUESTION MARK + return rune(0xbf), true + case "iscr": + // MATHEMATICAL SCRIPT SMALL I + return rune(0x01d4be), true + case "isin": + // ELEMENT OF + return rune(0x2208), true + case "isinE": + // ELEMENT OF WITH TWO HORIZONTAL STROKES + return rune(0x22f9), true + case "isindot": + // ELEMENT OF WITH DOT ABOVE + return rune(0x22f5), true + case "isinsv": + // ELEMENT OF WITH VERTICAL BAR AT END OF HORIZONTAL STROKE + return rune(0x22f3), true + case "isins": + // SMALL ELEMENT OF WITH VERTICAL BAR AT END OF HORIZONTAL STROKE + return rune(0x22f4), true + case "isinv": + // ELEMENT OF + return rune(0x2208), true + case "isinvb": + // ELEMENT OF WITH UNDERBAR + return rune(0x22f8), true + case "it": + // INVISIBLE TIMES + return rune(0x2062), true + case "itilde": + // LATIN SMALL LETTER I WITH TILDE + return rune(0x0129), true + case "iukcy": + // CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + return rune(0x0456), true + case "iuml": + // LATIN SMALL LETTER I WITH DIAERESIS + return rune(0xef), true + } + + case 'j': + switch name { + case "jcirc": + // LATIN SMALL LETTER J WITH CIRCUMFLEX + return rune(0x0135), true + case "jcy": + // CYRILLIC SMALL LETTER SHORT I + return rune(0x0439), true + case "jfr": + // MATHEMATICAL FRAKTUR SMALL J + return rune(0x01d527), true + case "jmath": + // LATIN SMALL LETTER DOTLESS J + return rune(0x0237), true + case "jnodot": + // LATIN SMALL LETTER DOTLESS J + return rune(0x0237), true + case "jopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL J + return rune(0x01d55b), true + case "jscr": + // MATHEMATICAL SCRIPT SMALL J + return rune(0x01d4bf), true + case "jsercy": + // CYRILLIC SMALL LETTER JE + return rune(0x0458), true + case "jukcy": + // CYRILLIC SMALL LETTER UKRAINIAN IE + return rune(0x0454), true + } + + case 'k': + switch name { + case "kappav": + // GREEK KAPPA SYMBOL + return rune(0x03f0), true + case "kappa": + // GREEK SMALL LETTER KAPPA + return rune(0x03ba), true + case "kcedil": + // LATIN SMALL LETTER K WITH CEDILLA + return rune(0x0137), true + case "kcy": + // CYRILLIC SMALL LETTER KA + return rune(0x043a), true + case "kfr": + // MATHEMATICAL FRAKTUR SMALL K + return rune(0x01d528), true + case "kgr": + // GREEK SMALL LETTER KAPPA + return rune(0x03ba), true + case "kgreen": + // LATIN SMALL LETTER KRA + return rune(0x0138), true + case "khcy": + // CYRILLIC SMALL LETTER HA + return rune(0x0445), true + case "khgr": + // GREEK SMALL LETTER CHI + return rune(0x03c7), true + case "kjcy": + // CYRILLIC SMALL LETTER KJE + return rune(0x045c), true + case "kopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL K + return rune(0x01d55c), true + case "koppa": + // GREEK LETTER KOPPA + return rune(0x03de), true + case "kscr": + // MATHEMATICAL SCRIPT SMALL K + return rune(0x01d4c0), true + } + + case 'l': + switch name { + case "lAarr": + // LEFTWARDS TRIPLE ARROW + return rune(0x21da), true + case "lArr": + // LEFTWARDS DOUBLE ARROW + return rune(0x21d0), true + case "lAtail": + // LEFTWARDS DOUBLE ARROW-TAIL + return rune(0x291b), true + case "lBarr": + // LEFTWARDS TRIPLE DASH ARROW + return rune(0x290e), true + case "lE": + // LESS-THAN OVER EQUAL TO + return rune(0x2266), true + case "lEg": + // LESS-THAN ABOVE DOUBLE-LINE EQUAL ABOVE GREATER-THAN + return rune(0x2a8b), true + case "lHar": + // LEFTWARDS HARPOON WITH BARB UP ABOVE LEFTWARDS HARPOON WITH BARB DOWN + return rune(0x2962), true + case "lacute": + // LATIN SMALL LETTER L WITH ACUTE + return rune(0x013a), true + case "laemptyv": + // EMPTY SET WITH LEFT ARROW ABOVE + return rune(0x29b4), true + case "lagran": + // SCRIPT CAPITAL L + return rune(0x2112), true + case "lambda": + // GREEK SMALL LETTER LAMDA + return rune(0x03bb), true + case "lang": + // MATHEMATICAL LEFT ANGLE BRACKET + return rune(0x27e8), true + case "langd": + // LEFT ANGLE BRACKET WITH DOT + return rune(0x2991), true + case "langle": + // MATHEMATICAL LEFT ANGLE BRACKET + return rune(0x27e8), true + case "lap": + // LESS-THAN OR APPROXIMATE + return rune(0x2a85), true + case "laquo": + // LEFT-POINTING DOUBLE ANGLE QUOTATION MARK + return rune(0xab), true + case "larr2": + // LEFTWARDS PAIRED ARROWS + return rune(0x21c7), true + case "larrb": + // LEFTWARDS ARROW TO BAR + return rune(0x21e4), true + case "larrhk": + // LEFTWARDS ARROW WITH HOOK + return rune(0x21a9), true + case "larrlp": + // LEFTWARDS ARROW WITH LOOP + return rune(0x21ab), true + case "larrtl": + // LEFTWARDS ARROW WITH TAIL + return rune(0x21a2), true + case "larr": + // LEFTWARDS ARROW + return rune(0x2190), true + case "larrbfs": + // LEFTWARDS ARROW FROM BAR TO BLACK DIAMOND + return rune(0x291f), true + case "larrfs": + // LEFTWARDS ARROW TO BLACK DIAMOND + return rune(0x291d), true + case "larrpl": + // LEFT-SIDE ARC ANTICLOCKWISE ARROW + return rune(0x2939), true + case "larrsim": + // LEFTWARDS ARROW ABOVE TILDE OPERATOR + return rune(0x2973), true + case "latail": + // LEFTWARDS ARROW-TAIL + return rune(0x2919), true + case "lat": + // LARGER THAN + return rune(0x2aab), true + case "late": + // LARGER THAN OR EQUAL TO + return rune(0x2aad), true + case "lates": + // LARGER THAN OR slanted EQUAL + return rune(0x2aad), true + case "lbarr": + // LEFTWARDS DOUBLE DASH ARROW + return rune(0x290c), true + case "lbbrk": + // LIGHT LEFT TORTOISE SHELL BRACKET ORNAMENT + return rune(0x2772), true + case "lbrace": + // LEFT CURLY BRACKET + return rune(0x7b), true + case "lbrack": + // LEFT SQUARE BRACKET + return rune(0x5b), true + case "lbrke": + // LEFT SQUARE BRACKET WITH UNDERBAR + return rune(0x298b), true + case "lbrksld": + // LEFT SQUARE BRACKET WITH TICK IN BOTTOM CORNER + return rune(0x298f), true + case "lbrkslu": + // LEFT SQUARE BRACKET WITH TICK IN TOP CORNER + return rune(0x298d), true + case "lcaron": + // LATIN SMALL LETTER L WITH CARON + return rune(0x013e), true + case "lcedil": + // LATIN SMALL LETTER L WITH CEDILLA + return rune(0x013c), true + case "lceil": + // LEFT CEILING + return rune(0x2308), true + case "lcub": + // LEFT CURLY BRACKET + return rune(0x7b), true + case "lcy": + // CYRILLIC SMALL LETTER EL + return rune(0x043b), true + case "ldca": + // ARROW POINTING DOWNWARDS THEN CURVING LEFTWARDS + return rune(0x2936), true + case "ldharb": + // LEFTWARDS HARPOON WITH BARB DOWN TO BAR + return rune(0x2956), true + case "ldot": + // LESS-THAN WITH DOT + return rune(0x22d6), true + case "ldquor": + // DOUBLE LOW-9 QUOTATION MARK + return rune(0x201e), true + case "ldquo": + // LEFT DOUBLE QUOTATION MARK + return rune(0x201c), true + case "ldrdhar": + // LEFTWARDS HARPOON WITH BARB DOWN ABOVE RIGHTWARDS HARPOON WITH BARB DOWN + return rune(0x2967), true + case "ldrdshar": + // LEFT BARB DOWN RIGHT BARB DOWN HARPOON + return rune(0x2950), true + case "ldrushar": + // LEFT BARB DOWN RIGHT BARB UP HARPOON + return rune(0x294b), true + case "ldsh": + // DOWNWARDS ARROW WITH TIP LEFTWARDS + return rune(0x21b2), true + case "leftarrowtail": + // LEFTWARDS ARROW WITH TAIL + return rune(0x21a2), true + case "leftarrow": + // LEFTWARDS ARROW + return rune(0x2190), true + case "leftharpoondown": + // LEFTWARDS HARPOON WITH BARB DOWNWARDS + return rune(0x21bd), true + case "leftharpoonup": + // LEFTWARDS HARPOON WITH BARB UPWARDS + return rune(0x21bc), true + case "leftleftarrows": + // LEFTWARDS PAIRED ARROWS + return rune(0x21c7), true + case "leftrightarrows": + // LEFTWARDS ARROW OVER RIGHTWARDS ARROW + return rune(0x21c6), true + case "leftrightarrow": + // LEFT RIGHT ARROW + return rune(0x2194), true + case "leftrightharpoons": + // LEFTWARDS HARPOON OVER RIGHTWARDS HARPOON + return rune(0x21cb), true + case "leftrightsquigarrow": + // LEFT RIGHT WAVE ARROW + return rune(0x21ad), true + case "le": + // LESS-THAN OR EQUAL TO + return rune(0x2264), true + case "leftthreetimes": + // LEFT SEMIDIRECT PRODUCT + return rune(0x22cb), true + case "leg": + // LESS-THAN EQUAL TO OR GREATER-THAN + return rune(0x22da), true + case "leq": + // LESS-THAN OR EQUAL TO + return rune(0x2264), true + case "leqq": + // LESS-THAN OVER EQUAL TO + return rune(0x2266), true + case "leqslant": + // LESS-THAN OR SLANTED EQUAL TO + return rune(0x2a7d), true + case "lesg": + // LESS-THAN slanted EQUAL TO OR GREATER-THAN + return rune(0x22da), true + case "lessdot": + // LESS-THAN WITH DOT + return rune(0x22d6), true + case "lesseqgtr": + // LESS-THAN EQUAL TO OR GREATER-THAN + return rune(0x22da), true + case "lessgtr": + // LESS-THAN OR GREATER-THAN + return rune(0x2276), true + case "lesssim": + // LESS-THAN OR EQUIVALENT TO + return rune(0x2272), true + case "les": + // LESS-THAN OR SLANTED EQUAL TO + return rune(0x2a7d), true + case "lescc": + // LESS-THAN CLOSED BY CURVE ABOVE SLANTED EQUAL + return rune(0x2aa8), true + case "lesdot": + // LESS-THAN OR SLANTED EQUAL TO WITH DOT INSIDE + return rune(0x2a7f), true + case "lesdoto": + // LESS-THAN OR SLANTED EQUAL TO WITH DOT ABOVE + return rune(0x2a81), true + case "lesdotor": + // LESS-THAN OR SLANTED EQUAL TO WITH DOT ABOVE RIGHT + return rune(0x2a83), true + case "lesges": + // LESS-THAN ABOVE SLANTED EQUAL ABOVE GREATER-THAN ABOVE SLANTED EQUAL + return rune(0x2a93), true + case "lessapprox": + // LESS-THAN OR APPROXIMATE + return rune(0x2a85), true + case "lesseqqgtr": + // LESS-THAN ABOVE DOUBLE-LINE EQUAL ABOVE GREATER-THAN + return rune(0x2a8b), true + case "lfbowtie": + // BOWTIE WITH LEFT HALF BLACK + return rune(0x29d1), true + case "lfisht": + // LEFT FISH TAIL + return rune(0x297c), true + case "lfloor": + // LEFT FLOOR + return rune(0x230a), true + case "lfr": + // MATHEMATICAL FRAKTUR SMALL L + return rune(0x01d529), true + case "lftimes": + // TIMES WITH LEFT HALF BLACK + return rune(0x29d4), true + case "lg": + // LESS-THAN OR GREATER-THAN + return rune(0x2276), true + case "lgE": + // LESS-THAN ABOVE GREATER-THAN ABOVE DOUBLE-LINE EQUAL + return rune(0x2a91), true + case "lgr": + // GREEK SMALL LETTER LAMDA + return rune(0x03bb), true + case "lhard": + // LEFTWARDS HARPOON WITH BARB DOWNWARDS + return rune(0x21bd), true + case "lharu": + // LEFTWARDS HARPOON WITH BARB UPWARDS + return rune(0x21bc), true + case "lharul": + // LEFTWARDS HARPOON WITH BARB UP ABOVE LONG DASH + return rune(0x296a), true + case "lhblk": + // LOWER HALF BLOCK + return rune(0x2584), true + case "ljcy": + // CYRILLIC SMALL LETTER LJE + return rune(0x0459), true + case "llarr": + // LEFTWARDS PAIRED ARROWS + return rune(0x21c7), true + case "ll": + // MUCH LESS-THAN + return rune(0x226a), true + case "llcorner": + // BOTTOM LEFT CORNER + return rune(0x231e), true + case "llhard": + // LEFTWARDS HARPOON WITH BARB DOWN BELOW LONG DASH + return rune(0x296b), true + case "lltrif": + // BLACK LOWER LEFT TRIANGLE + return rune(0x25e3), true + case "lltri": + // LOWER LEFT TRIANGLE + return rune(0x25fa), true + case "lmidot": + // LATIN SMALL LETTER L WITH MIDDLE DOT + return rune(0x0140), true + case "lmoust": + // UPPER LEFT OR LOWER RIGHT CURLY BRACKET SECTION + return rune(0x23b0), true + case "lmoustache": + // UPPER LEFT OR LOWER RIGHT CURLY BRACKET SECTION + return rune(0x23b0), true + case "lnE": + // LESS-THAN BUT NOT EQUAL TO + return rune(0x2268), true + case "lnap": + // LESS-THAN AND NOT APPROXIMATE + return rune(0x2a89), true + case "lnapprox": + // LESS-THAN AND NOT APPROXIMATE + return rune(0x2a89), true + case "lneqq": + // LESS-THAN BUT NOT EQUAL TO + return rune(0x2268), true + case "lne": + // LESS-THAN AND SINGLE-LINE NOT EQUAL TO + return rune(0x2a87), true + case "lneq": + // LESS-THAN AND SINGLE-LINE NOT EQUAL TO + return rune(0x2a87), true + case "lnsim": + // LESS-THAN BUT NOT EQUIVALENT TO + return rune(0x22e6), true + case "loang": + // MATHEMATICAL LEFT WHITE TORTOISE SHELL BRACKET + return rune(0x27ec), true + case "loarr": + // LEFTWARDS OPEN-HEADED ARROW + return rune(0x21fd), true + case "lobrk": + // MATHEMATICAL LEFT WHITE SQUARE BRACKET + return rune(0x27e6), true + case "locub": + // LEFT WHITE CURLY BRACKET + return rune(0x2983), true + case "longleftarrow": + // LONG LEFTWARDS ARROW + return rune(0x27f5), true + case "longleftrightarrow": + // LONG LEFT RIGHT ARROW + return rune(0x27f7), true + case "longmapsto": + // LONG RIGHTWARDS ARROW FROM BAR + return rune(0x27fc), true + case "longrightarrow": + // LONG RIGHTWARDS ARROW + return rune(0x27f6), true + case "looparrowleft": + // LEFTWARDS ARROW WITH LOOP + return rune(0x21ab), true + case "looparrowright": + // RIGHTWARDS ARROW WITH LOOP + return rune(0x21ac), true + case "lopar": + // LEFT WHITE PARENTHESIS + return rune(0x2985), true + case "lopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL L + return rune(0x01d55d), true + case "loplus": + // PLUS SIGN IN LEFT HALF CIRCLE + return rune(0x2a2d), true + case "lotimes": + // MULTIPLICATION SIGN IN LEFT HALF CIRCLE + return rune(0x2a34), true + case "lowast": + // LOW ASTERISK + return rune(0x204e), true + case "lowbar": + // LOW LINE + return rune(0x5f), true + case "lowint": + // INTEGRAL WITH UNDERBAR + return rune(0x2a1c), true + case "loz": + // LOZENGE + return rune(0x25ca), true + case "lozenge": + // LOZENGE + return rune(0x25ca), true + case "lozf": + // BLACK LOZENGE + return rune(0x29eb), true + case "lpargt": + // SPHERICAL ANGLE OPENING LEFT + return rune(0x29a0), true + case "lparlt": + // LEFT ARC LESS-THAN BRACKET + return rune(0x2993), true + case "lpar": + // LEFT PARENTHESIS + return rune(0x28), true + case "lrarr2": + // LEFTWARDS ARROW OVER RIGHTWARDS ARROW + return rune(0x21c6), true + case "lrarr": + // LEFTWARDS ARROW OVER RIGHTWARDS ARROW + return rune(0x21c6), true + case "lrcorner": + // BOTTOM RIGHT CORNER + return rune(0x231f), true + case "lrhar": + // LEFTWARDS HARPOON OVER RIGHTWARDS HARPOON + return rune(0x21cb), true + case "lrhar2": + // LEFTWARDS HARPOON OVER RIGHTWARDS HARPOON + return rune(0x21cb), true + case "lrhard": + // RIGHTWARDS HARPOON WITH BARB DOWN BELOW LONG DASH + return rune(0x296d), true + case "lrm": + // LEFT-TO-RIGHT MARK + return rune(0x200e), true + case "lrtri": + // RIGHT TRIANGLE + return rune(0x22bf), true + case "lsaquo": + // SINGLE LEFT-POINTING ANGLE QUOTATION MARK + return rune(0x2039), true + case "lscr": + // MATHEMATICAL SCRIPT SMALL L + return rune(0x01d4c1), true + case "lsh": + // UPWARDS ARROW WITH TIP LEFTWARDS + return rune(0x21b0), true + case "lsim": + // LESS-THAN OR EQUIVALENT TO + return rune(0x2272), true + case "lsime": + // LESS-THAN ABOVE SIMILAR OR EQUAL + return rune(0x2a8d), true + case "lsimg": + // LESS-THAN ABOVE SIMILAR ABOVE GREATER-THAN + return rune(0x2a8f), true + case "lsqb": + // LEFT SQUARE BRACKET + return rune(0x5b), true + case "lsquor": + // SINGLE LOW-9 QUOTATION MARK + return rune(0x201a), true + case "lsquo": + // LEFT SINGLE QUOTATION MARK + return rune(0x2018), true + case "lstrok": + // LATIN SMALL LETTER L WITH STROKE + return rune(0x0142), true + case "ltcc": + // LESS-THAN CLOSED BY CURVE + return rune(0x2aa6), true + case "ltcir": + // LESS-THAN WITH CIRCLE INSIDE + return rune(0x2a79), true + case "ltdot": + // LESS-THAN WITH DOT + return rune(0x22d6), true + case "lthree": + // LEFT SEMIDIRECT PRODUCT + return rune(0x22cb), true + case "ltimes": + // LEFT NORMAL FACTOR SEMIDIRECT PRODUCT + return rune(0x22c9), true + case "ltlarr": + // LESS-THAN ABOVE LEFTWARDS ARROW + return rune(0x2976), true + case "ltquest": + // LESS-THAN WITH QUESTION MARK ABOVE + return rune(0x2a7b), true + case "ltrPar": + // DOUBLE RIGHT ARC LESS-THAN BRACKET + return rune(0x2996), true + case "ltrie": + // NORMAL SUBGROUP OF OR EQUAL TO + return rune(0x22b4), true + case "ltrif": + // BLACK LEFT-POINTING SMALL TRIANGLE + return rune(0x25c2), true + case "ltri": + // WHITE LEFT-POINTING SMALL TRIANGLE + return rune(0x25c3), true + case "ltrivb": + // LEFT TRIANGLE BESIDE VERTICAL BAR + return rune(0x29cf), true + case "lt": + // LESS-THAN SIGN + return rune(0x3c), true + case "luharb": + // LEFTWARDS HARPOON WITH BARB UP TO BAR + return rune(0x2952), true + case "lurdshar": + // LEFT BARB UP RIGHT BARB DOWN HARPOON + return rune(0x294a), true + case "luruhar": + // LEFTWARDS HARPOON WITH BARB UP ABOVE RIGHTWARDS HARPOON WITH BARB UP + return rune(0x2966), true + case "lurushar": + // LEFT BARB UP RIGHT BARB UP HARPOON + return rune(0x294e), true + case "lvertneqq": + // LESS-THAN BUT NOT EQUAL TO - with vertical stroke + return rune(0x2268), true + case "lvnE": + // LESS-THAN BUT NOT EQUAL TO - with vertical stroke + return rune(0x2268), true + } + + case 'm': + switch name { + case "mDDot": + // GEOMETRIC PROPORTION + return rune(0x223a), true + case "macr": + // MACRON + return rune(0xaf), true + case "male": + // MALE SIGN + return rune(0x2642), true + case "malt": + // MALTESE CROSS + return rune(0x2720), true + case "maltese": + // MALTESE CROSS + return rune(0x2720), true + case "mapstodown": + // DOWNWARDS ARROW FROM BAR + return rune(0x21a7), true + case "mapsto": + // RIGHTWARDS ARROW FROM BAR + return rune(0x21a6), true + case "map": + // RIGHTWARDS ARROW FROM BAR + return rune(0x21a6), true + case "mapstoleft": + // LEFTWARDS ARROW FROM BAR + return rune(0x21a4), true + case "mapstoup": + // UPWARDS ARROW FROM BAR + return rune(0x21a5), true + case "marker": + // BLACK VERTICAL RECTANGLE + return rune(0x25ae), true + case "mcomma": + // MINUS SIGN WITH COMMA ABOVE + return rune(0x2a29), true + case "mcy": + // CYRILLIC SMALL LETTER EM + return rune(0x043c), true + case "mdash": + // EM DASH + return rune(0x2014), true + case "measuredangle": + // MEASURED ANGLE + return rune(0x2221), true + case "mfr": + // MATHEMATICAL FRAKTUR SMALL M + return rune(0x01d52a), true + case "mgr": + // GREEK SMALL LETTER MU + return rune(0x03bc), true + case "mho": + // INVERTED OHM SIGN + return rune(0x2127), true + case "micro": + // MICRO SIGN + return rune(0xb5), true + case "mid": + // DIVIDES + return rune(0x2223), true + case "midast": + // ASTERISK + return rune(0x2a), true + case "midcir": + // VERTICAL LINE WITH CIRCLE BELOW + return rune(0x2af0), true + case "middot": + // MIDDLE DOT + return rune(0xb7), true + case "minus": + // MINUS SIGN + return rune(0x2212), true + case "minusb": + // SQUARED MINUS + return rune(0x229f), true + case "minusd": + // DOT MINUS + return rune(0x2238), true + case "minusdu": + // MINUS SIGN WITH DOT BELOW + return rune(0x2a2a), true + case "mlcp": + // TRANSVERSAL INTERSECTION + return rune(0x2adb), true + case "mldr": + // HORIZONTAL ELLIPSIS + return rune(0x2026), true + case "mnplus": + // MINUS-OR-PLUS SIGN + return rune(0x2213), true + case "models": + // MODELS + return rune(0x22a7), true + case "mopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL M + return rune(0x01d55e), true + case "mp": + // MINUS-OR-PLUS SIGN + return rune(0x2213), true + case "mscr": + // MATHEMATICAL SCRIPT SMALL M + return rune(0x01d4c2), true + case "mstpos": + // INVERTED LAZY S + return rune(0x223e), true + case "multimap": + // MULTIMAP + return rune(0x22b8), true + case "mumap": + // MULTIMAP + return rune(0x22b8), true + case "mu": + // GREEK SMALL LETTER MU + return rune(0x03bc), true + } + + case 'n': + switch name { + case "nGg": + // VERY MUCH GREATER-THAN with slash + return rune(0x22d9), true + case "nGtv": + // MUCH GREATER THAN with slash + return rune(0x226b), true + case "nGt": + // MUCH GREATER THAN with vertical line + return rune(0x226b), true + case "nLeftarrow": + // LEFTWARDS DOUBLE ARROW WITH STROKE + return rune(0x21cd), true + case "nLeftrightarrow": + // LEFT RIGHT DOUBLE ARROW WITH STROKE + return rune(0x21ce), true + case "nLl": + // VERY MUCH LESS-THAN with slash + return rune(0x22d8), true + case "nLtv": + // MUCH LESS THAN with slash + return rune(0x226a), true + case "nLt": + // MUCH LESS THAN with vertical line + return rune(0x226a), true + case "nRightarrow": + // RIGHTWARDS DOUBLE ARROW WITH STROKE + return rune(0x21cf), true + case "nVDash": + // NEGATED DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE + return rune(0x22af), true + case "nVdash": + // DOES NOT FORCE + return rune(0x22ae), true + case "nabla": + // NABLA + return rune(0x2207), true + case "nacute": + // LATIN SMALL LETTER N WITH ACUTE + return rune(0x0144), true + case "nang": + // ANGLE with vertical line + return rune(0x2220), true + case "nap": + // NOT ALMOST EQUAL TO + return rune(0x2249), true + case "napE": + // APPROXIMATELY EQUAL OR EQUAL TO with slash + return rune(0x2a70), true + case "napid": + // TRIPLE TILDE with slash + return rune(0x224b), true + case "napos": + // LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + return rune(0x0149), true + case "napprox": + // NOT ALMOST EQUAL TO + return rune(0x2249), true + case "naturals": + // DOUBLE-STRUCK CAPITAL N + return rune(0x2115), true + case "natur": + // MUSIC NATURAL SIGN + return rune(0x266e), true + case "natural": + // MUSIC NATURAL SIGN + return rune(0x266e), true + case "nbsp": + // NO-BREAK SPACE + return rune(0xa0), true + case "nbump": + // GEOMETRICALLY EQUIVALENT TO with slash + return rune(0x224e), true + case "nbumpe": + // DIFFERENCE BETWEEN with slash + return rune(0x224f), true + case "ncap": + // INTERSECTION WITH OVERBAR + return rune(0x2a43), true + case "ncaron": + // LATIN SMALL LETTER N WITH CARON + return rune(0x0148), true + case "ncedil": + // LATIN SMALL LETTER N WITH CEDILLA + return rune(0x0146), true + case "ncong": + // NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO + return rune(0x2247), true + case "ncongdot": + // CONGRUENT WITH DOT ABOVE with slash + return rune(0x2a6d), true + case "ncup": + // UNION WITH OVERBAR + return rune(0x2a42), true + case "ncy": + // CYRILLIC SMALL LETTER EN + return rune(0x043d), true + case "ndash": + // EN DASH + return rune(0x2013), true + case "neArr": + // NORTH EAST DOUBLE ARROW + return rune(0x21d7), true + case "nearrow": + // NORTH EAST ARROW + return rune(0x2197), true + case "nearr": + // NORTH EAST ARROW + return rune(0x2197), true + case "nedot": + // APPROACHES THE LIMIT with slash + return rune(0x2250), true + case "nesim": + // MINUS TILDE with slash + return rune(0x2242), true + case "nexist": + // THERE DOES NOT EXIST + return rune(0x2204), true + case "nexists": + // THERE DOES NOT EXIST + return rune(0x2204), true + case "ne": + // NOT EQUAL TO + return rune(0x2260), true + case "nearhk": + // NORTH EAST ARROW WITH HOOK + return rune(0x2924), true + case "neonwarr": + // NORTH EAST ARROW CROSSING NORTH WEST ARROW + return rune(0x2931), true + case "neosearr": + // NORTH EAST ARROW CROSSING SOUTH EAST ARROW + return rune(0x292e), true + case "nequiv": + // NOT IDENTICAL TO + return rune(0x2262), true + case "nesear": + // NORTH EAST ARROW AND SOUTH EAST ARROW + return rune(0x2928), true + case "neswsarr": + // NORTH EAST AND SOUTH WEST ARROW + return rune(0x2922), true + case "nfr": + // MATHEMATICAL FRAKTUR SMALL N + return rune(0x01d52b), true + case "ngE": + // GREATER-THAN OVER EQUAL TO with slash + return rune(0x2267), true + case "ngeqq": + // GREATER-THAN OVER EQUAL TO with slash + return rune(0x2267), true + case "nge": + // NEITHER GREATER-THAN NOR EQUAL TO + return rune(0x2271), true + case "ngeq": + // NEITHER GREATER-THAN NOR EQUAL TO + return rune(0x2271), true + case "ngeqslant": + // GREATER-THAN OR SLANTED EQUAL TO with slash + return rune(0x2a7e), true + case "nges": + // GREATER-THAN OR SLANTED EQUAL TO with slash + return rune(0x2a7e), true + case "ngr": + // GREEK SMALL LETTER NU + return rune(0x03bd), true + case "ngsim": + // NEITHER GREATER-THAN NOR EQUIVALENT TO + return rune(0x2275), true + case "ngt": + // NOT GREATER-THAN + return rune(0x226f), true + case "ngtr": + // NOT GREATER-THAN + return rune(0x226f), true + case "nhArr": + // LEFT RIGHT DOUBLE ARROW WITH STROKE + return rune(0x21ce), true + case "nharr": + // LEFT RIGHT ARROW WITH STROKE + return rune(0x21ae), true + case "nhpar": + // PARALLEL WITH HORIZONTAL STROKE + return rune(0x2af2), true + case "niv": + // CONTAINS AS MEMBER + return rune(0x220b), true + case "ni": + // CONTAINS AS MEMBER + return rune(0x220b), true + case "nisd": + // CONTAINS WITH LONG HORIZONTAL STROKE + return rune(0x22fa), true + case "nis": + // SMALL CONTAINS WITH VERTICAL BAR AT END OF HORIZONTAL STROKE + return rune(0x22fc), true + case "njcy": + // CYRILLIC SMALL LETTER NJE + return rune(0x045a), true + case "nlArr": + // LEFTWARDS DOUBLE ARROW WITH STROKE + return rune(0x21cd), true + case "nlE": + // LESS-THAN OVER EQUAL TO with slash + return rune(0x2266), true + case "nlarr": + // LEFTWARDS ARROW WITH STROKE + return rune(0x219a), true + case "nldr": + // TWO DOT LEADER + return rune(0x2025), true + case "nleftarrow": + // LEFTWARDS ARROW WITH STROKE + return rune(0x219a), true + case "nleftrightarrow": + // LEFT RIGHT ARROW WITH STROKE + return rune(0x21ae), true + case "nleqq": + // LESS-THAN OVER EQUAL TO with slash + return rune(0x2266), true + case "nless": + // NOT LESS-THAN + return rune(0x226e), true + case "nle": + // NEITHER LESS-THAN NOR EQUAL TO + return rune(0x2270), true + case "nleq": + // NEITHER LESS-THAN NOR EQUAL TO + return rune(0x2270), true + case "nleqslant": + // LESS-THAN OR SLANTED EQUAL TO with slash + return rune(0x2a7d), true + case "nles": + // LESS-THAN OR SLANTED EQUAL TO with slash + return rune(0x2a7d), true + case "nlsim": + // NEITHER LESS-THAN NOR EQUIVALENT TO + return rune(0x2274), true + case "nlt": + // NOT LESS-THAN + return rune(0x226e), true + case "nltri": + // NOT NORMAL SUBGROUP OF + return rune(0x22ea), true + case "nltrie": + // NOT NORMAL SUBGROUP OF OR EQUAL TO + return rune(0x22ec), true + case "nltrivb": + // LEFT TRIANGLE BESIDE VERTICAL BAR with slash + return rune(0x29cf), true + case "nmid": + // DOES NOT DIVIDE + return rune(0x2224), true + case "nopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL N + return rune(0x01d55f), true + case "notin": + // NOT AN ELEMENT OF + return rune(0x2209), true + case "notinE": + // ELEMENT OF WITH TWO HORIZONTAL STROKES with slash + return rune(0x22f9), true + case "notindot": + // ELEMENT OF WITH DOT ABOVE with slash + return rune(0x22f5), true + case "notinva": + // NOT AN ELEMENT OF + return rune(0x2209), true + case "notinvb": + // SMALL ELEMENT OF WITH OVERBAR + return rune(0x22f7), true + case "notinvc": + // ELEMENT OF WITH OVERBAR + return rune(0x22f6), true + case "notni": + // DOES NOT CONTAIN AS MEMBER + return rune(0x220c), true + case "notniva": + // DOES NOT CONTAIN AS MEMBER + return rune(0x220c), true + case "notnivb": + // SMALL CONTAINS WITH OVERBAR + return rune(0x22fe), true + case "notnivc": + // CONTAINS WITH OVERBAR + return rune(0x22fd), true + case "not": + // NOT SIGN + return rune(0xac), true + case "npart": + // PARTIAL DIFFERENTIAL with slash + return rune(0x2202), true + case "npar": + // NOT PARALLEL TO + return rune(0x2226), true + case "nparallel": + // NOT PARALLEL TO + return rune(0x2226), true + case "nparsl": + // DOUBLE SOLIDUS OPERATOR with reverse slash + return rune(0x2afd), true + case "npolint": + // LINE INTEGRATION NOT INCLUDING THE POLE + return rune(0x2a14), true + case "nprsim": + // PRECEDES OR EQUIVALENT TO with slash + return rune(0x227e), true + case "npr": + // DOES NOT PRECEDE + return rune(0x2280), true + case "nprcue": + // DOES NOT PRECEDE OR EQUAL + return rune(0x22e0), true + case "nprec": + // DOES NOT PRECEDE + return rune(0x2280), true + case "npre": + // PRECEDES ABOVE SINGLE-LINE EQUALS SIGN with slash + return rune(0x2aaf), true + case "npreceq": + // PRECEDES ABOVE SINGLE-LINE EQUALS SIGN with slash + return rune(0x2aaf), true + case "nrArr": + // RIGHTWARDS DOUBLE ARROW WITH STROKE + return rune(0x21cf), true + case "nrarrw": + // RIGHTWARDS WAVE ARROW with slash + return rune(0x219d), true + case "nrarr": + // RIGHTWARDS ARROW WITH STROKE + return rune(0x219b), true + case "nrarrc": + // WAVE ARROW POINTING DIRECTLY RIGHT with slash + return rune(0x2933), true + case "nrightarrow": + // RIGHTWARDS ARROW WITH STROKE + return rune(0x219b), true + case "nrtri": + // DOES NOT CONTAIN AS NORMAL SUBGROUP + return rune(0x22eb), true + case "nrtrie": + // DOES NOT CONTAIN AS NORMAL SUBGROUP OR EQUAL + return rune(0x22ed), true + case "nsGt": + // DOUBLE NESTED GREATER-THAN with slash + return rune(0x2aa2), true + case "nsLt": + // DOUBLE NESTED LESS-THAN with slash + return rune(0x2aa1), true + case "nscsim": + // SUCCEEDS OR EQUIVALENT TO with slash + return rune(0x227f), true + case "nsc": + // DOES NOT SUCCEED + return rune(0x2281), true + case "nsccue": + // DOES NOT SUCCEED OR EQUAL + return rune(0x22e1), true + case "nsce": + // SUCCEEDS ABOVE SINGLE-LINE EQUALS SIGN with slash + return rune(0x2ab0), true + case "nscr": + // MATHEMATICAL SCRIPT SMALL N + return rune(0x01d4c3), true + case "nshortmid": + // DOES NOT DIVIDE + return rune(0x2224), true + case "nshortparallel": + // NOT PARALLEL TO + return rune(0x2226), true + case "nsim": + // NOT TILDE + return rune(0x2241), true + case "nsime": + // NOT ASYMPTOTICALLY EQUAL TO + return rune(0x2244), true + case "nsimeq": + // NOT ASYMPTOTICALLY EQUAL TO + return rune(0x2244), true + case "nsmid": + // DOES NOT DIVIDE + return rune(0x2224), true + case "nspar": + // NOT PARALLEL TO + return rune(0x2226), true + case "nsqsub": + // SQUARE IMAGE OF with slash + return rune(0x228f), true + case "nsqsube": + // NOT SQUARE IMAGE OF OR EQUAL TO + return rune(0x22e2), true + case "nsqsup": + // SQUARE ORIGINAL OF with slash + return rune(0x2290), true + case "nsqsupe": + // NOT SQUARE ORIGINAL OF OR EQUAL TO + return rune(0x22e3), true + case "nsubset": + // SUBSET OF with vertical line + return rune(0x2282), true + case "nsub": + // NOT A SUBSET OF + return rune(0x2284), true + case "nsubE": + // SUBSET OF ABOVE EQUALS SIGN with slash + return rune(0x2ac5), true + case "nsube": + // NEITHER A SUBSET OF NOR EQUAL TO + return rune(0x2288), true + case "nsubseteq": + // NEITHER A SUBSET OF NOR EQUAL TO + return rune(0x2288), true + case "nsubseteqq": + // SUBSET OF ABOVE EQUALS SIGN with slash + return rune(0x2ac5), true + case "nsucc": + // DOES NOT SUCCEED + return rune(0x2281), true + case "nsucceq": + // SUCCEEDS ABOVE SINGLE-LINE EQUALS SIGN with slash + return rune(0x2ab0), true + case "nsupset": + // SUPERSET OF with vertical line + return rune(0x2283), true + case "nsup": + // NOT A SUPERSET OF + return rune(0x2285), true + case "nsupE": + // SUPERSET OF ABOVE EQUALS SIGN with slash + return rune(0x2ac6), true + case "nsupe": + // NEITHER A SUPERSET OF NOR EQUAL TO + return rune(0x2289), true + case "nsupseteq": + // NEITHER A SUPERSET OF NOR EQUAL TO + return rune(0x2289), true + case "nsupseteqq": + // SUPERSET OF ABOVE EQUALS SIGN with slash + return rune(0x2ac6), true + case "ntgl": + // NEITHER GREATER-THAN NOR LESS-THAN + return rune(0x2279), true + case "ntilde": + // LATIN SMALL LETTER N WITH TILDE + return rune(0xf1), true + case "ntlg": + // NEITHER LESS-THAN NOR GREATER-THAN + return rune(0x2278), true + case "ntriangleleft": + // NOT NORMAL SUBGROUP OF + return rune(0x22ea), true + case "ntrianglelefteq": + // NOT NORMAL SUBGROUP OF OR EQUAL TO + return rune(0x22ec), true + case "ntriangleright": + // DOES NOT CONTAIN AS NORMAL SUBGROUP + return rune(0x22eb), true + case "ntrianglerighteq": + // DOES NOT CONTAIN AS NORMAL SUBGROUP OR EQUAL + return rune(0x22ed), true + case "numero": + // NUMERO SIGN + return rune(0x2116), true + case "numsp": + // FIGURE SPACE + return rune(0x2007), true + case "nu": + // GREEK SMALL LETTER NU + return rune(0x03bd), true + case "num": + // NUMBER SIGN + return rune(0x23), true + case "nvDash": + // NOT TRUE + return rune(0x22ad), true + case "nvHarr": + // LEFT RIGHT DOUBLE ARROW WITH VERTICAL STROKE + return rune(0x2904), true + case "nvap": + // EQUIVALENT TO with vertical line + return rune(0x224d), true + case "nvbrtri": + // VERTICAL BAR BESIDE RIGHT TRIANGLE with slash + return rune(0x29d0), true + case "nvdash": + // DOES NOT PROVE + return rune(0x22ac), true + case "nvge": + // GREATER-THAN OR EQUAL TO with vertical line + return rune(0x2265), true + case "nvgt": + // GREATER-THAN SIGN with vertical line + return rune(0x3e), true + case "nvinfin": + // INFINITY NEGATED WITH VERTICAL BAR + return rune(0x29de), true + case "nvlArr": + // LEFTWARDS DOUBLE ARROW WITH VERTICAL STROKE + return rune(0x2902), true + case "nvle": + // LESS-THAN OR EQUAL TO with vertical line + return rune(0x2264), true + case "nvltrie": + // NORMAL SUBGROUP OF OR EQUAL TO with vertical line + return rune(0x22b4), true + case "nvlt": + // LESS-THAN SIGN with vertical line + return rune(0x3c), true + case "nvrArr": + // RIGHTWARDS DOUBLE ARROW WITH VERTICAL STROKE + return rune(0x2903), true + case "nvrtrie": + // CONTAINS AS NORMAL SUBGROUP OR EQUAL TO with vertical line + return rune(0x22b5), true + case "nvsim": + // TILDE OPERATOR with vertical line + return rune(0x223c), true + case "nwArr": + // NORTH WEST DOUBLE ARROW + return rune(0x21d6), true + case "nwarhk": + // NORTH WEST ARROW WITH HOOK + return rune(0x2923), true + case "nwarrow": + // NORTH WEST ARROW + return rune(0x2196), true + case "nwarr": + // NORTH WEST ARROW + return rune(0x2196), true + case "nwnear": + // NORTH WEST ARROW AND NORTH EAST ARROW + return rune(0x2927), true + case "nwonearr": + // NORTH WEST ARROW CROSSING NORTH EAST ARROW + return rune(0x2932), true + case "nwsesarr": + // NORTH WEST AND SOUTH EAST ARROW + return rune(0x2921), true + } + + case 'o': + switch name { + case "oS": + // CIRCLED LATIN CAPITAL LETTER S + return rune(0x24c8), true + case "oacgr": + // GREEK SMALL LETTER OMICRON WITH TONOS + return rune(0x03cc), true + case "oacute": + // LATIN SMALL LETTER O WITH ACUTE + return rune(0xf3), true + case "oast": + // CIRCLED ASTERISK OPERATOR + return rune(0x229b), true + case "obsol": + // CIRCLED REVERSE SOLIDUS + return rune(0x29b8), true + case "ocir": + // CIRCLED RING OPERATOR + return rune(0x229a), true + case "ocirc": + // LATIN SMALL LETTER O WITH CIRCUMFLEX + return rune(0xf4), true + case "ocy": + // CYRILLIC SMALL LETTER O + return rune(0x043e), true + case "odash": + // CIRCLED DASH + return rune(0x229d), true + case "odblac": + // LATIN SMALL LETTER O WITH DOUBLE ACUTE + return rune(0x0151), true + case "odiv": + // CIRCLED DIVISION SIGN + return rune(0x2a38), true + case "odot": + // CIRCLED DOT OPERATOR + return rune(0x2299), true + case "odsold": + // CIRCLED ANTICLOCKWISE-ROTATED DIVISION SIGN + return rune(0x29bc), true + case "oelig": + // LATIN SMALL LIGATURE OE + return rune(0x0153), true + case "ofcir": + // CIRCLED BULLET + return rune(0x29bf), true + case "ofr": + // MATHEMATICAL FRAKTUR SMALL O + return rune(0x01d52c), true + case "ogon": + // OGONEK + return rune(0x02db), true + case "ogr": + // GREEK SMALL LETTER OMICRON + return rune(0x03bf), true + case "ograve": + // LATIN SMALL LETTER O WITH GRAVE + return rune(0xf2), true + case "ogt": + // CIRCLED GREATER-THAN + return rune(0x29c1), true + case "ohacgr": + // GREEK SMALL LETTER OMEGA WITH TONOS + return rune(0x03ce), true + case "ohbar": + // CIRCLE WITH HORIZONTAL BAR + return rune(0x29b5), true + case "ohgr": + // GREEK SMALL LETTER OMEGA + return rune(0x03c9), true + case "ohm": + // GREEK CAPITAL LETTER OMEGA + return rune(0x03a9), true + case "oint": + // CONTOUR INTEGRAL + return rune(0x222e), true + case "olarr": + // ANTICLOCKWISE OPEN CIRCLE ARROW + return rune(0x21ba), true + case "olcir": + // CIRCLED WHITE BULLET + return rune(0x29be), true + case "olcross": + // CIRCLE WITH SUPERIMPOSED X + return rune(0x29bb), true + case "oline": + // OVERLINE + return rune(0x203e), true + case "olt": + // CIRCLED LESS-THAN + return rune(0x29c0), true + case "omacr": + // LATIN SMALL LETTER O WITH MACRON + return rune(0x014d), true + case "omega": + // GREEK SMALL LETTER OMEGA + return rune(0x03c9), true + case "omicron": + // GREEK SMALL LETTER OMICRON + return rune(0x03bf), true + case "omid": + // CIRCLED VERTICAL BAR + return rune(0x29b6), true + case "ominus": + // CIRCLED MINUS + return rune(0x2296), true + case "oopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL O + return rune(0x01d560), true + case "opar": + // CIRCLED PARALLEL + return rune(0x29b7), true + case "operp": + // CIRCLED PERPENDICULAR + return rune(0x29b9), true + case "opfgamma": + // DOUBLE-STRUCK SMALL GAMMA + return rune(0x213d), true + case "opfpi": + // DOUBLE-STRUCK CAPITAL PI + return rune(0x213f), true + case "opfsum": + // DOUBLE-STRUCK N-ARY SUMMATION + return rune(0x2140), true + case "oplus": + // CIRCLED PLUS + return rune(0x2295), true + case "orarr": + // CLOCKWISE OPEN CIRCLE ARROW + return rune(0x21bb), true + case "or": + // LOGICAL OR + return rune(0x2228), true + case "orderof": + // SCRIPT SMALL O + return rune(0x2134), true + case "order": + // SCRIPT SMALL O + return rune(0x2134), true + case "ord": + // LOGICAL OR WITH HORIZONTAL DASH + return rune(0x2a5d), true + case "ordf": + // FEMININE ORDINAL INDICATOR + return rune(0xaa), true + case "ordm": + // MASCULINE ORDINAL INDICATOR + return rune(0xba), true + case "origof": + // ORIGINAL OF + return rune(0x22b6), true + case "oror": + // TWO INTERSECTING LOGICAL OR + return rune(0x2a56), true + case "orslope": + // SLOPING LARGE OR + return rune(0x2a57), true + case "orv": + // LOGICAL OR WITH MIDDLE STEM + return rune(0x2a5b), true + case "oscr": + // SCRIPT SMALL O + return rune(0x2134), true + case "oslash": + // LATIN SMALL LETTER O WITH STROKE + return rune(0xf8), true + case "osol": + // CIRCLED DIVISION SLASH + return rune(0x2298), true + case "otilde": + // LATIN SMALL LETTER O WITH TILDE + return rune(0xf5), true + case "otimes": + // CIRCLED TIMES + return rune(0x2297), true + case "otimesas": + // CIRCLED MULTIPLICATION SIGN WITH CIRCUMFLEX ACCENT + return rune(0x2a36), true + case "ouml": + // LATIN SMALL LETTER O WITH DIAERESIS + return rune(0xf6), true + case "ovbar": + // APL FUNCTIONAL SYMBOL CIRCLE STILE + return rune(0x233d), true + case "ovrbrk": + // TOP SQUARE BRACKET + return rune(0x23b4), true + case "ovrcub": + // TOP CURLY BRACKET + return rune(0x23de), true + case "ovrpar": + // TOP PARENTHESIS + return rune(0x23dc), true + case "oxuarr": + // UP ARROW THROUGH CIRCLE + return rune(0x29bd), true + } + + case 'p': + switch name { + case "part": + // PARTIAL DIFFERENTIAL + return rune(0x2202), true + case "par": + // PARALLEL TO + return rune(0x2225), true + case "parallel": + // PARALLEL TO + return rune(0x2225), true + case "para": + // PILCROW SIGN + return rune(0xb6), true + case "parsim": + // PARALLEL WITH TILDE OPERATOR + return rune(0x2af3), true + case "parsl": + // DOUBLE SOLIDUS OPERATOR + return rune(0x2afd), true + case "pcy": + // CYRILLIC SMALL LETTER PE + return rune(0x043f), true + case "percnt": + // PERCENT SIGN + return rune(0x25), true + case "period": + // FULL STOP + return rune(0x2e), true + case "permil": + // PER MILLE SIGN + return rune(0x2030), true + case "perp": + // UP TACK + return rune(0x22a5), true + case "pertenk": + // PER TEN THOUSAND SIGN + return rune(0x2031), true + case "pfr": + // MATHEMATICAL FRAKTUR SMALL P + return rune(0x01d52d), true + case "pgr": + // GREEK SMALL LETTER PI + return rune(0x03c0), true + case "phgr": + // GREEK SMALL LETTER PHI + return rune(0x03c6), true + case "phis": + // GREEK PHI SYMBOL + return rune(0x03d5), true + case "phiv": + // GREEK PHI SYMBOL + return rune(0x03d5), true + case "phi": + // GREEK SMALL LETTER PHI + return rune(0x03c6), true + case "phmmat": + // SCRIPT CAPITAL M + return rune(0x2133), true + case "phone": + // BLACK TELEPHONE + return rune(0x260e), true + case "pitchfork": + // PITCHFORK + return rune(0x22d4), true + case "piv": + // GREEK PI SYMBOL + return rune(0x03d6), true + case "pi": + // GREEK SMALL LETTER PI + return rune(0x03c0), true + case "planck": + // PLANCK CONSTANT OVER TWO PI + return rune(0x210f), true + case "planckh": + // PLANCK CONSTANT + return rune(0x210e), true + case "plankv": + // PLANCK CONSTANT OVER TWO PI + return rune(0x210f), true + case "plusacir": + // PLUS SIGN WITH CIRCUMFLEX ACCENT ABOVE + return rune(0x2a23), true + case "plusb": + // SQUARED PLUS + return rune(0x229e), true + case "pluscir": + // PLUS SIGN WITH SMALL CIRCLE ABOVE + return rune(0x2a22), true + case "plusdo": + // DOT PLUS + return rune(0x2214), true + case "plusdu": + // PLUS SIGN WITH DOT BELOW + return rune(0x2a25), true + case "pluse": + // PLUS SIGN ABOVE EQUALS SIGN + return rune(0x2a72), true + case "plusmn": + // PLUS-MINUS SIGN + return rune(0xb1), true + case "plussim": + // PLUS SIGN WITH TILDE BELOW + return rune(0x2a26), true + case "plustrif": + // PLUS SIGN WITH BLACK TRIANGLE + return rune(0x2a28), true + case "plustwo": + // PLUS SIGN WITH SUBSCRIPT TWO + return rune(0x2a27), true + case "plus": + // PLUS SIGN + return rune(0x2b), true + case "pm": + // PLUS-MINUS SIGN + return rune(0xb1), true + case "pointint": + // INTEGRAL AROUND A POINT OPERATOR + return rune(0x2a15), true + case "popf": + // MATHEMATICAL DOUBLE-STRUCK SMALL P + return rune(0x01d561), true + case "pound": + // POUND SIGN + return rune(0xa3), true + case "prod": + // N-ARY PRODUCT + return rune(0x220f), true + case "prop": + // PROPORTIONAL TO + return rune(0x221d), true + case "propto": + // PROPORTIONAL TO + return rune(0x221d), true + case "pr": + // PRECEDES + return rune(0x227a), true + case "prE": + // PRECEDES ABOVE EQUALS SIGN + return rune(0x2ab3), true + case "prap": + // PRECEDES ABOVE ALMOST EQUAL TO + return rune(0x2ab7), true + case "prcue": + // PRECEDES OR EQUAL TO + return rune(0x227c), true + case "prec": + // PRECEDES + return rune(0x227a), true + case "preccurlyeq": + // PRECEDES OR EQUAL TO + return rune(0x227c), true + case "precnsim": + // PRECEDES BUT NOT EQUIVALENT TO + return rune(0x22e8), true + case "precsim": + // PRECEDES OR EQUIVALENT TO + return rune(0x227e), true + case "pre": + // PRECEDES ABOVE SINGLE-LINE EQUALS SIGN + return rune(0x2aaf), true + case "precapprox": + // PRECEDES ABOVE ALMOST EQUAL TO + return rune(0x2ab7), true + case "preceq": + // PRECEDES ABOVE SINGLE-LINE EQUALS SIGN + return rune(0x2aaf), true + case "precnapprox": + // PRECEDES ABOVE NOT ALMOST EQUAL TO + return rune(0x2ab9), true + case "precneqq": + // PRECEDES ABOVE NOT EQUAL TO + return rune(0x2ab5), true + case "primes": + // DOUBLE-STRUCK CAPITAL P + return rune(0x2119), true + case "prime": + // PRIME + return rune(0x2032), true + case "prnE": + // PRECEDES ABOVE NOT EQUAL TO + return rune(0x2ab5), true + case "prnap": + // PRECEDES ABOVE NOT ALMOST EQUAL TO + return rune(0x2ab9), true + case "prnsim": + // PRECEDES BUT NOT EQUIVALENT TO + return rune(0x22e8), true + case "profalar": + // ALL AROUND-PROFILE + return rune(0x232e), true + case "profline": + // ARC + return rune(0x2312), true + case "profsurf": + // SEGMENT + return rune(0x2313), true + case "prsim": + // PRECEDES OR EQUIVALENT TO + return rune(0x227e), true + case "prurel": + // PRECEDES UNDER RELATION + return rune(0x22b0), true + case "pscr": + // MATHEMATICAL SCRIPT SMALL P + return rune(0x01d4c5), true + case "psgr": + // GREEK SMALL LETTER PSI + return rune(0x03c8), true + case "psi": + // GREEK SMALL LETTER PSI + return rune(0x03c8), true + case "puncsp": + // PUNCTUATION SPACE + return rune(0x2008), true + } + + case 'q': + switch name { + case "qfr": + // MATHEMATICAL FRAKTUR SMALL Q + return rune(0x01d52e), true + case "qint": + // QUADRUPLE INTEGRAL OPERATOR + return rune(0x2a0c), true + case "qopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL Q + return rune(0x01d562), true + case "qprime": + // QUADRUPLE PRIME + return rune(0x2057), true + case "qscr": + // MATHEMATICAL SCRIPT SMALL Q + return rune(0x01d4c6), true + case "quaternions": + // DOUBLE-STRUCK CAPITAL H + return rune(0x210d), true + case "quatint": + // QUATERNION INTEGRAL OPERATOR + return rune(0x2a16), true + case "questeq": + // QUESTIONED EQUAL TO + return rune(0x225f), true + case "quest": + // QUESTION MARK + return rune(0x3f), true + case "quot": + // QUOTATION MARK + return rune(0x22), true + } + + case 'r': + switch name { + case "rAarr": + // RIGHTWARDS TRIPLE ARROW + return rune(0x21db), true + case "rArr": + // RIGHTWARDS DOUBLE ARROW + return rune(0x21d2), true + case "rAtail": + // RIGHTWARDS DOUBLE ARROW-TAIL + return rune(0x291c), true + case "rBarr": + // RIGHTWARDS TRIPLE DASH ARROW + return rune(0x290f), true + case "rHar": + // RIGHTWARDS HARPOON WITH BARB UP ABOVE RIGHTWARDS HARPOON WITH BARB DOWN + return rune(0x2964), true + case "race": + // REVERSED TILDE with underline + return rune(0x223d), true + case "racute": + // LATIN SMALL LETTER R WITH ACUTE + return rune(0x0155), true + case "radic": + // SQUARE ROOT + return rune(0x221a), true + case "raemptyv": + // EMPTY SET WITH RIGHT ARROW ABOVE + return rune(0x29b3), true + case "rang": + // MATHEMATICAL RIGHT ANGLE BRACKET + return rune(0x27e9), true + case "rangd": + // RIGHT ANGLE BRACKET WITH DOT + return rune(0x2992), true + case "range": + // REVERSED ANGLE WITH UNDERBAR + return rune(0x29a5), true + case "rangle": + // MATHEMATICAL RIGHT ANGLE BRACKET + return rune(0x27e9), true + case "raquo": + // RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK + return rune(0xbb), true + case "rarr2": + // RIGHTWARDS PAIRED ARROWS + return rune(0x21c9), true + case "rarr3": + // THREE RIGHTWARDS ARROWS + return rune(0x21f6), true + case "rarrb": + // RIGHTWARDS ARROW TO BAR + return rune(0x21e5), true + case "rarrhk": + // RIGHTWARDS ARROW WITH HOOK + return rune(0x21aa), true + case "rarrlp": + // RIGHTWARDS ARROW WITH LOOP + return rune(0x21ac), true + case "rarrtl": + // RIGHTWARDS ARROW WITH TAIL + return rune(0x21a3), true + case "rarrw": + // RIGHTWARDS WAVE ARROW + return rune(0x219d), true + case "rarr": + // RIGHTWARDS ARROW + return rune(0x2192), true + case "rarrap": + // RIGHTWARDS ARROW ABOVE ALMOST EQUAL TO + return rune(0x2975), true + case "rarrbfs": + // RIGHTWARDS ARROW FROM BAR TO BLACK DIAMOND + return rune(0x2920), true + case "rarrc": + // WAVE ARROW POINTING DIRECTLY RIGHT + return rune(0x2933), true + case "rarrfs": + // RIGHTWARDS ARROW TO BLACK DIAMOND + return rune(0x291e), true + case "rarrpl": + // RIGHTWARDS ARROW WITH PLUS BELOW + return rune(0x2945), true + case "rarrsim": + // RIGHTWARDS ARROW ABOVE TILDE OPERATOR + return rune(0x2974), true + case "rarrx": + // RIGHTWARDS ARROW THROUGH X + return rune(0x2947), true + case "ratail": + // RIGHTWARDS ARROW-TAIL + return rune(0x291a), true + case "ratio": + // RATIO + return rune(0x2236), true + case "rationals": + // DOUBLE-STRUCK CAPITAL Q + return rune(0x211a), true + case "rbarr": + // RIGHTWARDS DOUBLE DASH ARROW + return rune(0x290d), true + case "rbbrk": + // LIGHT RIGHT TORTOISE SHELL BRACKET ORNAMENT + return rune(0x2773), true + case "rbrace": + // RIGHT CURLY BRACKET + return rune(0x7d), true + case "rbrack": + // RIGHT SQUARE BRACKET + return rune(0x5d), true + case "rbrke": + // RIGHT SQUARE BRACKET WITH UNDERBAR + return rune(0x298c), true + case "rbrksld": + // RIGHT SQUARE BRACKET WITH TICK IN BOTTOM CORNER + return rune(0x298e), true + case "rbrkslu": + // RIGHT SQUARE BRACKET WITH TICK IN TOP CORNER + return rune(0x2990), true + case "rcaron": + // LATIN SMALL LETTER R WITH CARON + return rune(0x0159), true + case "rcedil": + // LATIN SMALL LETTER R WITH CEDILLA + return rune(0x0157), true + case "rceil": + // RIGHT CEILING + return rune(0x2309), true + case "rcub": + // RIGHT CURLY BRACKET + return rune(0x7d), true + case "rcy": + // CYRILLIC SMALL LETTER ER + return rune(0x0440), true + case "rdca": + // ARROW POINTING DOWNWARDS THEN CURVING RIGHTWARDS + return rune(0x2937), true + case "rdharb": + // RIGHTWARDS HARPOON WITH BARB DOWN TO BAR + return rune(0x2957), true + case "rdiag": + // BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT + return rune(0x2571), true + case "rdiofdi": + // RISING DIAGONAL CROSSING FALLING DIAGONAL + return rune(0x292b), true + case "rdldhar": + // RIGHTWARDS HARPOON WITH BARB DOWN ABOVE LEFTWARDS HARPOON WITH BARB DOWN + return rune(0x2969), true + case "rdosearr": + // RISING DIAGONAL CROSSING SOUTH EAST ARROW + return rune(0x2930), true + case "rdquor": + // RIGHT DOUBLE QUOTATION MARK + return rune(0x201d), true + case "rdquo": + // RIGHT DOUBLE QUOTATION MARK + return rune(0x201d), true + case "rdsh": + // DOWNWARDS ARROW WITH TIP RIGHTWARDS + return rune(0x21b3), true + case "realpart": + // BLACK-LETTER CAPITAL R + return rune(0x211c), true + case "reals": + // DOUBLE-STRUCK CAPITAL R + return rune(0x211d), true + case "real": + // BLACK-LETTER CAPITAL R + return rune(0x211c), true + case "realine": + // SCRIPT CAPITAL R + return rune(0x211b), true + case "rect": + // WHITE RECTANGLE + return rune(0x25ad), true + case "reg": + // REGISTERED SIGN + return rune(0xae), true + case "rfbowtie": + // BOWTIE WITH RIGHT HALF BLACK + return rune(0x29d2), true + case "rfisht": + // RIGHT FISH TAIL + return rune(0x297d), true + case "rfloor": + // RIGHT FLOOR + return rune(0x230b), true + case "rfr": + // MATHEMATICAL FRAKTUR SMALL R + return rune(0x01d52f), true + case "rftimes": + // TIMES WITH RIGHT HALF BLACK + return rune(0x29d5), true + case "rgr": + // GREEK SMALL LETTER RHO + return rune(0x03c1), true + case "rhard": + // RIGHTWARDS HARPOON WITH BARB DOWNWARDS + return rune(0x21c1), true + case "rharu": + // RIGHTWARDS HARPOON WITH BARB UPWARDS + return rune(0x21c0), true + case "rharul": + // RIGHTWARDS HARPOON WITH BARB UP ABOVE LONG DASH + return rune(0x296c), true + case "rhov": + // GREEK RHO SYMBOL + return rune(0x03f1), true + case "rho": + // GREEK SMALL LETTER RHO + return rune(0x03c1), true + case "rightarrowtail": + // RIGHTWARDS ARROW WITH TAIL + return rune(0x21a3), true + case "rightarrow": + // RIGHTWARDS ARROW + return rune(0x2192), true + case "rightharpoondown": + // RIGHTWARDS HARPOON WITH BARB DOWNWARDS + return rune(0x21c1), true + case "rightharpoonup": + // RIGHTWARDS HARPOON WITH BARB UPWARDS + return rune(0x21c0), true + case "rightleftarrows": + // RIGHTWARDS ARROW OVER LEFTWARDS ARROW + return rune(0x21c4), true + case "rightleftharpoons": + // RIGHTWARDS HARPOON OVER LEFTWARDS HARPOON + return rune(0x21cc), true + case "rightrightarrows": + // RIGHTWARDS PAIRED ARROWS + return rune(0x21c9), true + case "rightsquigarrow": + // RIGHTWARDS WAVE ARROW + return rune(0x219d), true + case "rightthreetimes": + // RIGHT SEMIDIRECT PRODUCT + return rune(0x22cc), true + case "rimply": + // RIGHT DOUBLE ARROW WITH ROUNDED HEAD + return rune(0x2970), true + case "ring": + // RING ABOVE + return rune(0x02da), true + case "risingdotseq": + // IMAGE OF OR APPROXIMATELY EQUAL TO + return rune(0x2253), true + case "rlarr2": + // RIGHTWARDS ARROW OVER LEFTWARDS ARROW + return rune(0x21c4), true + case "rlarr": + // RIGHTWARDS ARROW OVER LEFTWARDS ARROW + return rune(0x21c4), true + case "rlhar": + // RIGHTWARDS HARPOON OVER LEFTWARDS HARPOON + return rune(0x21cc), true + case "rlhar2": + // RIGHTWARDS HARPOON OVER LEFTWARDS HARPOON + return rune(0x21cc), true + case "rlm": + // RIGHT-TO-LEFT MARK + return rune(0x200f), true + case "rmoust": + // UPPER RIGHT OR LOWER LEFT CURLY BRACKET SECTION + return rune(0x23b1), true + case "rmoustache": + // UPPER RIGHT OR LOWER LEFT CURLY BRACKET SECTION + return rune(0x23b1), true + case "rnmid": + // DOES NOT DIVIDE WITH REVERSED NEGATION SLASH + return rune(0x2aee), true + case "roang": + // MATHEMATICAL RIGHT WHITE TORTOISE SHELL BRACKET + return rune(0x27ed), true + case "roarr": + // RIGHTWARDS OPEN-HEADED ARROW + return rune(0x21fe), true + case "robrk": + // MATHEMATICAL RIGHT WHITE SQUARE BRACKET + return rune(0x27e7), true + case "rocub": + // RIGHT WHITE CURLY BRACKET + return rune(0x2984), true + case "ropar": + // RIGHT WHITE PARENTHESIS + return rune(0x2986), true + case "ropf": + // MATHEMATICAL DOUBLE-STRUCK SMALL R + return rune(0x01d563), true + case "roplus": + // PLUS SIGN IN RIGHT HALF CIRCLE + return rune(0x2a2e), true + case "rotimes": + // MULTIPLICATION SIGN IN RIGHT HALF CIRCLE + return rune(0x2a35), true + case "rpargt": + // RIGHT ARC GREATER-THAN BRACKET + return rune(0x2994), true + case "rpar": + // RIGHT PARENTHESIS + return rune(0x29), true + case "rppolint": + // LINE INTEGRATION WITH RECTANGULAR PATH AROUND POLE + return rune(0x2a12), true + case "rrarr": + // RIGHTWARDS PAIRED ARROWS + return rune(0x21c9), true + case "rsaquo": + // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + return rune(0x203a), true + case "rscr": + // MATHEMATICAL SCRIPT SMALL R + return rune(0x01d4c7), true + case "rsh": + // UPWARDS ARROW WITH TIP RIGHTWARDS + return rune(0x21b1), true + case "rsolbar": + // REVERSE SOLIDUS WITH HORIZONTAL STROKE + return rune(0x29f7), true + case "rsqb": + // RIGHT SQUARE BRACKET + return rune(0x5d), true + case "rsquor": + // RIGHT SINGLE QUOTATION MARK + return rune(0x2019), true + case "rsquo": + // RIGHT SINGLE QUOTATION MARK + return rune(0x2019), true + case "rthree": + // RIGHT SEMIDIRECT PRODUCT + return rune(0x22cc), true + case "rtimes": + // RIGHT NORMAL FACTOR SEMIDIRECT PRODUCT + return rune(0x22ca), true + case "rtrie": + // CONTAINS AS NORMAL SUBGROUP OR EQUAL TO + return rune(0x22b5), true + case "rtrif": + // BLACK RIGHT-POINTING SMALL TRIANGLE + return rune(0x25b8), true + case "rtri": + // WHITE RIGHT-POINTING SMALL TRIANGLE + return rune(0x25b9), true + case "rtriltri": + // RIGHT TRIANGLE ABOVE LEFT TRIANGLE + return rune(0x29ce), true + case "ruharb": + // RIGHTWARDS HARPOON WITH BARB UP TO BAR + return rune(0x2953), true + case "ruluhar": + // RIGHTWARDS HARPOON WITH BARB UP ABOVE LEFTWARDS HARPOON WITH BARB UP + return rune(0x2968), true + case "rx": + // PRESCRIPTION TAKE + return rune(0x211e), true + } + + case 's': + switch name { + case "sacute": + // LATIN SMALL LETTER S WITH ACUTE + return rune(0x015b), true + case "samalg": + // N-ARY COPRODUCT + return rune(0x2210), true + case "sampi": + // GREEK LETTER SAMPI + return rune(0x03e0), true + case "sbquo": + // SINGLE LOW-9 QUOTATION MARK + return rune(0x201a), true + case "sbsol": + // SMALL REVERSE SOLIDUS + return rune(0xfe68), true + case "sc": + // SUCCEEDS + return rune(0x227b), true + case "scE": + // SUCCEEDS ABOVE EQUALS SIGN + return rune(0x2ab4), true + case "scap": + // SUCCEEDS ABOVE ALMOST EQUAL TO + return rune(0x2ab8), true + case "scaron": + // LATIN SMALL LETTER S WITH CARON + return rune(0x0161), true + case "sccue": + // SUCCEEDS OR EQUAL TO + return rune(0x227d), true + case "scedil": + // LATIN SMALL LETTER S WITH CEDILLA + return rune(0x015f), true + case "sce": + // SUCCEEDS ABOVE SINGLE-LINE EQUALS SIGN + return rune(0x2ab0), true + case "scirc": + // LATIN SMALL LETTER S WITH CIRCUMFLEX + return rune(0x015d), true + case "scnE": + // SUCCEEDS ABOVE NOT EQUAL TO + return rune(0x2ab6), true + case "scnap": + // SUCCEEDS ABOVE NOT ALMOST EQUAL TO + return rune(0x2aba), true + case "scnsim": + // SUCCEEDS BUT NOT EQUIVALENT TO + return rune(0x22e9), true + case "scpolint": + // LINE INTEGRATION WITH SEMICIRCULAR PATH AROUND POLE + return rune(0x2a13), true + case "scsim": + // SUCCEEDS OR EQUIVALENT TO + return rune(0x227f), true + case "scy": + // CYRILLIC SMALL LETTER ES + return rune(0x0441), true + case "sdotb": + // SQUARED DOT OPERATOR + return rune(0x22a1), true + case "sdot": + // DOT OPERATOR + return rune(0x22c5), true + case "sdote": + // EQUALS SIGN WITH DOT BELOW + return rune(0x2a66), true + case "seArr": + // SOUTH EAST DOUBLE ARROW + return rune(0x21d8), true + case "searhk": + // SOUTH EAST ARROW WITH HOOK + return rune(0x2925), true + case "searrow": + // SOUTH EAST ARROW + return rune(0x2198), true + case "searr": + // SOUTH EAST ARROW + return rune(0x2198), true + case "sect": + // SECTION SIGN + return rune(0xa7), true + case "semi": + // SEMICOLON + return rune(0x3b), true + case "seonearr": + // SOUTH EAST ARROW CROSSING NORTH EAST ARROW + return rune(0x292d), true + case "seswar": + // SOUTH EAST ARROW AND SOUTH WEST ARROW + return rune(0x2929), true + case "setminus": + // SET MINUS + return rune(0x2216), true + case "setmn": + // SET MINUS + return rune(0x2216), true + case "sext": + // SIX POINTED BLACK STAR + return rune(0x2736), true + case "sfgr": + // GREEK SMALL LETTER FINAL SIGMA + return rune(0x03c2), true + case "sfrown": + // FROWN + return rune(0x2322), true + case "sfr": + // MATHEMATICAL FRAKTUR SMALL S + return rune(0x01d530), true + case "sgr": + // GREEK SMALL LETTER SIGMA + return rune(0x03c3), true + case "sharp": + // MUSIC SHARP SIGN + return rune(0x266f), true + case "shchcy": + // CYRILLIC SMALL LETTER SHCHA + return rune(0x0449), true + case "shcy": + // CYRILLIC SMALL LETTER SHA + return rune(0x0448), true + case "shortmid": + // DIVIDES + return rune(0x2223), true + case "shortparallel": + // PARALLEL TO + return rune(0x2225), true + case "shuffle": + // SHUFFLE PRODUCT + return rune(0x29e2), true + case "shy": + // SOFT HYPHEN + return rune(0xad), true + case "sigma": + // GREEK SMALL LETTER SIGMA + return rune(0x03c3), true + case "sigmaf": + // GREEK SMALL LETTER FINAL SIGMA + return rune(0x03c2), true + case "sigmav": + // GREEK SMALL LETTER FINAL SIGMA + return rune(0x03c2), true + case "sim": + // TILDE OPERATOR + return rune(0x223c), true + case "simdot": + // TILDE OPERATOR WITH DOT ABOVE + return rune(0x2a6a), true + case "sime": + // ASYMPTOTICALLY EQUAL TO + return rune(0x2243), true + case "simeq": + // ASYMPTOTICALLY EQUAL TO + return rune(0x2243), true + case "simg": + // SIMILAR OR GREATER-THAN + return rune(0x2a9e), true + case "simgE": + // SIMILAR ABOVE GREATER-THAN ABOVE EQUALS SIGN + return rune(0x2aa0), true + case "siml": + // SIMILAR OR LESS-THAN + return rune(0x2a9d), true + case "simlE": + // SIMILAR ABOVE LESS-THAN ABOVE EQUALS SIGN + return rune(0x2a9f), true + case "simne": + // APPROXIMATELY BUT NOT ACTUALLY EQUAL TO + return rune(0x2246), true + case "simplus": + // PLUS SIGN WITH TILDE ABOVE + return rune(0x2a24), true + case "simrarr": + // TILDE OPERATOR ABOVE RIGHTWARDS ARROW + return rune(0x2972), true + case "slarr": + // LEFTWARDS ARROW + return rune(0x2190), true + case "slint": + // INTEGRAL AVERAGE WITH SLASH + return rune(0x2a0f), true + case "smallsetminus": + // SET MINUS + return rune(0x2216), true + case "smashp": + // SMASH PRODUCT + return rune(0x2a33), true + case "smeparsl": + // EQUALS SIGN AND SLANTED PARALLEL WITH TILDE ABOVE + return rune(0x29e4), true + case "smid": + // DIVIDES + return rune(0x2223), true + case "smile": + // SMILE + return rune(0x2323), true + case "smt": + // SMALLER THAN + return rune(0x2aaa), true + case "smte": + // SMALLER THAN OR EQUAL TO + return rune(0x2aac), true + case "smtes": + // SMALLER THAN OR slanted EQUAL + return rune(0x2aac), true + case "softcy": + // CYRILLIC SMALL LETTER SOFT SIGN + return rune(0x044c), true + case "solbar": + // APL FUNCTIONAL SYMBOL SLASH BAR + return rune(0x233f), true + case "solb": + // SQUARED RISING DIAGONAL SLASH + return rune(0x29c4), true + case "sol": + // SOLIDUS + return rune(0x2f), true + case "sopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL S + return rune(0x01d564), true + case "spades": + // BLACK SPADE SUIT + return rune(0x2660), true + case "spadesuit": + // BLACK SPADE SUIT + return rune(0x2660), true + case "spar": + // PARALLEL TO + return rune(0x2225), true + case "sqcap": + // SQUARE CAP + return rune(0x2293), true + case "sqcaps": + // SQUARE CAP with serifs + return rune(0x2293), true + case "sqcup": + // SQUARE CUP + return rune(0x2294), true + case "sqcups": + // SQUARE CUP with serifs + return rune(0x2294), true + case "sqsub": + // SQUARE IMAGE OF + return rune(0x228f), true + case "sqsube": + // SQUARE IMAGE OF OR EQUAL TO + return rune(0x2291), true + case "sqsubset": + // SQUARE IMAGE OF + return rune(0x228f), true + case "sqsubseteq": + // SQUARE IMAGE OF OR EQUAL TO + return rune(0x2291), true + case "sqsup": + // SQUARE ORIGINAL OF + return rune(0x2290), true + case "sqsupe": + // SQUARE ORIGINAL OF OR EQUAL TO + return rune(0x2292), true + case "sqsupset": + // SQUARE ORIGINAL OF + return rune(0x2290), true + case "sqsupseteq": + // SQUARE ORIGINAL OF OR EQUAL TO + return rune(0x2292), true + case "squ": + // WHITE SQUARE + return rune(0x25a1), true + case "square": + // WHITE SQUARE + return rune(0x25a1), true + case "squarf": + // BLACK SMALL SQUARE + return rune(0x25aa), true + case "squb": + // SQUARED SQUARE + return rune(0x29c8), true + case "squerr": + // ERROR-BARRED WHITE SQUARE + return rune(0x29ee), true + case "squf": + // BLACK SMALL SQUARE + return rune(0x25aa), true + case "squferr": + // ERROR-BARRED BLACK SQUARE + return rune(0x29ef), true + case "srarr": + // RIGHTWARDS ARROW + return rune(0x2192), true + case "sscr": + // MATHEMATICAL SCRIPT SMALL S + return rune(0x01d4c8), true + case "ssetmn": + // SET MINUS + return rune(0x2216), true + case "ssmile": + // SMILE + return rune(0x2323), true + case "sstarf": + // STAR OPERATOR + return rune(0x22c6), true + case "starf": + // BLACK STAR + return rune(0x2605), true + case "star": + // WHITE STAR + return rune(0x2606), true + case "stigma": + // GREEK LETTER STIGMA + return rune(0x03da), true + case "straightepsilon": + // GREEK LUNATE EPSILON SYMBOL + return rune(0x03f5), true + case "straightphi": + // GREEK PHI SYMBOL + return rune(0x03d5), true + case "strns": + // MACRON + return rune(0xaf), true + case "sub": + // SUBSET OF + return rune(0x2282), true + case "subE": + // SUBSET OF ABOVE EQUALS SIGN + return rune(0x2ac5), true + case "subdot": + // SUBSET WITH DOT + return rune(0x2abd), true + case "sube": + // SUBSET OF OR EQUAL TO + return rune(0x2286), true + case "subedot": + // SUBSET OF OR EQUAL TO WITH DOT ABOVE + return rune(0x2ac3), true + case "submult": + // SUBSET WITH MULTIPLICATION SIGN BELOW + return rune(0x2ac1), true + case "subnE": + // SUBSET OF ABOVE NOT EQUAL TO + return rune(0x2acb), true + case "subne": + // SUBSET OF WITH NOT EQUAL TO + return rune(0x228a), true + case "subplus": + // SUBSET WITH PLUS SIGN BELOW + return rune(0x2abf), true + case "subrarr": + // SUBSET ABOVE RIGHTWARDS ARROW + return rune(0x2979), true + case "subset": + // SUBSET OF + return rune(0x2282), true + case "subseteq": + // SUBSET OF OR EQUAL TO + return rune(0x2286), true + case "subseteqq": + // SUBSET OF ABOVE EQUALS SIGN + return rune(0x2ac5), true + case "subsetneq": + // SUBSET OF WITH NOT EQUAL TO + return rune(0x228a), true + case "subsetneqq": + // SUBSET OF ABOVE NOT EQUAL TO + return rune(0x2acb), true + case "subsim": + // SUBSET OF ABOVE TILDE OPERATOR + return rune(0x2ac7), true + case "subsub": + // SUBSET ABOVE SUBSET + return rune(0x2ad5), true + case "subsup": + // SUBSET ABOVE SUPERSET + return rune(0x2ad3), true + case "succ": + // SUCCEEDS + return rune(0x227b), true + case "succapprox": + // SUCCEEDS ABOVE ALMOST EQUAL TO + return rune(0x2ab8), true + case "succcurlyeq": + // SUCCEEDS OR EQUAL TO + return rune(0x227d), true + case "succeq": + // SUCCEEDS ABOVE SINGLE-LINE EQUALS SIGN + return rune(0x2ab0), true + case "succnapprox": + // SUCCEEDS ABOVE NOT ALMOST EQUAL TO + return rune(0x2aba), true + case "succneqq": + // SUCCEEDS ABOVE NOT EQUAL TO + return rune(0x2ab6), true + case "succnsim": + // SUCCEEDS BUT NOT EQUIVALENT TO + return rune(0x22e9), true + case "succsim": + // SUCCEEDS OR EQUIVALENT TO + return rune(0x227f), true + case "sum": + // N-ARY SUMMATION + return rune(0x2211), true + case "sumint": + // SUMMATION WITH INTEGRAL + return rune(0x2a0b), true + case "sung": + // EIGHTH NOTE + return rune(0x266a), true + case "sup": + // SUPERSET OF + return rune(0x2283), true + case "sup1": + // SUPERSCRIPT ONE + return rune(0xb9), true + case "sup2": + // SUPERSCRIPT TWO + return rune(0xb2), true + case "sup3": + // SUPERSCRIPT THREE + return rune(0xb3), true + case "supE": + // SUPERSET OF ABOVE EQUALS SIGN + return rune(0x2ac6), true + case "supdot": + // SUPERSET WITH DOT + return rune(0x2abe), true + case "supdsub": + // SUPERSET BESIDE AND JOINED BY DASH WITH SUBSET + return rune(0x2ad8), true + case "supe": + // SUPERSET OF OR EQUAL TO + return rune(0x2287), true + case "supedot": + // SUPERSET OF OR EQUAL TO WITH DOT ABOVE + return rune(0x2ac4), true + case "suphsol": + // SUPERSET PRECEDING SOLIDUS + return rune(0x27c9), true + case "suphsub": + // SUPERSET BESIDE SUBSET + return rune(0x2ad7), true + case "suplarr": + // SUPERSET ABOVE LEFTWARDS ARROW + return rune(0x297b), true + case "supmult": + // SUPERSET WITH MULTIPLICATION SIGN BELOW + return rune(0x2ac2), true + case "supnE": + // SUPERSET OF ABOVE NOT EQUAL TO + return rune(0x2acc), true + case "supne": + // SUPERSET OF WITH NOT EQUAL TO + return rune(0x228b), true + case "supplus": + // SUPERSET WITH PLUS SIGN BELOW + return rune(0x2ac0), true + case "supset": + // SUPERSET OF + return rune(0x2283), true + case "supseteq": + // SUPERSET OF OR EQUAL TO + return rune(0x2287), true + case "supseteqq": + // SUPERSET OF ABOVE EQUALS SIGN + return rune(0x2ac6), true + case "supsetneq": + // SUPERSET OF WITH NOT EQUAL TO + return rune(0x228b), true + case "supsetneqq": + // SUPERSET OF ABOVE NOT EQUAL TO + return rune(0x2acc), true + case "supsim": + // SUPERSET OF ABOVE TILDE OPERATOR + return rune(0x2ac8), true + case "supsub": + // SUPERSET ABOVE SUBSET + return rune(0x2ad4), true + case "supsup": + // SUPERSET ABOVE SUPERSET + return rune(0x2ad6), true + case "swArr": + // SOUTH WEST DOUBLE ARROW + return rune(0x21d9), true + case "swarhk": + // SOUTH WEST ARROW WITH HOOK + return rune(0x2926), true + case "swarrow": + // SOUTH WEST ARROW + return rune(0x2199), true + case "swarr": + // SOUTH WEST ARROW + return rune(0x2199), true + case "swnwar": + // SOUTH WEST ARROW AND NORTH WEST ARROW + return rune(0x292a), true + case "szlig": + // LATIN SMALL LETTER SHARP S + return rune(0xdf), true + } + + case 't': + switch name { + case "target": + // POSITION INDICATOR + return rune(0x2316), true + case "tau": + // GREEK SMALL LETTER TAU + return rune(0x03c4), true + case "tbrk": + // TOP SQUARE BRACKET + return rune(0x23b4), true + case "tcaron": + // LATIN SMALL LETTER T WITH CARON + return rune(0x0165), true + case "tcedil": + // LATIN SMALL LETTER T WITH CEDILLA + return rune(0x0163), true + case "tcy": + // CYRILLIC SMALL LETTER TE + return rune(0x0442), true + case "tdot": + // COMBINING THREE DOTS ABOVE + return rune(0x20db), true + case "telrec": + // TELEPHONE RECORDER + return rune(0x2315), true + case "tfr": + // MATHEMATICAL FRAKTUR SMALL T + return rune(0x01d531), true + case "tgr": + // GREEK SMALL LETTER TAU + return rune(0x03c4), true + case "there4": + // THEREFORE + return rune(0x2234), true + case "therefore": + // THEREFORE + return rune(0x2234), true + case "thermod": + // THERMODYNAMIC + return rune(0x29e7), true + case "thetasym": + // GREEK THETA SYMBOL + return rune(0x03d1), true + case "thetas": + // GREEK SMALL LETTER THETA + return rune(0x03b8), true + case "thetav": + // GREEK THETA SYMBOL + return rune(0x03d1), true + case "theta": + // GREEK SMALL LETTER THETA + return rune(0x03b8), true + case "thgr": + // GREEK SMALL LETTER THETA + return rune(0x03b8), true + case "thickapprox": + // ALMOST EQUAL TO + return rune(0x2248), true + case "thicksim": + // TILDE OPERATOR + return rune(0x223c), true + case "thinsp": + // THIN SPACE + return rune(0x2009), true + case "thkap": + // ALMOST EQUAL TO + return rune(0x2248), true + case "thksim": + // TILDE OPERATOR + return rune(0x223c), true + case "thorn": + // LATIN SMALL LETTER THORN + return rune(0xfe), true + case "tilde": + // SMALL TILDE + return rune(0x02dc), true + case "timeint": + // INTEGRAL WITH TIMES SIGN + return rune(0x2a18), true + case "timesb": + // SQUARED TIMES + return rune(0x22a0), true + case "timesbar": + // MULTIPLICATION SIGN WITH UNDERBAR + return rune(0x2a31), true + case "timesd": + // MULTIPLICATION SIGN WITH DOT ABOVE + return rune(0x2a30), true + case "times": + // MULTIPLICATION SIGN + return rune(0xd7), true + case "tint": + // TRIPLE INTEGRAL + return rune(0x222d), true + case "toea": + // NORTH EAST ARROW AND SOUTH EAST ARROW + return rune(0x2928), true + case "top": + // DOWN TACK + return rune(0x22a4), true + case "topbot": + // APL FUNCTIONAL SYMBOL I-BEAM + return rune(0x2336), true + case "topcir": + // DOWN TACK WITH CIRCLE BELOW + return rune(0x2af1), true + case "topfork": + // PITCHFORK WITH TEE TOP + return rune(0x2ada), true + case "topf": + // MATHEMATICAL DOUBLE-STRUCK SMALL T + return rune(0x01d565), true + case "tosa": + // SOUTH EAST ARROW AND SOUTH WEST ARROW + return rune(0x2929), true + case "tprime": + // TRIPLE PRIME + return rune(0x2034), true + case "trade": + // TRADE MARK SIGN + return rune(0x2122), true + case "triS": + // S IN TRIANGLE + return rune(0x29cc), true + case "trianglelefteq": + // NORMAL SUBGROUP OF OR EQUAL TO + return rune(0x22b4), true + case "triangleq": + // DELTA EQUAL TO + return rune(0x225c), true + case "trianglerighteq": + // CONTAINS AS NORMAL SUBGROUP OR EQUAL TO + return rune(0x22b5), true + case "triangle": + // WHITE UP-POINTING SMALL TRIANGLE + return rune(0x25b5), true + case "triangledown": + // WHITE DOWN-POINTING SMALL TRIANGLE + return rune(0x25bf), true + case "triangleleft": + // WHITE LEFT-POINTING SMALL TRIANGLE + return rune(0x25c3), true + case "triangleright": + // WHITE RIGHT-POINTING SMALL TRIANGLE + return rune(0x25b9), true + case "tribar": + // TRIANGLE WITH UNDERBAR + return rune(0x29cb), true + case "tridot": + // WHITE UP-POINTING TRIANGLE WITH DOT + return rune(0x25ec), true + case "tridoto": + // TRIANGLE WITH DOT ABOVE + return rune(0x29ca), true + case "trie": + // DELTA EQUAL TO + return rune(0x225c), true + case "triminus": + // MINUS SIGN IN TRIANGLE + return rune(0x2a3a), true + case "triplus": + // PLUS SIGN IN TRIANGLE + return rune(0x2a39), true + case "trisb": + // TRIANGLE WITH SERIFS AT BOTTOM + return rune(0x29cd), true + case "tritime": + // MULTIPLICATION SIGN IN TRIANGLE + return rune(0x2a3b), true + case "trpezium": + // WHITE TRAPEZIUM + return rune(0x23e2), true + case "tscr": + // MATHEMATICAL SCRIPT SMALL T + return rune(0x01d4c9), true + case "tscy": + // CYRILLIC SMALL LETTER TSE + return rune(0x0446), true + case "tshcy": + // CYRILLIC SMALL LETTER TSHE + return rune(0x045b), true + case "tstrok": + // LATIN SMALL LETTER T WITH STROKE + return rune(0x0167), true + case "tverbar": + // TRIPLE VERTICAL BAR DELIMITER + return rune(0x2980), true + case "twixt": + // BETWEEN + return rune(0x226c), true + case "twoheadleftarrow": + // LEFTWARDS TWO HEADED ARROW + return rune(0x219e), true + case "twoheadrightarrow": + // RIGHTWARDS TWO HEADED ARROW + return rune(0x21a0), true + } + + case 'u': + switch name { + case "uAarr": + // UPWARDS TRIPLE ARROW + return rune(0x290a), true + case "uArr": + // UPWARDS DOUBLE ARROW + return rune(0x21d1), true + case "uHar": + // UPWARDS HARPOON WITH BARB LEFT BESIDE UPWARDS HARPOON WITH BARB RIGHT + return rune(0x2963), true + case "uacgr": + // GREEK SMALL LETTER UPSILON WITH TONOS + return rune(0x03cd), true + case "uacute": + // LATIN SMALL LETTER U WITH ACUTE + return rune(0xfa), true + case "uarr2": + // UPWARDS PAIRED ARROWS + return rune(0x21c8), true + case "uarr": + // UPWARDS ARROW + return rune(0x2191), true + case "uarrb": + // UPWARDS ARROW TO BAR + return rune(0x2912), true + case "uarrln": + // UPWARDS ARROW WITH HORIZONTAL STROKE + return rune(0x2909), true + case "ubrcy": + // CYRILLIC SMALL LETTER SHORT U + return rune(0x045e), true + case "ubreve": + // LATIN SMALL LETTER U WITH BREVE + return rune(0x016d), true + case "ucirc": + // LATIN SMALL LETTER U WITH CIRCUMFLEX + return rune(0xfb), true + case "ucy": + // CYRILLIC SMALL LETTER U + return rune(0x0443), true + case "udarr": + // UPWARDS ARROW LEFTWARDS OF DOWNWARDS ARROW + return rune(0x21c5), true + case "udblac": + // LATIN SMALL LETTER U WITH DOUBLE ACUTE + return rune(0x0171), true + case "udhar": + // UPWARDS HARPOON WITH BARB LEFT BESIDE DOWNWARDS HARPOON WITH BARB RIGHT + return rune(0x296e), true + case "udiagr": + // GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + return rune(0x03b0), true + case "udigr": + // GREEK SMALL LETTER UPSILON WITH DIALYTIKA + return rune(0x03cb), true + case "udrbrk": + // BOTTOM SQUARE BRACKET + return rune(0x23b5), true + case "udrcub": + // BOTTOM CURLY BRACKET + return rune(0x23df), true + case "udrpar": + // BOTTOM PARENTHESIS + return rune(0x23dd), true + case "ufisht": + // UP FISH TAIL + return rune(0x297e), true + case "ufr": + // MATHEMATICAL FRAKTUR SMALL U + return rune(0x01d532), true + case "ugr": + // GREEK SMALL LETTER UPSILON + return rune(0x03c5), true + case "ugrave": + // LATIN SMALL LETTER U WITH GRAVE + return rune(0xf9), true + case "uharl": + // UPWARDS HARPOON WITH BARB LEFTWARDS + return rune(0x21bf), true + case "uharr": + // UPWARDS HARPOON WITH BARB RIGHTWARDS + return rune(0x21be), true + case "uhblk": + // UPPER HALF BLOCK + return rune(0x2580), true + case "ulcorn": + // TOP LEFT CORNER + return rune(0x231c), true + case "ulcorner": + // TOP LEFT CORNER + return rune(0x231c), true + case "ulcrop": + // TOP LEFT CROP + return rune(0x230f), true + case "uldlshar": + // UP BARB LEFT DOWN BARB LEFT HARPOON + return rune(0x2951), true + case "ulharb": + // UPWARDS HARPOON WITH BARB LEFT TO BAR + return rune(0x2958), true + case "ultri": + // UPPER LEFT TRIANGLE + return rune(0x25f8), true + case "umacr": + // LATIN SMALL LETTER U WITH MACRON + return rune(0x016b), true + case "uml": + // DIAERESIS + return rune(0xa8), true + case "uogon": + // LATIN SMALL LETTER U WITH OGONEK + return rune(0x0173), true + case "uopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL U + return rune(0x01d566), true + case "uparrow": + // UPWARDS ARROW + return rune(0x2191), true + case "updownarrow": + // UP DOWN ARROW + return rune(0x2195), true + case "upharpoonleft": + // UPWARDS HARPOON WITH BARB LEFTWARDS + return rune(0x21bf), true + case "upharpoonright": + // UPWARDS HARPOON WITH BARB RIGHTWARDS + return rune(0x21be), true + case "upint": + // INTEGRAL WITH OVERBAR + return rune(0x2a1b), true + case "uplus": + // MULTISET UNION + return rune(0x228e), true + case "upsih": + // GREEK UPSILON WITH HOOK SYMBOL + return rune(0x03d2), true + case "upsilon": + // GREEK SMALL LETTER UPSILON + return rune(0x03c5), true + case "upsi": + // GREEK SMALL LETTER UPSILON + return rune(0x03c5), true + case "upuparrows": + // UPWARDS PAIRED ARROWS + return rune(0x21c8), true + case "urcorn": + // TOP RIGHT CORNER + return rune(0x231d), true + case "urcorner": + // TOP RIGHT CORNER + return rune(0x231d), true + case "urcrop": + // TOP RIGHT CROP + return rune(0x230e), true + case "urdrshar": + // UP BARB RIGHT DOWN BARB RIGHT HARPOON + return rune(0x294f), true + case "urharb": + // UPWARDS HARPOON WITH BARB RIGHT TO BAR + return rune(0x2954), true + case "uring": + // LATIN SMALL LETTER U WITH RING ABOVE + return rune(0x016f), true + case "urtrif": + // BLACK UPPER RIGHT TRIANGLE + return rune(0x25e5), true + case "urtri": + // UPPER RIGHT TRIANGLE + return rune(0x25f9), true + case "uscr": + // MATHEMATICAL SCRIPT SMALL U + return rune(0x01d4ca), true + case "utdot": + // UP RIGHT DIAGONAL ELLIPSIS + return rune(0x22f0), true + case "utilde": + // LATIN SMALL LETTER U WITH TILDE + return rune(0x0169), true + case "utrif": + // BLACK UP-POINTING SMALL TRIANGLE + return rune(0x25b4), true + case "utri": + // WHITE UP-POINTING SMALL TRIANGLE + return rune(0x25b5), true + case "uuarr": + // UPWARDS PAIRED ARROWS + return rune(0x21c8), true + case "uuml": + // LATIN SMALL LETTER U WITH DIAERESIS + return rune(0xfc), true + case "uwangle": + // OBLIQUE ANGLE OPENING DOWN + return rune(0x29a7), true + } + + case 'v': + switch name { + case "vArr": + // UP DOWN DOUBLE ARROW + return rune(0x21d5), true + case "vBar": + // SHORT UP TACK WITH UNDERBAR + return rune(0x2ae8), true + case "vBarv": + // SHORT UP TACK ABOVE SHORT DOWN TACK + return rune(0x2ae9), true + case "vDash": + // TRUE + return rune(0x22a8), true + case "vDdash": + // VERTICAL BAR TRIPLE RIGHT TURNSTILE + return rune(0x2ae2), true + case "vangrt": + // RIGHT ANGLE VARIANT WITH SQUARE + return rune(0x299c), true + case "varepsilon": + // GREEK LUNATE EPSILON SYMBOL + return rune(0x03f5), true + case "varkappa": + // GREEK KAPPA SYMBOL + return rune(0x03f0), true + case "varnothing": + // EMPTY SET + return rune(0x2205), true + case "varphi": + // GREEK PHI SYMBOL + return rune(0x03d5), true + case "varpi": + // GREEK PI SYMBOL + return rune(0x03d6), true + case "varpropto": + // PROPORTIONAL TO + return rune(0x221d), true + case "varr": + // UP DOWN ARROW + return rune(0x2195), true + case "varrho": + // GREEK RHO SYMBOL + return rune(0x03f1), true + case "varsigma": + // GREEK SMALL LETTER FINAL SIGMA + return rune(0x03c2), true + case "varsubsetneq": + // SUBSET OF WITH NOT EQUAL TO - variant with stroke through bottom members + return rune(0x228a), true + case "varsubsetneqq": + // SUBSET OF ABOVE NOT EQUAL TO - variant with stroke through bottom members + return rune(0x2acb), true + case "varsupsetneq": + // SUPERSET OF WITH NOT EQUAL TO - variant with stroke through bottom members + return rune(0x228b), true + case "varsupsetneqq": + // SUPERSET OF ABOVE NOT EQUAL TO - variant with stroke through bottom members + return rune(0x2acc), true + case "vartheta": + // GREEK THETA SYMBOL + return rune(0x03d1), true + case "vartriangleleft": + // NORMAL SUBGROUP OF + return rune(0x22b2), true + case "vartriangleright": + // CONTAINS AS NORMAL SUBGROUP + return rune(0x22b3), true + case "vbrtri": + // VERTICAL BAR BESIDE RIGHT TRIANGLE + return rune(0x29d0), true + case "vcy": + // CYRILLIC SMALL LETTER VE + return rune(0x0432), true + case "vdash": + // RIGHT TACK + return rune(0x22a2), true + case "vee": + // LOGICAL OR + return rune(0x2228), true + case "veeBar": + // LOGICAL OR WITH DOUBLE UNDERBAR + return rune(0x2a63), true + case "veebar": + // XOR + return rune(0x22bb), true + case "veeeq": + // EQUIANGULAR TO + return rune(0x225a), true + case "vellip": + // VERTICAL ELLIPSIS + return rune(0x22ee), true + case "vellip4": + // DOTTED FENCE + return rune(0x2999), true + case "vellipv": + // TRIPLE COLON OPERATOR + return rune(0x2af6), true + case "verbar": + // VERTICAL LINE + return rune(0x7c), true + case "vert3": + // TRIPLE VERTICAL BAR BINARY RELATION + return rune(0x2af4), true + case "vert": + // VERTICAL LINE + return rune(0x7c), true + case "vfr": + // MATHEMATICAL FRAKTUR SMALL V + return rune(0x01d533), true + case "vldash": + // LEFT SQUARE BRACKET LOWER CORNER + return rune(0x23a3), true + case "vltri": + // NORMAL SUBGROUP OF + return rune(0x22b2), true + case "vnsub": + // SUBSET OF with vertical line + return rune(0x2282), true + case "vnsup": + // SUPERSET OF with vertical line + return rune(0x2283), true + case "vopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL V + return rune(0x01d567), true + case "vprime": + // PRIME + return rune(0x2032), true + case "vprop": + // PROPORTIONAL TO + return rune(0x221d), true + case "vrtri": + // CONTAINS AS NORMAL SUBGROUP + return rune(0x22b3), true + case "vscr": + // MATHEMATICAL SCRIPT SMALL V + return rune(0x01d4cb), true + case "vsubnE": + // SUBSET OF ABOVE NOT EQUAL TO - variant with stroke through bottom members + return rune(0x2acb), true + case "vsubne": + // SUBSET OF WITH NOT EQUAL TO - variant with stroke through bottom members + return rune(0x228a), true + case "vsupnE": + // SUPERSET OF ABOVE NOT EQUAL TO - variant with stroke through bottom members + return rune(0x2acc), true + case "vsupne": + // SUPERSET OF WITH NOT EQUAL TO - variant with stroke through bottom members + return rune(0x228b), true + case "vzigzag": + // VERTICAL ZIGZAG LINE + return rune(0x299a), true + } + + case 'w': + switch name { + case "wcirc": + // LATIN SMALL LETTER W WITH CIRCUMFLEX + return rune(0x0175), true + case "wedbar": + // LOGICAL AND WITH UNDERBAR + return rune(0x2a5f), true + case "wedge": + // LOGICAL AND + return rune(0x2227), true + case "wedgeq": + // ESTIMATES + return rune(0x2259), true + case "weierp": + // SCRIPT CAPITAL P + return rune(0x2118), true + case "wfr": + // MATHEMATICAL FRAKTUR SMALL W + return rune(0x01d534), true + case "wopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL W + return rune(0x01d568), true + case "wp": + // SCRIPT CAPITAL P + return rune(0x2118), true + case "wreath": + // WREATH PRODUCT + return rune(0x2240), true + case "wr": + // WREATH PRODUCT + return rune(0x2240), true + case "wscr": + // MATHEMATICAL SCRIPT SMALL W + return rune(0x01d4cc), true + } + + case 'x': + switch name { + case "xandand": + // TWO LOGICAL AND OPERATOR + return rune(0x2a07), true + case "xbsol": + // BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT + return rune(0x2571), true + case "xcap": + // N-ARY INTERSECTION + return rune(0x22c2), true + case "xcirc": + // LARGE CIRCLE + return rune(0x25ef), true + case "xcup": + // N-ARY UNION + return rune(0x22c3), true + case "xcupdot": + // N-ARY UNION OPERATOR WITH DOT + return rune(0x2a03), true + case "xdtri": + // WHITE DOWN-POINTING TRIANGLE + return rune(0x25bd), true + case "xfr": + // MATHEMATICAL FRAKTUR SMALL X + return rune(0x01d535), true + case "xgr": + // GREEK SMALL LETTER XI + return rune(0x03be), true + case "xhArr": + // LONG LEFT RIGHT DOUBLE ARROW + return rune(0x27fa), true + case "xharr": + // LONG LEFT RIGHT ARROW + return rune(0x27f7), true + case "xi": + // GREEK SMALL LETTER XI + return rune(0x03be), true + case "xlArr": + // LONG LEFTWARDS DOUBLE ARROW + return rune(0x27f8), true + case "xlarr": + // LONG LEFTWARDS ARROW + return rune(0x27f5), true + case "xmap": + // LONG RIGHTWARDS ARROW FROM BAR + return rune(0x27fc), true + case "xnis": + // CONTAINS WITH VERTICAL BAR AT END OF HORIZONTAL STROKE + return rune(0x22fb), true + case "xodot": + // N-ARY CIRCLED DOT OPERATOR + return rune(0x2a00), true + case "xopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL X + return rune(0x01d569), true + case "xoplus": + // N-ARY CIRCLED PLUS OPERATOR + return rune(0x2a01), true + case "xoror": + // TWO LOGICAL OR OPERATOR + return rune(0x2a08), true + case "xotime": + // N-ARY CIRCLED TIMES OPERATOR + return rune(0x2a02), true + case "xrArr": + // LONG RIGHTWARDS DOUBLE ARROW + return rune(0x27f9), true + case "xrarr": + // LONG RIGHTWARDS ARROW + return rune(0x27f6), true + case "xscr": + // MATHEMATICAL SCRIPT SMALL X + return rune(0x01d4cd), true + case "xsol": + // BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT + return rune(0x2572), true + case "xsqcap": + // N-ARY SQUARE INTERSECTION OPERATOR + return rune(0x2a05), true + case "xsqcup": + // N-ARY SQUARE UNION OPERATOR + return rune(0x2a06), true + case "xsqu": + // WHITE MEDIUM SQUARE + return rune(0x25fb), true + case "xsquf": + // BLACK MEDIUM SQUARE + return rune(0x25fc), true + case "xtimes": + // N-ARY TIMES OPERATOR + return rune(0x2a09), true + case "xuplus": + // N-ARY UNION OPERATOR WITH PLUS + return rune(0x2a04), true + case "xutri": + // WHITE UP-POINTING TRIANGLE + return rune(0x25b3), true + case "xvee": + // N-ARY LOGICAL OR + return rune(0x22c1), true + case "xwedge": + // N-ARY LOGICAL AND + return rune(0x22c0), true + } + + case 'y': + switch name { + case "yacute": + // LATIN SMALL LETTER Y WITH ACUTE + return rune(0xfd), true + case "yacy": + // CYRILLIC SMALL LETTER YA + return rune(0x044f), true + case "ycirc": + // LATIN SMALL LETTER Y WITH CIRCUMFLEX + return rune(0x0177), true + case "ycy": + // CYRILLIC SMALL LETTER YERU + return rune(0x044b), true + case "yen": + // YEN SIGN + return rune(0xa5), true + case "yfr": + // MATHEMATICAL FRAKTUR SMALL Y + return rune(0x01d536), true + case "yicy": + // CYRILLIC SMALL LETTER YI + return rune(0x0457), true + case "yopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL Y + return rune(0x01d56a), true + case "yscr": + // MATHEMATICAL SCRIPT SMALL Y + return rune(0x01d4ce), true + case "yucy": + // CYRILLIC SMALL LETTER YU + return rune(0x044e), true + case "yuml": + // LATIN SMALL LETTER Y WITH DIAERESIS + return rune(0xff), true + } + + case 'z': + switch name { + case "zacute": + // LATIN SMALL LETTER Z WITH ACUTE + return rune(0x017a), true + case "zcaron": + // LATIN SMALL LETTER Z WITH CARON + return rune(0x017e), true + case "zcy": + // CYRILLIC SMALL LETTER ZE + return rune(0x0437), true + case "zdot": + // LATIN SMALL LETTER Z WITH DOT ABOVE + return rune(0x017c), true + case "zeetrf": + // BLACK-LETTER CAPITAL Z + return rune(0x2128), true + case "zeta": + // GREEK SMALL LETTER ZETA + return rune(0x03b6), true + case "zfr": + // MATHEMATICAL FRAKTUR SMALL Z + return rune(0x01d537), true + case "zgr": + // GREEK SMALL LETTER ZETA + return rune(0x03b6), true + case "zhcy": + // CYRILLIC SMALL LETTER ZHE + return rune(0x0436), true + case "zigrarr": + // RIGHTWARDS SQUIGGLE ARROW + return rune(0x21dd), true + case "zopf": + // MATHEMATICAL DOUBLE-STRUCK SMALL Z + return rune(0x01d56b), true + case "zscr": + // MATHEMATICAL SCRIPT SMALL Z + return rune(0x01d4cf), true + case "zwj": + // ZERO WIDTH JOINER + return rune(0x200d), true + case "zwnj": + // ZERO WIDTH NON-JOINER + return rune(0x200c), true + } + } + return -1, false +} + +/* + ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ +*/ diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 563294309..146c278cb 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -519,8 +519,6 @@ parse_attribute :: proc(doc: ^Document) -> (attr: Attr, offset: int, err: Error) _ = expect(t, .Eq) or_return value := expect(t, .String) or_return - error(t, t.offset, "String: %v\n", value) - attr.key = strings.intern_get(&doc.intern, key.text) attr.val = strings.intern_get(&doc.intern, value.text) diff --git a/core/unicode/tools/generate_entity_table.odin b/core/unicode/tools/generate_entity_table.odin new file mode 100644 index 000000000..075ec1cca --- /dev/null +++ b/core/unicode/tools/generate_entity_table.odin @@ -0,0 +1,287 @@ +package xml_example + +import "core:encoding/xml" +import "core:os" +import "core:path" +import "core:mem" +import "core:strings" +import "core:strconv" +import "core:slice" +import "core:fmt" + +/* + Silent error handler for the parser. +*/ +Error_Handler :: proc(pos: xml.Pos, fmt: string, args: ..any) {} + +OPTIONS :: xml.Options{ flags = { .Ignore_Unsupported, }, expected_doctype = "unicode", } + +Entity :: struct { + name: string, + codepoint: rune, + description: string, +} + +generate_encoding_entity_table :: proc() { + using fmt + + filename := path.join(ODIN_ROOT, "tests", "core", "assets", "XML", "unicode.xml") + defer delete(filename) + + generated_filename := path.join(ODIN_ROOT, "core", "encoding", "entity", "generated.odin") + defer delete(generated_filename) + + doc, err := xml.parse(filename, OPTIONS, Error_Handler) + defer xml.destroy(doc) + + if err != .None { + printf("Load/Parse error: %v\n", err) + if err == .File_Error { + printf("\"%v\" not found. Did you run \"tests\\download_assets.py\"?", filename) + } + os.exit(1) + } + + printf("\"%v\" loaded and parsed.\n", filename) + + generated_buf: strings.Builder + defer strings.destroy_builder(&generated_buf) + w := strings.to_writer(&generated_buf) + + charlist, charlist_ok := xml.find_child_by_ident(doc.root, "charlist") + if !charlist_ok { + eprintln("Could not locate top-level `` tag.") + os.exit(1) + } + + printf("Found `` with %v children.\n", len(charlist.children)) + + entity_map: map[string]Entity + names: [dynamic]string + + min_name_length := max(int) + max_name_length := min(int) + shortest_name: string + longest_name: string + + count := 0 + for char in charlist.children { + if char.ident != "character" { + eprintf("Expected ``, got `<%v>`\n", char.ident) + os.exit(1) + } + + if codepoint_string, ok := xml.find_attribute_val_by_key(char, "dec"); !ok { + eprintln("`` attribute not found.") + os.exit(1) + } else { + codepoint := strconv.atoi(codepoint_string) + + desc, desc_ok := xml.find_child_by_ident(char, "description") + description := desc.value if desc_ok else "" + + /* + For us to be interested in this codepoint, it has to have at least one entity. + */ + + nth := 0 + for { + character_entity, entity_ok := xml.find_child_by_ident(char, "entity", nth) + if !entity_ok { break } + + nth += 1 + if name, name_ok := xml.find_attribute_val_by_key(character_entity, "id"); name_ok { + + if len(name) == 0 { + /* + Invalid name. Skip. + */ + continue + } + + if name == "\"\"" { + printf("%#v\n", char) + printf("%#v\n", character_entity) + } + + if len(name) > max_name_length { longest_name = name } + if len(name) < min_name_length { shortest_name = name } + + min_name_length = min(min_name_length, len(name)) + max_name_length = max(max_name_length, len(name)) + + e := Entity{ + name = name, + codepoint = rune(codepoint), + description = description, + } + + if _, seen := entity_map[name]; seen { + continue + } + + entity_map[name] = e + append(&names, name) + count += 1 + } + } + } + } + + /* + Sort by name. + */ + slice.sort(names[:]) + + printf("Found %v unique `&name;` -> rune mappings.\n", count) + printf("Shortest name: %v (%v)\n", shortest_name, min_name_length) + printf("Longest name: %v (%v)\n", longest_name, max_name_length) + + // println(rune_to_string(1234)) + + /* + Generate table. + */ + wprintln(w, "package unicode_entity") + wprintln(w, "") + wprintln(w, GENERATED) + wprintln(w, "") + wprintf (w, TABLE_FILE_PROLOG) + wprintln(w, "") + + wprintf (w, "// `&%v;`\n", shortest_name) + wprintf (w, "XML_NAME_TO_RUNE_MIN_LENGTH :: %v\n", min_name_length) + wprintf (w, "// `&%v;`\n", longest_name) + wprintf (w, "XML_NAME_TO_RUNE_MAX_LENGTH :: %v\n", max_name_length) + wprintln(w, "") + + wprintln(w, +` +/* + Input: + entity_name - a string, like "copy" that describes a user-encoded Unicode entity as used in XML. + + Output: + "decoded" - The decoded rune if found by name, or -1 otherwise. + "ok" - true if found, false if not. + + IMPORTANT: XML processors (including browsers) treat these names as case-sensitive. So do we. +*/ +named_xml_entity_to_rune :: proc(name: string) -> (decoded: rune, ok: bool) { + /* + Early out if the name is too short or too long. + min as a precaution in case the generated table has a bogus value. + */ + if len(name) < min(1, XML_NAME_TO_RUNE_MIN_LENGTH) || len(name) > XML_NAME_TO_RUNE_MAX_LENGTH { + return -1, false + } + + switch rune(name[0]) { +`) + + prefix := '?' + should_close := false + + for v in names { + if rune(v[0]) != prefix { + if should_close { + wprintln(w, "\t\t}\n") + } + + prefix = rune(v[0]) + wprintf (w, "\tcase '%v':\n", prefix) + wprintln(w, "\t\tswitch name {") + } + + e := entity_map[v] + + wprintf(w, "\t\t\tcase \"%v\": \n", e.name) + wprintf(w, "\t\t\t\t// %v\n", e.description) + wprintf(w, "\t\t\t\treturn %v, true\n", rune_to_string(e.codepoint)) + + should_close = true + } + wprintln(w, "\t\t}") + wprintln(w, "\t}") + wprintln(w, "\treturn -1, false") + wprintln(w, "}\n") + wprintln(w, GENERATED) + + println() + println(strings.to_string(generated_buf)) + println() + + written := os.write_entire_file(generated_filename, transmute([]byte)strings.to_string(generated_buf)) + + if written { + fmt.printf("Successfully written generated \"%v\".", generated_filename) + } else { + fmt.printf("Failed to write generated \"%v\".", generated_filename) + } + + delete(entity_map) + delete(names) + for name in &names { + free(&name) + } +} + +GENERATED :: `/* + ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ +*/` + +TABLE_FILE_PROLOG :: `/* + This file is generated from "https://www.w3.org/2003/entities/2007xml/unicode.xml". + + UPDATE: + - Ensure the XML file was downloaded using "tests\core\download_assets.py". + - Run "core/unicode/tools/generate_entity_table.odin" + + Odin unicode generated tables: https://github.com/odin-lang/Odin/tree/master/core/encoding/entity + + Copyright © 2021 World Wide Web Consortium, (Massachusetts Institute of Technology, + European Research Consortium for Informatics and Mathematics, Keio University, Beihang). + + All Rights Reserved. + + This work is distributed under the W3C® Software License [1] in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + [1] http://www.w3.org/Consortium/Legal/copyright-software + + See also: LICENSE_table.md +*/ +` + +rune_to_string :: proc(r: rune) -> (res: string) { + res = fmt.tprintf("%08x", int(r)) + for len(res) > 2 && res[:2] == "00" { + res = res[2:] + } + return fmt.tprintf("rune(0x%v)", res) +} + +is_dotted_name :: proc(name: string) -> (dotted: bool) { + for r in name { + if r == '.' { return true} + } + return false +} + +main :: proc() { + using fmt + + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + generate_encoding_entity_table() + + if len(track.allocation_map) > 0 { + println() + for _, v in track.allocation_map { + printf("%v Leaked %v bytes.\n", v.location, v.size) + } + } + println("Done and cleaned up!") +} \ No newline at end of file From 3d72e80ccf0f382f03a1c9407c4728862c5bca91 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 2 Dec 2021 21:07:40 +0100 Subject: [PATCH 11/79] [xml] Implement optional unboxing of CDATA and decoding of tag values. --- core/encoding/entity/entity.odin | 39 ++++++----- .../entity/example/entity_example.odin | 67 +++---------------- core/encoding/entity/example/test.html | 2 + core/encoding/xml/xml_reader.odin | 41 ++++++------ 4 files changed, 56 insertions(+), 93 deletions(-) diff --git a/core/encoding/entity/entity.odin b/core/encoding/entity/entity.odin index e40896819..8742446e6 100644 --- a/core/encoding/entity/entity.odin +++ b/core/encoding/entity/entity.odin @@ -60,16 +60,22 @@ COMMENT_END :: "-->" Default: CDATA and comments are passed through unchanged. */ XML_Decode_Option :: enum u8 { + /* + Do not decode & entities. It decodes by default. + If given, overrides `Decode_CDATA`. + */ + No_Entity_Decode, + /* CDATA is unboxed. */ - CDATA_Unbox, + Unbox_CDATA, /* Unboxed CDATA is decoded as well. - Ignored if `.CDATA_Unbox` is not given. + Ignored if `.Unbox_CDATA` is not given. */ - CDATA_Decode, + Decode_CDATA, /* Comments are stripped. @@ -129,7 +135,7 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator := } case: - if in_data && .CDATA_Decode not_in options { + if in_data && .Decode_CDATA not_in options { /* Unboxed, but undecoded. */ @@ -145,17 +151,20 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator := */ write_string(&builder, entity) } else { - if decoded, ok := xml_decode_entity(entity); ok { - write_rune(&builder, decoded) - } else { - /* - Decode failed. Pass through original. - */ - write_string(&builder, "&") - write_string(&builder, entity) - write_string(&builder, ";") + + if .No_Entity_Decode not_in options { + if decoded, ok := xml_decode_entity(entity); ok { + write_rune(&builder, decoded) + continue + } } + /* + Literal passthrough because the decode failed or we want entities not decoded. + */ + write_string(&builder, "&") + write_string(&builder, entity) + write_string(&builder, ";") } } else { write_rune(&builder, t.r) @@ -290,7 +299,7 @@ _handle_xml_special :: proc(t: ^Tokenizer, builder: ^strings.Builder, options: X if string(t.src[t.offset:][:len(CDATA_START)]) == CDATA_START { t.read_offset += len(CDATA_START) - 1 - if .CDATA_Unbox in options && .CDATA_Decode in options { + if .Unbox_CDATA in options && .Decode_CDATA in options { /* We're unboxing _and_ decoding CDATA */ @@ -315,7 +324,7 @@ _handle_xml_special :: proc(t: ^Tokenizer, builder: ^strings.Builder, options: X cdata := string(t.src[offset : t.read_offset]) - if .CDATA_Unbox in options { + if .Unbox_CDATA in options { cdata = cdata[len(CDATA_START):] cdata = cdata[:len(cdata) - len(CDATA_END)] } diff --git a/core/encoding/entity/example/entity_example.odin b/core/encoding/entity/example/entity_example.odin index 8758d9ad9..161a44827 100644 --- a/core/encoding/entity/example/entity_example.odin +++ b/core/encoding/entity/example/entity_example.odin @@ -1,19 +1,11 @@ package unicode_entity_example import "core:encoding/xml" -import "core:encoding/entity" import "core:strings" import "core:mem" import "core:fmt" import "core:time" -OPTIONS :: xml.Options{ - flags = { - .Ignore_Unsupported, .Intern_Comments, - }, - expected_doctype = "", -} - doc_print :: proc(doc: ^xml.Document) { buf: strings.Builder defer strings.destroy_builder(&buf) @@ -29,6 +21,13 @@ _entities :: proc() { DOC :: #load("../../../../tests/core/assets/XML/unicode.xml") + OPTIONS :: xml.Options{ + flags = { + .Ignore_Unsupported, .Intern_Comments, + }, + expected_doctype = "", + } + parse_duration: time.Duration { @@ -50,57 +49,11 @@ _entities :: proc() { _main :: proc() { using fmt - doc, err := xml.parse(#load("test.html")) + options := xml.Options{ flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities }} + doc, _ := xml.parse(#load("test.html"), options) + defer xml.destroy(doc) doc_print(doc) - - if false { - val := doc.root.children[1].children[2].value - - println() - replaced, ok := entity.decode_xml(val) - defer delete(replaced) - - printf("Before: '%v', Err: %v\n", val, err) - printf("Passthrough: '%v'\nOK: %v\n", replaced, ok) - println() - } - - if false { - val := doc.root.children[1].children[2].value - - println() - replaced, ok := entity.decode_xml(val, { .CDATA_Unbox }) - defer delete(replaced) - - printf("Before: '%v', Err: %v\n", val, err) - printf("CDATA_Unbox: '%v'\nOK: %v\n", replaced, ok) - println() - } - - if true { - val := doc.root.children[1].children[2].value - - println() - replaced, ok := entity.decode_xml(val, { .CDATA_Unbox, .CDATA_Decode }) - defer delete(replaced) - - printf("Before: '%v', Err: %v\n", val, err) - printf("CDATA_Decode: '%v'\nOK: %v\n", replaced, ok) - println() - } - - if true { - val := doc.root.children[1].children[1].value - - println() - replaced, ok := entity.decode_xml(val, { .Comment_Strip }) - defer delete(replaced) - - printf("Before: '%v', Err: %v\n", val, err) - printf("Comment_Strip: '%v'\nOK: %v\n", replaced, ok) - println() - } } main :: proc() { diff --git a/core/encoding/entity/example/test.html b/core/encoding/entity/example/test.html index 60e32bf03..62a0bb35a 100644 --- a/core/encoding/entity/example/test.html +++ b/core/encoding/entity/example/test.html @@ -16,9 +16,11 @@
Foozle]! © 42&;1234&
+
Foozle]! © 42&;1234&
+
| | | fj ` \ ® ϱ ∳
diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 146c278cb..6f49b8e08 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -18,10 +18,6 @@ package xml - We do NOT support UTF-16. If you have a UTF-16 XML file, please convert it to UTF-8 first. Also, our condolences. - <[!ELEMENT and <[!ATTLIST are not supported, and will be either ignored or return an error depending on the parser options. - TODO: - - Optional CDATA unboxing. - - Optional `>`, ` `, ` ` and other escape substitution in tag bodies. - MAYBE: - XML writer? - Serialize/deserialize Odin types? @@ -31,6 +27,7 @@ package xml */ import "core:strings" +import "core:encoding/entity" import "core:mem" import "core:os" @@ -196,12 +193,6 @@ Error :: enum { Duplicate_Attribute, Conflicting_Options, - - /* - Unhandled TODO: - */ - Unhandled_CDATA_Unboxing, - Unhandled_SGML_Entity_Decoding, } /* @@ -422,8 +413,25 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err /* This should be a tag's body text. */ - body_text := scan_string(t, t.offset) or_return - element.value = strings.intern_get(&doc.intern, body_text) + body_text := scan_string(t, t.offset) or_return + + decode_opts := entity.XML_Decode_Options{ .Comment_Strip } + + if .Decode_SGML_Entities not_in opts.flags { + decode_opts += { .No_Entity_Decode } + } + if .Unbox_CDATA in opts.flags { + decode_opts += { .Unbox_CDATA, .Decode_CDATA } + } + + decoded, decode_err := entity.decode_xml(body_text, decode_opts) + defer delete(decoded) + + if decode_err == .None { + element.value = strings.intern_get(&doc.intern, decoded) + } else { + element.value = strings.intern_get(&doc.intern, body_text) + } } } @@ -488,15 +496,6 @@ validate_options :: proc(options: Options) -> (validated: Options, err: Error) { if .Error_on_Unsupported in validated.flags && .Ignore_Unsupported in validated.flags { return options, .Conflicting_Options } - - if .Unbox_CDATA in validated.flags { - return options, .Unhandled_CDATA_Unboxing - } - - if .Decode_SGML_Entities in validated.flags { - return options, .Unhandled_SGML_Entity_Decoding - } - return validated, .None } From d65d6edb0e1887871c4de6a4e8a1630927153eae Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 5 Dec 2021 02:17:48 +0100 Subject: [PATCH 12/79] [xml] Improve XML tests, test `core:encoding/entity`. --- core/encoding/entity/entity.odin | 7 + .../entity/example/entity_example.odin | 1 + core/encoding/entity/example/test.html | 2 +- tests/core/assets/XML/entities.html | 29 ++ ...-xliff-1.0.xliff => nl_NL-xliff-1.2.xliff} | 0 tests/core/assets/XML/utf8.xml | 2 +- tests/core/encoding/xml/test_core_xml.odin | 426 +++++++++++------- 7 files changed, 291 insertions(+), 176 deletions(-) create mode 100644 tests/core/assets/XML/entities.html rename tests/core/assets/XML/{nl_NL-xliff-1.0.xliff => nl_NL-xliff-1.2.xliff} (100%) diff --git a/core/encoding/entity/entity.odin b/core/encoding/entity/entity.odin index 8742446e6..db1a5ad0b 100644 --- a/core/encoding/entity/entity.odin +++ b/core/encoding/entity/entity.odin @@ -115,7 +115,14 @@ decode_xml :: proc(input: string, options := XML_Decode_Options{}, allocator := We don't need to check if we need to write a `<`, because if it isn't CDATA or a comment, it couldn't have been part of an XML tag body to be decoded here. + + Keep in mind that we could already *be* inside a CDATA tag. + If so, write `>` as a literal and continue. */ + if in_data { + write_rune(&builder, '<') + continue + } in_data = _handle_xml_special(&t, &builder, options) or_return case ']': diff --git a/core/encoding/entity/example/entity_example.odin b/core/encoding/entity/example/entity_example.odin index 161a44827..882203f48 100644 --- a/core/encoding/entity/example/entity_example.odin +++ b/core/encoding/entity/example/entity_example.odin @@ -50,6 +50,7 @@ _main :: proc() { using fmt options := xml.Options{ flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities }} + doc, _ := xml.parse(#load("test.html"), options) defer xml.destroy(doc) diff --git a/core/encoding/entity/example/test.html b/core/encoding/entity/example/test.html index 62a0bb35a..ebbc6470c 100644 --- a/core/encoding/entity/example/test.html +++ b/core/encoding/entity/example/test.html @@ -22,7 +22,7 @@
- | | | fj ` \ ® ϱ ∳ + | | | fj ` \ ® ϱ ∳ ⁏
\ No newline at end of file diff --git a/tests/core/assets/XML/entities.html b/tests/core/assets/XML/entities.html new file mode 100644 index 000000000..05a6b107e --- /dev/null +++ b/tests/core/assets/XML/entities.html @@ -0,0 +1,29 @@ + + + Entity Reference Test + + + +

Entity Reference Test

+
+ Foozle]! © 42&;1234& +
+ + +
+ Foozle]! © 42&;1234& +
+ +
+ | | | fj ` \ ® ϱ ∳ ⁏ +
+ + \ No newline at end of file diff --git a/tests/core/assets/XML/nl_NL-xliff-1.0.xliff b/tests/core/assets/XML/nl_NL-xliff-1.2.xliff similarity index 100% rename from tests/core/assets/XML/nl_NL-xliff-1.0.xliff rename to tests/core/assets/XML/nl_NL-xliff-1.2.xliff diff --git a/tests/core/assets/XML/utf8.xml b/tests/core/assets/XML/utf8.xml index c9ed3bf69..6e1a897ea 100644 --- a/tests/core/assets/XML/utf8.xml +++ b/tests/core/assets/XML/utf8.xml @@ -4,5 +4,5 @@ <부끄러운:barzle> ရှက်စရာ ဇီးကွက် Owl of Shame - More CDATA Hello, world! Nonsense. + More CDATA Hello, world! Nonsense. \ No newline at end of file diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin index c2e0aa172..5cb59e001 100644 --- a/tests/core/encoding/xml/test_core_xml.odin +++ b/tests/core/encoding/xml/test_core_xml.odin @@ -3,16 +3,16 @@ package test_core_xml import "core:encoding/xml" import "core:testing" import "core:mem" +import "core:strings" +import "core:io" import "core:fmt" +import "core:hash" Silent :: proc(pos: xml.Pos, fmt: string, args: ..any) { // Custom (silent) error handler. } -OPTIONS :: xml.Options{ - flags = { - .Ignore_Unsupported, .Intern_Comments, - }, +OPTIONS :: xml.Options{ flags = { .Ignore_Unsupported, .Intern_Comments, }, expected_doctype = "", } @@ -22,76 +22,153 @@ TEST_fail := 0 TEST :: struct { filename: string, options: xml.Options, - expected: struct { - error: xml.Error, - xml_version: string, - xml_encoding: string, - doctype: string, - }, + err: xml.Error, + crc32: u32, } +/* + Relative to ODIN_ROOT +*/ +TEST_FILE_PATH_PREFIX :: "tests/core/assets/XML" + TESTS :: []TEST{ /* First we test that certain files parse without error. */ + { - filename = "assets/XML/utf8.xml", - options = OPTIONS, - expected = { - error = .None, - xml_version = "1.0", - xml_encoding = "utf-8", - doctype = "恥ずべきフクロウ", + /* + + + <恥ずべきフクロウ 올빼미_id="Foozle Hello, world!"]]>Barzle"> + <부끄러운:barzle> + ရှက်စရာ ဇီးကွက် + Owl of Shame + More CDATA Hello, world! Nonsense. + + */ + + /* + Tests UTF-8 idents and values. + Test namespaced ident. + Tests that nested partial CDATA start doesn't trip up parser. + */ + filename = "utf8.xml", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, + }, + expected_doctype = "恥ずべきフクロウ", }, + crc32 = 0x30d82264, }, + { - filename = "assets/XML/nl_NL-qt-ts.ts", - options = OPTIONS, - expected = { - error = .None, - xml_version = "1.0", - xml_encoding = "utf-8", - doctype = "TS", + /* + Same as above. + Unbox CDATA in data tag. + */ + filename = "utf8.xml", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, + }, + expected_doctype = "恥ずべきフクロウ", }, + crc32 = 0x6d38ac58, }, + { - filename = "assets/XML/nl_NL-xliff-1.0.xliff", - options = OPTIONS, - expected = { - error = .None, - xml_version = "1.0", - xml_encoding = "UTF-8", - doctype = "", + /* + Simple Qt TS translation file. + `core:i18n` requires it to be parsed properly. + */ + filename = "nl_NL-qt-ts.ts", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities, + }, + expected_doctype = "TS", }, + crc32 = 0x7bce2630, }, + { - filename = "assets/XML/nl_NL-xliff-2.0.xliff", - options = OPTIONS, - expected = { - error = .None, - xml_version = "1.0", - xml_encoding = "utf-8", - doctype = "", + /* + Simple XLiff 1.2 file. + `core:i18n` requires it to be parsed properly. + */ + filename = "nl_NL-xliff-1.2.xliff", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities, + }, + expected_doctype = "xliff", }, + crc32 = 0x43f19d61, + }, + + { + /* + Simple XLiff 2.0 file. + `core:i18n` requires it to be parsed properly. + */ + filename = "nl_NL-xliff-2.0.xliff", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities, + }, + expected_doctype = "xliff", + }, + crc32 = 0x961e7635, + }, + + { + filename = "entities.html", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, + }, + expected_doctype = "html", + }, + crc32 = 0xdb4a1e79, + }, + + { + filename = "entities.html", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, + }, + expected_doctype = "html", + }, + crc32 = 0x82588917, + }, + + { + filename = "entities.html", + options = { + flags = { + .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities, + }, + expected_doctype = "html", + }, + crc32 = 0x5e74d8a6, }, /* Then we test that certain errors are returned as expected. */ { - filename = "assets/XML/utf8.xml", + filename = "utf8.xml", options = { flags = { .Ignore_Unsupported, .Intern_Comments, }, expected_doctype = "Odin", }, - expected = { - error = .Invalid_DocType, - xml_version = "1.0", - xml_encoding = "utf-8", - doctype = "恥ずべきフクロウ", - }, + err = .Invalid_DocType, + crc32 = 0x49b83d0a, }, } @@ -115,6 +192,136 @@ when ODIN_TEST { } } +test_file_path :: proc(filename: string) -> (path: string) { + + path = fmt.tprintf("%v%v/%v", ODIN_ROOT, TEST_FILE_PATH_PREFIX, filename) + temp := transmute([]u8)path + + for r, i in path { + if r == '\\' { + temp[i] = '/' + } + } + return path +} + +doc_to_string :: proc(doc: ^xml.Document) -> (result: string) { + /* + Effectively a clone of the debug printer in the xml package. + We duplicate it here so that the way it prints an XML document to a string is stable. + + This way we can hash the output. If it changes, it means that the document or how it was parsed changed, + not how it was printed. One less source of variability. + */ + print :: proc(writer: io.Writer, doc: ^xml.Document) -> (written: int, err: io.Error) { + if doc == nil { return } + using fmt + + written += wprintf(writer, "[XML Prolog]\n") + + for attr in doc.prolog { + written += wprintf(writer, "\t%v: %v\n", attr.key, attr.val) + } + + written += wprintf(writer, "[Encoding] %v\n", doc.encoding) + + if len(doc.doctype.ident) > 0 { + written += wprintf(writer, "[DOCTYPE] %v\n", doc.doctype.ident) + + if len(doc.doctype.rest) > 0 { + wprintf(writer, "\t%v\n", doc.doctype.rest) + } + } + + for comment in doc.comments { + written += wprintf(writer, "[Pre-root comment] %v\n", comment) + } + + if doc.root != nil { + wprintln(writer, " --- ") + print_element(writer, doc.root) + wprintln(writer, " --- ") + } + + return written, .None + } + + print_element :: proc(writer: io.Writer, element: ^xml.Element, indent := 0) -> (written: int, err: io.Error) { + if element == nil { return } + using fmt + + tab :: proc(writer: io.Writer, indent: int) { + for _ in 0..=indent { + wprintf(writer, "\t") + } + } + + tab(writer, indent) + + if element.kind == .Element { + wprintf(writer, "<%v>\n", element.ident) + if len(element.value) > 0 { + tab(writer, indent + 1) + wprintf(writer, "[Value] %v\n", element.value) + } + + for attr in element.attribs { + tab(writer, indent + 1) + wprintf(writer, "[Attr] %v: %v\n", attr.key, attr.val) + } + + for child in element.children { + print_element(writer, child, indent + 1) + } + } else if element.kind == .Comment { + wprintf(writer, "[COMMENT] %v\n", element.value) + } + + return written, .None + } + + buf: strings.Builder + defer strings.destroy_builder(&buf) + + print(strings.to_writer(&buf), doc) + return strings.clone(strings.to_string(buf)) +} + +@test +run_tests :: proc(t: ^testing.T) { + using fmt + + for test in TESTS { + path := test_file_path(test.filename) + printf("\nTrying to parse %v\n\n", path) + + doc, err := xml.parse(path, test.options, Silent) + defer xml.destroy(doc) + + tree_string := doc_to_string(doc) + tree_bytes := transmute([]u8)tree_string + defer delete(tree_bytes) + + crc32 := hash.crc32(tree_bytes) + + failed := err != test.err + err_msg := tprintf("Expected return value %v, got %v", test.err, err) + expect(t, err == test.err, err_msg) + + failed |= crc32 != test.crc32 + err_msg = tprintf("Expected CRC 0x%08x, got 0x%08x", test.crc32, crc32) + expect(t, crc32 == test.crc32, err_msg) + + if failed { + /* + Don't fully print big trees. + */ + tree_string = tree_string[:min(2_048, len(tree_string))] + println(tree_string) + } + } +} + main :: proc() { t := testing.T{} @@ -132,133 +339,4 @@ main :: proc() { } fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) -} - -@test -run_tests :: proc(t: ^testing.T) { - using fmt - - count := 0 - - for test in TESTS { - printf("Trying to parse %v\n\n", test.filename) - - doc, err := xml.parse(test.filename, test.options, Silent) - defer xml.destroy(doc) - - err_msg := tprintf("Expected return value %v, got %v", test.expected.error, err) - expect(t, err == test.expected.error, err_msg) - - if len(test.expected.xml_version) > 0 { - xml_version := "" - for attr in doc.prolog { - if attr.key == "version" { - xml_version = attr.val - } - } - - err_msg = tprintf("Expected XML version %v, got %v", test.expected.xml_version, xml_version) - expect(t, xml_version == test.expected.xml_version, err_msg) - } - - if len(test.expected.xml_encoding) > 0 { - xml_encoding := "" - for attr in doc.prolog { - if attr.key == "encoding" { - xml_encoding = attr.val - } - } - - err_msg = tprintf("Expected XML encoding %v, got %v", test.expected.xml_encoding, xml_encoding) - expect(t, xml_encoding == test.expected.xml_encoding, err_msg) - } - - err_msg = tprintf("Expected DOCTYPE %v, got %v", test.expected.doctype, doc.doctype.ident) - expect(t, doc.doctype.ident == test.expected.doctype, err_msg) - - /* - File-specific tests. - */ - switch count { - case 0: - expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.") - attr := doc.root.attribs[0] - - attr_key_expected := "올빼미_id" - attr_val_expected := "Foozle Hello, world!\"]]>Barzle" - - attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) - expect(t, attr.key == attr_key_expected, attr_err) - - attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) - expect(t, attr.val == attr_val_expected, attr_err) - - expect(t, len(doc.root.children) > 0, "Expected the root tag to have children.") - child := doc.root.children[0] - - first_child_ident := "부끄러운:barzle" - attr_err = tprintf("Expected first child tag's ident to be %v, got %v", first_child_ident, child.ident) - expect(t, child.ident == first_child_ident, attr_err) - - case 2: - expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.") - - { - attr := doc.root.attribs[0] - - attr_key_expected := "version" - attr_val_expected := "1.2" - - attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) - expect(t, attr.key == attr_key_expected, attr_err) - - attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) - expect(t, attr.val == attr_val_expected, attr_err) - } - - { - attr := doc.root.attribs[1] - - attr_key_expected := "xmlns" - attr_val_expected := "urn:oasis:names:tc:xliff:document:1.2" - - attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) - expect(t, attr.key == attr_key_expected, attr_err) - - attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) - expect(t, attr.val == attr_val_expected, attr_err) - } - - case 3: - expect(t, len(doc.root.attribs) > 0, "Expected the root tag to have an attribute.") - - { - attr := doc.root.attribs[0] - - attr_key_expected := "xmlns" - attr_val_expected := "urn:oasis:names:tc:xliff:document:2.0" - - attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) - expect(t, attr.key == attr_key_expected, attr_err) - - attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) - expect(t, attr.val == attr_val_expected, attr_err) - } - - { - attr := doc.root.attribs[1] - - attr_key_expected := "version" - attr_val_expected := "2.0" - - attr_err := tprintf("Expected %v, got %v", attr_key_expected, attr.key) - expect(t, attr.key == attr_key_expected, attr_err) - - attr_err = tprintf("Expected %v, got %v", attr_val_expected, attr.val) - expect(t, attr.val == attr_val_expected, attr_err) - } - } - - count += 1 - } } \ No newline at end of file From d7200f61441b6acfc4f0b47e900095f08490da58 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 5 Dec 2021 02:40:13 +0100 Subject: [PATCH 13/79] Improve tests in general. Less spammy core tests: They don't print PASSes now, only logs and failures. `core:image` and `core:encoding/xml` tests also find their assets relative to `ODIN_ROOT` now. --- tests/core/compress/test_core_compress.odin | 9 +- tests/core/crypto/test_core_crypto.odin | 1691 +++++++++--------- tests/core/encoding/json/test_core_json.odin | 9 +- tests/core/encoding/xml/test_core_xml.odin | 43 +- tests/core/hash/test_core_hash.odin | 9 +- tests/core/image/test_core_image.odin | 51 +- tests/core/odin/test_parser.odin | 37 +- tests/core/strings/test_core_strings.odin | 59 +- 8 files changed, 948 insertions(+), 960 deletions(-) diff --git a/tests/core/compress/test_core_compress.odin b/tests/core/compress/test_core_compress.odin index c925c0258..908ef12e4 100644 --- a/tests/core/compress/test_core_compress.odin +++ b/tests/core/compress/test_core_compress.odin @@ -30,18 +30,15 @@ when ODIN_TEST { log :: testing.log } else { expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) TEST_count += 1 if !condition { TEST_fail += 1 - fmt.println(message) + fmt.printf("[%v] %v\n", loc, message) return } - fmt.println(" PASS") } log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) + fmt.printf("[%v] LOG:\n\t%v\n", loc, v) } } @@ -51,7 +48,7 @@ main :: proc() { zlib_test(&t) gzip_test(&t) - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("\n%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) } @test diff --git a/tests/core/crypto/test_core_crypto.odin b/tests/core/crypto/test_core_crypto.odin index 2ad00be66..6d3a9f8e4 100644 --- a/tests/core/crypto/test_core_crypto.odin +++ b/tests/core/crypto/test_core_crypto.odin @@ -1,15 +1,15 @@ package test_core_crypto /* - Copyright 2021 zhibog - Made available under the BSD-3 license. + Copyright 2021 zhibog + Made available under the BSD-3 license. - List of contributors: - zhibog, dotbmp: Initial implementation. - Jeroen van Rijn: Test runner setup. + List of contributors: + zhibog, dotbmp: Initial implementation. + Jeroen van Rijn: Test runner setup. - Tests for the hashing algorithms within the crypto library. - Where possible, the official test vectors are used to validate the implementation. + Tests for the hashing algorithms within the crypto library. + Where possible, the official test vectors are used to validate the implementation. */ import "core:testing" @@ -41,1065 +41,1062 @@ TEST_count := 0 TEST_fail := 0 when ODIN_TEST { - expect :: testing.expect - log :: testing.log + expect :: testing.expect + log :: testing.log } else { - expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) - TEST_count += 1 - if !condition { - TEST_fail += 1 - fmt.println(message) - return - } - fmt.println(" PASS") - } - log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) - } + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] LOG:\n\t%v\n", loc, v) + } } main :: proc() { - t := testing.T{} - test_md2(&t) - test_md4(&t) - test_md5(&t) - test_sha1(&t) - test_sha224(&t) - test_sha256(&t) - test_sha384(&t) - test_sha512(&t) - test_sha3_224(&t) - test_sha3_256(&t) - test_sha3_384(&t) - test_sha3_512(&t) - test_shake_128(&t) - test_shake_256(&t) - test_keccak_224(&t) - test_keccak_256(&t) - test_keccak_384(&t) - test_keccak_512(&t) - test_whirlpool(&t) - test_gost(&t) - test_streebog_256(&t) - test_streebog_512(&t) - test_blake_224(&t) - test_blake_256(&t) - test_blake_384(&t) - test_blake_512(&t) - test_blake2b(&t) - test_blake2s(&t) - test_ripemd_128(&t) - test_ripemd_160(&t) - test_ripemd_256(&t) - test_ripemd_320(&t) - test_tiger_128(&t) - test_tiger_160(&t) - test_tiger_192(&t) - test_tiger2_128(&t) - test_tiger2_160(&t) - test_tiger2_192(&t) - test_sm3(&t) - test_jh_224(&t) - test_jh_256(&t) - test_jh_384(&t) - test_jh_512(&t) - test_groestl_224(&t) - test_groestl_256(&t) - test_groestl_384(&t) - test_groestl_512(&t) - test_haval_128(&t) - test_haval_160(&t) - test_haval_192(&t) - test_haval_224(&t) - test_haval_256(&t) + t := testing.T{} + test_md2(&t) + test_md4(&t) + test_md5(&t) + test_sha1(&t) + test_sha224(&t) + test_sha256(&t) + test_sha384(&t) + test_sha512(&t) + test_sha3_224(&t) + test_sha3_256(&t) + test_sha3_384(&t) + test_sha3_512(&t) + test_shake_128(&t) + test_shake_256(&t) + test_keccak_224(&t) + test_keccak_256(&t) + test_keccak_384(&t) + test_keccak_512(&t) + test_whirlpool(&t) + test_gost(&t) + test_streebog_256(&t) + test_streebog_512(&t) + test_blake_224(&t) + test_blake_256(&t) + test_blake_384(&t) + test_blake_512(&t) + test_blake2b(&t) + test_blake2s(&t) + test_ripemd_128(&t) + test_ripemd_160(&t) + test_ripemd_256(&t) + test_ripemd_320(&t) + test_tiger_128(&t) + test_tiger_160(&t) + test_tiger_192(&t) + test_tiger2_128(&t) + test_tiger2_160(&t) + test_tiger2_192(&t) + test_sm3(&t) + test_jh_224(&t) + test_jh_256(&t) + test_jh_384(&t) + test_jh_512(&t) + test_groestl_224(&t) + test_groestl_256(&t) + test_groestl_384(&t) + test_groestl_512(&t) + test_haval_128(&t) + test_haval_160(&t) + test_haval_192(&t) + test_haval_224(&t) + test_haval_256(&t) - // "modern" crypto tests - test_chacha20(&t) - test_poly1305(&t) - test_chacha20poly1305(&t) - test_x25519(&t) - test_rand_bytes(&t) + // "modern" crypto tests + test_chacha20(&t) + test_poly1305(&t) + test_chacha20poly1305(&t) + test_x25519(&t) + test_rand_bytes(&t) - bench_modern(&t) + bench_modern(&t) - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("\n%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) } TestHash :: struct { - hash: string, - str: string, + hash: string, + str: string, } hex_string :: proc(bytes: []byte, allocator := context.temp_allocator) -> string { - lut: [16]byte = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'} - buf := make([]byte, len(bytes) * 2, allocator) - for i := 0; i < len(bytes); i += 1 { - buf[i * 2 + 0] = lut[bytes[i] >> 4 & 0xf] - buf[i * 2 + 1] = lut[bytes[i] & 0xf] - } - return string(buf) + lut: [16]byte = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'} + buf := make([]byte, len(bytes) * 2, allocator) + for i := 0; i < len(bytes); i += 1 { + buf[i * 2 + 0] = lut[bytes[i] >> 4 & 0xf] + buf[i * 2 + 1] = lut[bytes[i] & 0xf] + } + return string(buf) } @(test) test_md2 :: proc(t: ^testing.T) { - // Official test vectors from https://datatracker.ietf.org/doc/html/rfc1319 - test_vectors := [?]TestHash { - TestHash{"8350e5a3e24c153df2275c9f80692773", ""}, - TestHash{"32ec01ec4a6dac72c0ab96fb34c0b5d1", "a"}, - TestHash{"da853b0d3f88d99b30283a69e6ded6bb", "abc"}, - TestHash{"ab4f496bfb2a530b219ff33031fe06b0", "message digest"}, - TestHash{"4e8ddff3650292ab5a4108c3aa47940b", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"da33def2a42df13975352846c30338cd", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - TestHash{"d5976f79d83d3a0dc9806c3c66f3efd8", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := md2.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Official test vectors from https://datatracker.ietf.org/doc/html/rfc1319 + test_vectors := [?]TestHash { + TestHash{"8350e5a3e24c153df2275c9f80692773", ""}, + TestHash{"32ec01ec4a6dac72c0ab96fb34c0b5d1", "a"}, + TestHash{"da853b0d3f88d99b30283a69e6ded6bb", "abc"}, + TestHash{"ab4f496bfb2a530b219ff33031fe06b0", "message digest"}, + TestHash{"4e8ddff3650292ab5a4108c3aa47940b", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"da33def2a42df13975352846c30338cd", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + TestHash{"d5976f79d83d3a0dc9806c3c66f3efd8", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := md2.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_md4 :: proc(t: ^testing.T) { - // Official test vectors from https://datatracker.ietf.org/doc/html/rfc1320 - test_vectors := [?]TestHash { - TestHash{"31d6cfe0d16ae931b73c59d7e0c089c0", ""}, - TestHash{"bde52cb31de33e46245e05fbdbd6fb24", "a"}, - TestHash{"a448017aaf21d8525fc10ae87aa6729d", "abc"}, - TestHash{"d9130a8164549fe818874806e1c7014b", "message digest"}, - TestHash{"d79e1c308aa5bbcdeea8ed63df412da9", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"043f8582f241db351ce627e153e7f0e4", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - TestHash{"e33b4ddc9c38f2199c3e7b164fcc0536", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := md4.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Official test vectors from https://datatracker.ietf.org/doc/html/rfc1320 + test_vectors := [?]TestHash { + TestHash{"31d6cfe0d16ae931b73c59d7e0c089c0", ""}, + TestHash{"bde52cb31de33e46245e05fbdbd6fb24", "a"}, + TestHash{"a448017aaf21d8525fc10ae87aa6729d", "abc"}, + TestHash{"d9130a8164549fe818874806e1c7014b", "message digest"}, + TestHash{"d79e1c308aa5bbcdeea8ed63df412da9", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"043f8582f241db351ce627e153e7f0e4", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + TestHash{"e33b4ddc9c38f2199c3e7b164fcc0536", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := md4.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_md5 :: proc(t: ^testing.T) { - // Official test vectors from https://datatracker.ietf.org/doc/html/rfc1321 - test_vectors := [?]TestHash { - TestHash{"d41d8cd98f00b204e9800998ecf8427e", ""}, - TestHash{"0cc175b9c0f1b6a831c399e269772661", "a"}, - TestHash{"900150983cd24fb0d6963f7d28e17f72", "abc"}, - TestHash{"f96b697d7cb7938d525a2f31aaf161d0", "message digest"}, - TestHash{"c3fcd3d76192e4007dfb496cca67e13b", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"d174ab98d277d9f5a5611c2c9f419d9f", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - TestHash{"57edf4a22be3c955ac49da2e2107b67a", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := md5.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Official test vectors from https://datatracker.ietf.org/doc/html/rfc1321 + test_vectors := [?]TestHash { + TestHash{"d41d8cd98f00b204e9800998ecf8427e", ""}, + TestHash{"0cc175b9c0f1b6a831c399e269772661", "a"}, + TestHash{"900150983cd24fb0d6963f7d28e17f72", "abc"}, + TestHash{"f96b697d7cb7938d525a2f31aaf161d0", "message digest"}, + TestHash{"c3fcd3d76192e4007dfb496cca67e13b", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"d174ab98d277d9f5a5611c2c9f419d9f", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + TestHash{"57edf4a22be3c955ac49da2e2107b67a", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := md5.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha1 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""}, - TestHash{"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"}, - TestHash{"f9537c23893d2014f365adf8ffe33b8eb0297ed1", "abcdbcdecdefdefgefghfghighijhi"}, - TestHash{"346fb528a24b48f563cb061470bcfd23740427ad", "jkijkljklmklmnlmnomnopnopq"}, - TestHash{"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"}, - TestHash{"c729c8996ee0a6f74f4f3248e8957edf704fb624", "01234567012345670123456701234567"}, - TestHash{"84983e441c3bd26ebaae4aa1f95129e5e54670f1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"a49b2446a02c645bf419f995b67091253a04a259", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha1.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""}, + TestHash{"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"}, + TestHash{"f9537c23893d2014f365adf8ffe33b8eb0297ed1", "abcdbcdecdefdefgefghfghighijhi"}, + TestHash{"346fb528a24b48f563cb061470bcfd23740427ad", "jkijkljklmklmnlmnomnopnopq"}, + TestHash{"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"}, + TestHash{"c729c8996ee0a6f74f4f3248e8957edf704fb624", "01234567012345670123456701234567"}, + TestHash{"84983e441c3bd26ebaae4aa1f95129e5e54670f1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"a49b2446a02c645bf419f995b67091253a04a259", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha1.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha224 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f", ""}, - TestHash{"23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7", "abc"}, - TestHash{"75388b16512776cc5dba5da1fd890150b0c6455cb4f58b1952522525", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"c97ca9a559850ce97a04a96def6d99a9e0e0e2ab14e6b8df265fc0b3", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha2.hash_224(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"d14a028c2a3a2bc9476102bb288234c415a2b01f828ea62ac5b3e42f", ""}, + TestHash{"23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7", "abc"}, + TestHash{"75388b16512776cc5dba5da1fd890150b0c6455cb4f58b1952522525", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"c97ca9a559850ce97a04a96def6d99a9e0e0e2ab14e6b8df265fc0b3", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha2.hash_224(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha256 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ""}, - TestHash{"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc"}, - TestHash{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"cf5b16a778af8380036ce59e7b0492370b249b11e8f07a51afac45037afee9d1", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha2.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ""}, + TestHash{"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc"}, + TestHash{"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"cf5b16a778af8380036ce59e7b0492370b249b11e8f07a51afac45037afee9d1", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha2.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha384 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b", ""}, - TestHash{"cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7", "abc"}, - TestHash{"3391fdddfc8dc7393707a65b1b4709397cf8b1d162af05abfe8f450de5f36bc6b0455a8520bc4e6f5fe95b1fe3c8452b", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"09330c33f71147e83d192fc782cd1b4753111b173b3b05d22fa08086e3b0f712fcc7c71a557e2db966c3e9fa91746039", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha2.hash_384(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b", ""}, + TestHash{"cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7", "abc"}, + TestHash{"3391fdddfc8dc7393707a65b1b4709397cf8b1d162af05abfe8f450de5f36bc6b0455a8520bc4e6f5fe95b1fe3c8452b", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"09330c33f71147e83d192fc782cd1b4753111b173b3b05d22fa08086e3b0f712fcc7c71a557e2db966c3e9fa91746039", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha2.hash_384(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha512 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", ""}, - TestHash{"ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", "abc"}, - TestHash{"204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa17299aeadb6889018501d289e4900f7e4331b99dec4b5433ac7d329eeb6dd26545e96e55b874be909", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha2.hash_512(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", ""}, + TestHash{"ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", "abc"}, + TestHash{"204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa17299aeadb6889018501d289e4900f7e4331b99dec4b5433ac7d329eeb6dd26545e96e55b874be909", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha2.hash_512(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha3_224 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7", ""}, - TestHash{"e642824c3f8cf24ad09234ee7d3c766fc9a3a5168d0c94ad73b46fdf", "abc"}, - TestHash{"10241ac5187380bd501192e4e56b5280908727dd8fe0d10d4e5ad91e", "abcdbcdecdefdefgefghfghighijhi"}, - TestHash{"fd645fe07d814c397e85e85f92fe58b949f55efa4d3468b2468da45a", "jkijkljklmklmnlmnomnopnopq"}, - TestHash{"9e86ff69557ca95f405f081269685b38e3a819b309ee942f482b6a8b", "a"}, - TestHash{"6961f694b2ff3ed6f0c830d2c66da0c5e7ca9445f7c0dca679171112", "01234567012345670123456701234567"}, - TestHash{"8a24108b154ada21c9fd5574494479ba5c7e7ab76ef264ead0fcce33", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"543e6868e1666c1a643630df77367ae5a62a85070a51c14cbf665cbc", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha3.hash_224(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"6b4e03423667dbb73b6e15454f0eb1abd4597f9a1b078e3f5b5a6bc7", ""}, + TestHash{"e642824c3f8cf24ad09234ee7d3c766fc9a3a5168d0c94ad73b46fdf", "abc"}, + TestHash{"10241ac5187380bd501192e4e56b5280908727dd8fe0d10d4e5ad91e", "abcdbcdecdefdefgefghfghighijhi"}, + TestHash{"fd645fe07d814c397e85e85f92fe58b949f55efa4d3468b2468da45a", "jkijkljklmklmnlmnomnopnopq"}, + TestHash{"9e86ff69557ca95f405f081269685b38e3a819b309ee942f482b6a8b", "a"}, + TestHash{"6961f694b2ff3ed6f0c830d2c66da0c5e7ca9445f7c0dca679171112", "01234567012345670123456701234567"}, + TestHash{"8a24108b154ada21c9fd5574494479ba5c7e7ab76ef264ead0fcce33", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"543e6868e1666c1a643630df77367ae5a62a85070a51c14cbf665cbc", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha3.hash_224(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha3_256 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a", ""}, - TestHash{"3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532", "abc"}, - TestHash{"565ada1ced21278cfaffdde00dea0107964121ac25e4e978abc59412be74550a", "abcdbcdecdefdefgefghfghighijhi"}, - TestHash{"8cc1709d520f495ce972ece48b0d2e1f74ec80d53bc5c47457142158fae15d98", "jkijkljklmklmnlmnomnopnopq"}, - TestHash{"80084bf2fba02475726feb2cab2d8215eab14bc6bdd8bfb2c8151257032ecd8b", "a"}, - TestHash{"e4786de5f88f7d374b7288f225ea9f2f7654da200bab5d417e1fb52d49202767", "01234567012345670123456701234567"}, - TestHash{"41c0dba2a9d6240849100376a8235e2c82e1b9998a999e21db32dd97496d3376", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"916f6061fe879741ca6469b43971dfdb28b1a32dc36cb3254e812be27aad1d18", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha3.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a", ""}, + TestHash{"3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532", "abc"}, + TestHash{"565ada1ced21278cfaffdde00dea0107964121ac25e4e978abc59412be74550a", "abcdbcdecdefdefgefghfghighijhi"}, + TestHash{"8cc1709d520f495ce972ece48b0d2e1f74ec80d53bc5c47457142158fae15d98", "jkijkljklmklmnlmnomnopnopq"}, + TestHash{"80084bf2fba02475726feb2cab2d8215eab14bc6bdd8bfb2c8151257032ecd8b", "a"}, + TestHash{"e4786de5f88f7d374b7288f225ea9f2f7654da200bab5d417e1fb52d49202767", "01234567012345670123456701234567"}, + TestHash{"41c0dba2a9d6240849100376a8235e2c82e1b9998a999e21db32dd97496d3376", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"916f6061fe879741ca6469b43971dfdb28b1a32dc36cb3254e812be27aad1d18", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha3.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha3_384 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2ac3713831264adb47fb6bd1e058d5f004", ""}, - TestHash{"ec01498288516fc926459f58e2c6ad8df9b473cb0fc08c2596da7cf0e49be4b298d88cea927ac7f539f1edf228376d25", "abc"}, - TestHash{"9aa92dbb716ebb573def0d5e3cdd28d6add38ada310b602b8916e690a3257b7144e5ddd3d0dbbc559c48480d34d57a9a", "abcdbcdecdefdefgefghfghighijhi"}, - TestHash{"77c90323d7392bcdee8a3e7f74f19f47b7d1b1a825ac6a2d8d882a72317879cc26597035f1fc24fe65090b125a691282", "jkijkljklmklmnlmnomnopnopq"}, - TestHash{"1815f774f320491b48569efec794d249eeb59aae46d22bf77dafe25c5edc28d7ea44f93ee1234aa88f61c91912a4ccd9", "a"}, - TestHash{"51072590ad4c51b27ff8265590d74f92de7cc55284168e414ca960087c693285b08a283c6b19d77632994cb9eb93f1be", "01234567012345670123456701234567"}, - TestHash{"991c665755eb3a4b6bbdfb75c78a492e8c56a22c5c4d7e429bfdbc32b9d4ad5aa04a1f076e62fea19eef51acd0657c22", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"79407d3b5916b59c3e30b09822974791c313fb9ecc849e406f23592d04f625dc8c709b98b43b3852b337216179aa7fc7", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha3.hash_384(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"0c63a75b845e4f7d01107d852e4c2485c51a50aaaa94fc61995e71bbee983a2ac3713831264adb47fb6bd1e058d5f004", ""}, + TestHash{"ec01498288516fc926459f58e2c6ad8df9b473cb0fc08c2596da7cf0e49be4b298d88cea927ac7f539f1edf228376d25", "abc"}, + TestHash{"9aa92dbb716ebb573def0d5e3cdd28d6add38ada310b602b8916e690a3257b7144e5ddd3d0dbbc559c48480d34d57a9a", "abcdbcdecdefdefgefghfghighijhi"}, + TestHash{"77c90323d7392bcdee8a3e7f74f19f47b7d1b1a825ac6a2d8d882a72317879cc26597035f1fc24fe65090b125a691282", "jkijkljklmklmnlmnomnopnopq"}, + TestHash{"1815f774f320491b48569efec794d249eeb59aae46d22bf77dafe25c5edc28d7ea44f93ee1234aa88f61c91912a4ccd9", "a"}, + TestHash{"51072590ad4c51b27ff8265590d74f92de7cc55284168e414ca960087c693285b08a283c6b19d77632994cb9eb93f1be", "01234567012345670123456701234567"}, + TestHash{"991c665755eb3a4b6bbdfb75c78a492e8c56a22c5c4d7e429bfdbc32b9d4ad5aa04a1f076e62fea19eef51acd0657c22", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"79407d3b5916b59c3e30b09822974791c313fb9ecc849e406f23592d04f625dc8c709b98b43b3852b337216179aa7fc7", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha3.hash_384(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sha3_512 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a615b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26", ""}, - TestHash{"b751850b1a57168a5693cd924b6b096e08f621827444f70d884f5d0240d2712e10e116e9192af3c91a7ec57647e3934057340b4cf408d5a56592f8274eec53f0", "abc"}, - TestHash{"9f9a327944a35988d67effc4fa748b3c07744f736ac70b479d8e12a3d10d6884d00a7ef593690305462e9e9030a67c51636fd346fd8fa0ee28a5ac2aee103d2e", "abcdbcdecdefdefgefghfghighijhi"}, - TestHash{"dbb124a0deda966eb4d199d0844fa0beb0770ea1ccddabcd335a7939a931ac6fb4fa6aebc6573f462ced2e4e7178277803be0d24d8bc2864626d9603109b7891", "jkijkljklmklmnlmnomnopnopq"}, - TestHash{"697f2d856172cb8309d6b8b97dac4de344b549d4dee61edfb4962d8698b7fa803f4f93ff24393586e28b5b957ac3d1d369420ce53332712f997bd336d09ab02a", "a"}, - TestHash{"5679e353bc8eeea3e801ca60448b249bcfd3ac4a6c3abe429a807bcbd4c9cd12da87a5a9dc74fde64c0d44718632cae966b078397c6f9ec155c6a238f2347cf1", "01234567012345670123456701234567"}, - TestHash{"04a371e84ecfb5b8b77cb48610fca8182dd457ce6f326a0fd3d7ec2f1e91636dee691fbe0c985302ba1b0d8dc78c086346b533b49c030d99a27daf1139d6e75e", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"afebb2ef542e6579c50cad06d2e578f9f8dd6881d7dc824d26360feebf18a4fa73e3261122948efcfd492e74e82e2189ed0fb440d187f382270cb455f21dd185", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, - } - for v, _ in test_vectors { - computed := sha3.hash_512(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a615b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26", ""}, + TestHash{"b751850b1a57168a5693cd924b6b096e08f621827444f70d884f5d0240d2712e10e116e9192af3c91a7ec57647e3934057340b4cf408d5a56592f8274eec53f0", "abc"}, + TestHash{"9f9a327944a35988d67effc4fa748b3c07744f736ac70b479d8e12a3d10d6884d00a7ef593690305462e9e9030a67c51636fd346fd8fa0ee28a5ac2aee103d2e", "abcdbcdecdefdefgefghfghighijhi"}, + TestHash{"dbb124a0deda966eb4d199d0844fa0beb0770ea1ccddabcd335a7939a931ac6fb4fa6aebc6573f462ced2e4e7178277803be0d24d8bc2864626d9603109b7891", "jkijkljklmklmnlmnomnopnopq"}, + TestHash{"697f2d856172cb8309d6b8b97dac4de344b549d4dee61edfb4962d8698b7fa803f4f93ff24393586e28b5b957ac3d1d369420ce53332712f997bd336d09ab02a", "a"}, + TestHash{"5679e353bc8eeea3e801ca60448b249bcfd3ac4a6c3abe429a807bcbd4c9cd12da87a5a9dc74fde64c0d44718632cae966b078397c6f9ec155c6a238f2347cf1", "01234567012345670123456701234567"}, + TestHash{"04a371e84ecfb5b8b77cb48610fca8182dd457ce6f326a0fd3d7ec2f1e91636dee691fbe0c985302ba1b0d8dc78c086346b533b49c030d99a27daf1139d6e75e", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"afebb2ef542e6579c50cad06d2e578f9f8dd6881d7dc824d26360feebf18a4fa73e3261122948efcfd492e74e82e2189ed0fb440d187f382270cb455f21dd185", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"}, + } + for v, _ in test_vectors { + computed := sha3.hash_512(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_shake_128 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"7f9c2ba4e88f827d616045507605853e", ""}, - TestHash{"f4202e3c5852f9182a0430fd8144f0a7", "The quick brown fox jumps over the lazy dog"}, - TestHash{"853f4538be0db9621a6cea659a06c110", "The quick brown fox jumps over the lazy dof"}, - } - for v, _ in test_vectors { - computed := shake.hash_128(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"7f9c2ba4e88f827d616045507605853e", ""}, + TestHash{"f4202e3c5852f9182a0430fd8144f0a7", "The quick brown fox jumps over the lazy dog"}, + TestHash{"853f4538be0db9621a6cea659a06c110", "The quick brown fox jumps over the lazy dof"}, + } + for v, _ in test_vectors { + computed := shake.hash_128(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_shake_256 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f", ""}, - TestHash{"2f671343d9b2e1604dc9dcf0753e5fe15c7c64a0d283cbbf722d411a0e36f6ca", "The quick brown fox jumps over the lazy dog"}, - TestHash{"46b1ebb2e142c38b9ac9081bef72877fe4723959640fa57119b366ce6899d401", "The quick brown fox jumps over the lazy dof"}, - } - for v, _ in test_vectors { - computed := shake.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f", ""}, + TestHash{"2f671343d9b2e1604dc9dcf0753e5fe15c7c64a0d283cbbf722d411a0e36f6ca", "The quick brown fox jumps over the lazy dog"}, + TestHash{"46b1ebb2e142c38b9ac9081bef72877fe4723959640fa57119b366ce6899d401", "The quick brown fox jumps over the lazy dof"}, + } + for v, _ in test_vectors { + computed := shake.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_keccak_224 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"f71837502ba8e10837bdd8d365adb85591895602fc552b48b7390abd", ""}, - TestHash{"c30411768506ebe1c2871b1ee2e87d38df342317300a9b97a95ec6a8", "abc"}, - } - for v, _ in test_vectors { - computed := keccak.hash_224(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"f71837502ba8e10837bdd8d365adb85591895602fc552b48b7390abd", ""}, + TestHash{"c30411768506ebe1c2871b1ee2e87d38df342317300a9b97a95ec6a8", "abc"}, + } + for v, _ in test_vectors { + computed := keccak.hash_224(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_keccak_256 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ""}, - TestHash{"4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45", "abc"}, - } - for v, _ in test_vectors { - computed := keccak.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", ""}, + TestHash{"4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45", "abc"}, + } + for v, _ in test_vectors { + computed := keccak.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_keccak_384 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"2c23146a63a29acf99e73b88f8c24eaa7dc60aa771780ccc006afbfa8fe2479b2dd2b21362337441ac12b515911957ff", ""}, - TestHash{"f7df1165f033337be098e7d288ad6a2f74409d7a60b49c36642218de161b1f99f8c681e4afaf31a34db29fb763e3c28e", "abc"}, - } - for v, _ in test_vectors { - computed := keccak.hash_384(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"2c23146a63a29acf99e73b88f8c24eaa7dc60aa771780ccc006afbfa8fe2479b2dd2b21362337441ac12b515911957ff", ""}, + TestHash{"f7df1165f033337be098e7d288ad6a2f74409d7a60b49c36642218de161b1f99f8c681e4afaf31a34db29fb763e3c28e", "abc"}, + } + for v, _ in test_vectors { + computed := keccak.hash_384(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_keccak_512 :: proc(t: ^testing.T) { - // Test vectors from - // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf - // https://www.di-mgt.com.au/sha_testvectors.html - test_vectors := [?]TestHash { - TestHash{"0eab42de4c3ceb9235fc91acffe746b29c29a8c366b7c60e4e67c466f36a4304c00fa9caf9d87976ba469bcbe06713b435f091ef2769fb160cdab33d3670680e", ""}, - TestHash{"18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", "abc"}, - } - for v, _ in test_vectors { - computed := keccak.hash_512(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://csrc.nist.gov/csrc/media/projects/cryptographic-standards-and-guidelines/documents/examples/sha_all.pdf + // https://www.di-mgt.com.au/sha_testvectors.html + test_vectors := [?]TestHash { + TestHash{"0eab42de4c3ceb9235fc91acffe746b29c29a8c366b7c60e4e67c466f36a4304c00fa9caf9d87976ba469bcbe06713b435f091ef2769fb160cdab33d3670680e", ""}, + TestHash{"18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", "abc"}, + } + for v, _ in test_vectors { + computed := keccak.hash_512(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_whirlpool :: proc(t: ^testing.T) { - // Test vectors from - // https://web.archive.org/web/20171129084214/http://www.larc.usp.br/~pbarreto/WhirlpoolPage.html - test_vectors := [?]TestHash { - TestHash{"19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3", ""}, - TestHash{"8aca2602792aec6f11a67206531fb7d7f0dff59413145e6973c45001d0087b42d11bc645413aeff63a42391a39145a591a92200d560195e53b478584fdae231a", "a"}, - TestHash{"33e24e6cbebf168016942df8a7174048f9cebc45cbd829c3b94b401a498acb11c5abcca7f2a1238aaf534371e87a4e4b19758965d5a35a7cad87cf5517043d97", "ab"}, - TestHash{"4e2448a4c6f486bb16b6562c73b4020bf3043e3a731bce721ae1b303d97e6d4c7181eebdb6c57e277d0e34957114cbd6c797fc9d95d8b582d225292076d4eef5", "abc"}, - TestHash{"bda164f0b930c43a1bacb5df880b205d15ac847add35145bf25d991ae74f0b72b1ac794f8aacda5fcb3c47038c954742b1857b5856519de4d1e54bfa2fa4eac5", "abcd"}, - TestHash{"5d745e26ccb20fe655d39c9e7f69455758fbae541cb892b3581e4869244ab35b4fd6078f5d28b1f1a217452a67d9801033d92724a221255a5e377fe9e9e5f0b2", "abcde"}, - TestHash{"a73e425459567308ba5f9eb2ae23570d0d0575eb1357ecf6ac88d4e0358b0ac3ea2371261f5d4c070211784b525911b9eec0ad968429bb7c7891d341cff4e811", "abcdef"}, - TestHash{"08b388f68fd3eb51906ac3d3c699b8e9c3ac65d7ceb49d2e34f8a482cbc3082bc401cead90e85a97b8647c948bf35e448740b79659f3bee42145f0bd653d1f25", "abcdefg"}, - TestHash{"1f1a84d30612820243afe2022712f9dac6d07c4c8bb41b40eacab0184c8d82275da5bcadbb35c7ca1960ff21c90acbae8c14e48d9309e4819027900e882c7ad9", "abcdefgh"}, - TestHash{"11882bc9a31ac1cf1c41dcd9fd6fdd3ccdb9b017fc7f4582680134f314d7bb49af4c71f5a920bc0a6a3c1ff9a00021bf361d9867fe636b0bc1da1552e4237de4", "abcdefghi"}, - TestHash{"717163de24809ffcf7ff6d5aba72b8d67c2129721953c252a4ddfb107614be857cbd76a9d5927de14633d6bdc9ddf335160b919db5c6f12cb2e6549181912eef", "abcdefghij"}, - TestHash{"b97de512e91e3828b40d2b0fdce9ceb3c4a71f9bea8d88e75c4fa854df36725fd2b52eb6544edcacd6f8beddfea403cb55ae31f03ad62a5ef54e42ee82c3fb35", "The quick brown fox jumps over the lazy dog"}, - TestHash{"c27ba124205f72e6847f3e19834f925cc666d0974167af915bb462420ed40cc50900d85a1f923219d832357750492d5c143011a76988344c2635e69d06f2d38c", "The quick brown fox jumps over the lazy eog"}, - } - for v, _ in test_vectors { - computed := whirlpool.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://web.archive.org/web/20171129084214/http://www.larc.usp.br/~pbarreto/WhirlpoolPage.html + test_vectors := [?]TestHash { + TestHash{"19fa61d75522a4669b44e39c1d2e1726c530232130d407f89afee0964997f7a73e83be698b288febcf88e3e03c4f0757ea8964e59b63d93708b138cc42a66eb3", ""}, + TestHash{"8aca2602792aec6f11a67206531fb7d7f0dff59413145e6973c45001d0087b42d11bc645413aeff63a42391a39145a591a92200d560195e53b478584fdae231a", "a"}, + TestHash{"33e24e6cbebf168016942df8a7174048f9cebc45cbd829c3b94b401a498acb11c5abcca7f2a1238aaf534371e87a4e4b19758965d5a35a7cad87cf5517043d97", "ab"}, + TestHash{"4e2448a4c6f486bb16b6562c73b4020bf3043e3a731bce721ae1b303d97e6d4c7181eebdb6c57e277d0e34957114cbd6c797fc9d95d8b582d225292076d4eef5", "abc"}, + TestHash{"bda164f0b930c43a1bacb5df880b205d15ac847add35145bf25d991ae74f0b72b1ac794f8aacda5fcb3c47038c954742b1857b5856519de4d1e54bfa2fa4eac5", "abcd"}, + TestHash{"5d745e26ccb20fe655d39c9e7f69455758fbae541cb892b3581e4869244ab35b4fd6078f5d28b1f1a217452a67d9801033d92724a221255a5e377fe9e9e5f0b2", "abcde"}, + TestHash{"a73e425459567308ba5f9eb2ae23570d0d0575eb1357ecf6ac88d4e0358b0ac3ea2371261f5d4c070211784b525911b9eec0ad968429bb7c7891d341cff4e811", "abcdef"}, + TestHash{"08b388f68fd3eb51906ac3d3c699b8e9c3ac65d7ceb49d2e34f8a482cbc3082bc401cead90e85a97b8647c948bf35e448740b79659f3bee42145f0bd653d1f25", "abcdefg"}, + TestHash{"1f1a84d30612820243afe2022712f9dac6d07c4c8bb41b40eacab0184c8d82275da5bcadbb35c7ca1960ff21c90acbae8c14e48d9309e4819027900e882c7ad9", "abcdefgh"}, + TestHash{"11882bc9a31ac1cf1c41dcd9fd6fdd3ccdb9b017fc7f4582680134f314d7bb49af4c71f5a920bc0a6a3c1ff9a00021bf361d9867fe636b0bc1da1552e4237de4", "abcdefghi"}, + TestHash{"717163de24809ffcf7ff6d5aba72b8d67c2129721953c252a4ddfb107614be857cbd76a9d5927de14633d6bdc9ddf335160b919db5c6f12cb2e6549181912eef", "abcdefghij"}, + TestHash{"b97de512e91e3828b40d2b0fdce9ceb3c4a71f9bea8d88e75c4fa854df36725fd2b52eb6544edcacd6f8beddfea403cb55ae31f03ad62a5ef54e42ee82c3fb35", "The quick brown fox jumps over the lazy dog"}, + TestHash{"c27ba124205f72e6847f3e19834f925cc666d0974167af915bb462420ed40cc50900d85a1f923219d832357750492d5c143011a76988344c2635e69d06f2d38c", "The quick brown fox jumps over the lazy eog"}, + } + for v, _ in test_vectors { + computed := whirlpool.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_gost :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"981e5f3ca30c841487830f84fb433e13ac1101569b9c13584ac483234cd656c0", ""}, - TestHash{"e74c52dd282183bf37af0079c9f78055715a103f17e3133ceff1aacf2f403011", "a"}, - TestHash{"b285056dbf18d7392d7677369524dd14747459ed8143997e163b2986f92fd42c", "abc"}, - TestHash{"bc6041dd2aa401ebfa6e9886734174febdb4729aa972d60f549ac39b29721ba0", "message digest"}, - TestHash{"9004294a361a508c586fe53d1f1b02746765e71b765472786e4770d565830a76", "The quick brown fox jumps over the lazy dog"}, - TestHash{"73b70a39497de53a6e08c67b6d4db853540f03e9389299d9b0156ef7e85d0f61", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - TestHash{"6bc7b38989b28cf93ae8842bf9d752905910a7528a61e5bce0782de43e610c90", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - TestHash{"2cefc2f7b7bdc514e18ea57fa74ff357e7fa17d652c75f69cb1be7893ede48eb", "This is message, length=32 bytes"}, - TestHash{"c3730c5cbccacf915ac292676f21e8bd4ef75331d9405e5f1a61dc3130a65011", "Suppose the original message has length = 50 bytes"}, - } - for v, _ in test_vectors { - computed := gost.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"981e5f3ca30c841487830f84fb433e13ac1101569b9c13584ac483234cd656c0", ""}, + TestHash{"e74c52dd282183bf37af0079c9f78055715a103f17e3133ceff1aacf2f403011", "a"}, + TestHash{"b285056dbf18d7392d7677369524dd14747459ed8143997e163b2986f92fd42c", "abc"}, + TestHash{"bc6041dd2aa401ebfa6e9886734174febdb4729aa972d60f549ac39b29721ba0", "message digest"}, + TestHash{"9004294a361a508c586fe53d1f1b02746765e71b765472786e4770d565830a76", "The quick brown fox jumps over the lazy dog"}, + TestHash{"73b70a39497de53a6e08c67b6d4db853540f03e9389299d9b0156ef7e85d0f61", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + TestHash{"6bc7b38989b28cf93ae8842bf9d752905910a7528a61e5bce0782de43e610c90", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + TestHash{"2cefc2f7b7bdc514e18ea57fa74ff357e7fa17d652c75f69cb1be7893ede48eb", "This is message, length=32 bytes"}, + TestHash{"c3730c5cbccacf915ac292676f21e8bd4ef75331d9405e5f1a61dc3130a65011", "Suppose the original message has length = 50 bytes"}, + } + for v, _ in test_vectors { + computed := gost.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_streebog_256 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"3f539a213e97c802cc229d474c6aa32a825a360b2a933a949fd925208d9ce1bb", ""}, - TestHash{"3e7dea7f2384b6c5a3d0e24aaa29c05e89ddd762145030ec22c71a6db8b2c1f4", "The quick brown fox jumps over the lazy dog"}, - TestHash{"36816a824dcbe7d6171aa58500741f2ea2757ae2e1784ab72c5c3c6c198d71da", "The quick brown fox jumps over the lazy dog."}, - } - for v, _ in test_vectors { - computed := streebog.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"3f539a213e97c802cc229d474c6aa32a825a360b2a933a949fd925208d9ce1bb", ""}, + TestHash{"3e7dea7f2384b6c5a3d0e24aaa29c05e89ddd762145030ec22c71a6db8b2c1f4", "The quick brown fox jumps over the lazy dog"}, + TestHash{"36816a824dcbe7d6171aa58500741f2ea2757ae2e1784ab72c5c3c6c198d71da", "The quick brown fox jumps over the lazy dog."}, + } + for v, _ in test_vectors { + computed := streebog.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_streebog_512 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"8e945da209aa869f0455928529bcae4679e9873ab707b55315f56ceb98bef0a7362f715528356ee83cda5f2aac4c6ad2ba3a715c1bcd81cb8e9f90bf4c1c1a8a", ""}, - TestHash{"d2b793a0bb6cb5904828b5b6dcfb443bb8f33efc06ad09368878ae4cdc8245b97e60802469bed1e7c21a64ff0b179a6a1e0bb74d92965450a0adab69162c00fe", "The quick brown fox jumps over the lazy dog"}, - TestHash{"fe0c42f267d921f940faa72bd9fcf84f9f1bd7e9d055e9816e4c2ace1ec83be82d2957cd59b86e123d8f5adee80b3ca08a017599a9fc1a14d940cf87c77df070", "The quick brown fox jumps over the lazy dog."}, - } - for v, _ in test_vectors { - computed := streebog.hash_512(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"8e945da209aa869f0455928529bcae4679e9873ab707b55315f56ceb98bef0a7362f715528356ee83cda5f2aac4c6ad2ba3a715c1bcd81cb8e9f90bf4c1c1a8a", ""}, + TestHash{"d2b793a0bb6cb5904828b5b6dcfb443bb8f33efc06ad09368878ae4cdc8245b97e60802469bed1e7c21a64ff0b179a6a1e0bb74d92965450a0adab69162c00fe", "The quick brown fox jumps over the lazy dog"}, + TestHash{"fe0c42f267d921f940faa72bd9fcf84f9f1bd7e9d055e9816e4c2ace1ec83be82d2957cd59b86e123d8f5adee80b3ca08a017599a9fc1a14d940cf87c77df070", "The quick brown fox jumps over the lazy dog."}, + } + for v, _ in test_vectors { + computed := streebog.hash_512(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_blake_224 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"7dc5313b1c04512a174bd6503b89607aecbee0903d40a8a569c94eed", ""}, - TestHash{"304c27fdbf308aea06955e331adc6814223a21fccd24c09fde9eda7b", "ube"}, - TestHash{"cfb6848add73e1cb47994c4765df33b8f973702705a30a71fe4747a3", "BLAKE"}, - } - for v, _ in test_vectors { - computed := blake.hash_224(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"7dc5313b1c04512a174bd6503b89607aecbee0903d40a8a569c94eed", ""}, + TestHash{"304c27fdbf308aea06955e331adc6814223a21fccd24c09fde9eda7b", "ube"}, + TestHash{"cfb6848add73e1cb47994c4765df33b8f973702705a30a71fe4747a3", "BLAKE"}, + } + for v, _ in test_vectors { + computed := blake.hash_224(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_blake_256 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"716f6e863f744b9ac22c97ec7b76ea5f5908bc5b2f67c61510bfc4751384ea7a", ""}, - TestHash{"e802fe2a73fbe5853408f051d040aeb3a76a4d7a0fc5c3415d1af090f76a2c81", "ube"}, - TestHash{"07663e00cf96fbc136cf7b1ee099c95346ba3920893d18cc8851f22ee2e36aa6", "BLAKE"}, - } - for v, _ in test_vectors { - computed := blake.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"716f6e863f744b9ac22c97ec7b76ea5f5908bc5b2f67c61510bfc4751384ea7a", ""}, + TestHash{"e802fe2a73fbe5853408f051d040aeb3a76a4d7a0fc5c3415d1af090f76a2c81", "ube"}, + TestHash{"07663e00cf96fbc136cf7b1ee099c95346ba3920893d18cc8851f22ee2e36aa6", "BLAKE"}, + } + for v, _ in test_vectors { + computed := blake.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_blake_384 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"c6cbd89c926ab525c242e6621f2f5fa73aa4afe3d9e24aed727faaadd6af38b620bdb623dd2b4788b1c8086984af8706", ""}, - TestHash{"8f22f120b2b99dd4fd32b98c8c83bd87abd6413f7317be936b1997511247fc68ae781c6f42113224ccbc1567b0e88593", "ube"}, - TestHash{"f28742f7243990875d07e6afcff962edabdf7e9d19ddea6eae31d094c7fa6d9b00c8213a02ddf1e2d9894f3162345d85", "BLAKE"}, - } - for v, _ in test_vectors { - computed := blake.hash_384(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"c6cbd89c926ab525c242e6621f2f5fa73aa4afe3d9e24aed727faaadd6af38b620bdb623dd2b4788b1c8086984af8706", ""}, + TestHash{"8f22f120b2b99dd4fd32b98c8c83bd87abd6413f7317be936b1997511247fc68ae781c6f42113224ccbc1567b0e88593", "ube"}, + TestHash{"f28742f7243990875d07e6afcff962edabdf7e9d19ddea6eae31d094c7fa6d9b00c8213a02ddf1e2d9894f3162345d85", "BLAKE"}, + } + for v, _ in test_vectors { + computed := blake.hash_384(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_blake_512 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"a8cfbbd73726062df0c6864dda65defe58ef0cc52a5625090fa17601e1eecd1b628e94f396ae402a00acc9eab77b4d4c2e852aaaa25a636d80af3fc7913ef5b8", ""}, - TestHash{"49a24ca8f230936f938c19484d46b58f13ea4448ddadafecdf01419b1e1dd922680be2de84069187973ab61b10574da2ee50cbeaade68ea9391c8ec041b76be0", "ube"}, - TestHash{"7bf805d0d8de36802b882e65d0515aa7682a2be97a9d9ec1399f4be2eff7de07684d7099124c8ac81c1c7c200d24ba68c6222e75062e04feb0e9dd589aa6e3b7", "BLAKE"}, - } - for v, _ in test_vectors { - computed := blake.hash_512(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"a8cfbbd73726062df0c6864dda65defe58ef0cc52a5625090fa17601e1eecd1b628e94f396ae402a00acc9eab77b4d4c2e852aaaa25a636d80af3fc7913ef5b8", ""}, + TestHash{"49a24ca8f230936f938c19484d46b58f13ea4448ddadafecdf01419b1e1dd922680be2de84069187973ab61b10574da2ee50cbeaade68ea9391c8ec041b76be0", "ube"}, + TestHash{"7bf805d0d8de36802b882e65d0515aa7682a2be97a9d9ec1399f4be2eff7de07684d7099124c8ac81c1c7c200d24ba68c6222e75062e04feb0e9dd589aa6e3b7", "BLAKE"}, + } + for v, _ in test_vectors { + computed := blake.hash_512(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_blake2b :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419d25e1031afee585313896444934eb04b903a685b1448b755d56f701afe9be2ce", ""}, - TestHash{"a8add4bdddfd93e4877d2746e62817b116364a1fa7bc148d95090bc7333b3673f82401cf7aa2e4cb1ecd90296e3f14cb5413f8ed77be73045b13914cdcd6a918", "The quick brown fox jumps over the lazy dog"}, - } - for v, _ in test_vectors { - computed := blake2b.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"786a02f742015903c6c6fd852552d272912f4740e15847618a86e217f71f5419d25e1031afee585313896444934eb04b903a685b1448b755d56f701afe9be2ce", ""}, + TestHash{"a8add4bdddfd93e4877d2746e62817b116364a1fa7bc148d95090bc7333b3673f82401cf7aa2e4cb1ecd90296e3f14cb5413f8ed77be73045b13914cdcd6a918", "The quick brown fox jumps over the lazy dog"}, + } + for v, _ in test_vectors { + computed := blake2b.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_blake2s :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"69217a3079908094e11121d042354a7c1f55b6482ca1a51e1b250dfd1ed0eef9", ""}, - TestHash{"606beeec743ccbeff6cbcdf5d5302aa855c256c29b88c8ed331ea1a6bf3c8812", "The quick brown fox jumps over the lazy dog"}, - } - for v, _ in test_vectors { - computed := blake2s.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"69217a3079908094e11121d042354a7c1f55b6482ca1a51e1b250dfd1ed0eef9", ""}, + TestHash{"606beeec743ccbeff6cbcdf5d5302aa855c256c29b88c8ed331ea1a6bf3c8812", "The quick brown fox jumps over the lazy dog"}, + } + for v, _ in test_vectors { + computed := blake2s.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_ripemd_128 :: proc(t: ^testing.T) { - // Test vectors from - // https://homes.esat.kuleuven.be/~bosselae/ripemd160.html - test_vectors := [?]TestHash { - TestHash{"cdf26213a150dc3ecb610f18f6b38b46", ""}, - TestHash{"86be7afa339d0fc7cfc785e72f578d33", "a"}, - TestHash{"c14a12199c66e4ba84636b0f69144c77", "abc"}, - TestHash{"9e327b3d6e523062afc1132d7df9d1b8", "message digest"}, - TestHash{"fd2aa607f71dc8f510714922b371834e", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"a1aa0689d0fafa2ddc22e88b49133a06", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"d1e959eb179c911faea4624c60c5c702", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - } - for v, _ in test_vectors { - computed := ripemd.hash_128(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://homes.esat.kuleuven.be/~bosselae/ripemd160.html + test_vectors := [?]TestHash { + TestHash{"cdf26213a150dc3ecb610f18f6b38b46", ""}, + TestHash{"86be7afa339d0fc7cfc785e72f578d33", "a"}, + TestHash{"c14a12199c66e4ba84636b0f69144c77", "abc"}, + TestHash{"9e327b3d6e523062afc1132d7df9d1b8", "message digest"}, + TestHash{"fd2aa607f71dc8f510714922b371834e", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"a1aa0689d0fafa2ddc22e88b49133a06", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"d1e959eb179c911faea4624c60c5c702", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + } + for v, _ in test_vectors { + computed := ripemd.hash_128(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_ripemd_160 :: proc(t: ^testing.T) { - // Test vectors from - // https://homes.esat.kuleuven.be/~bosselae/ripemd160.html - test_vectors := [?]TestHash { - TestHash{"9c1185a5c5e9fc54612808977ee8f548b2258d31", ""}, - TestHash{"0bdc9d2d256b3ee9daae347be6f4dc835a467ffe", "a"}, - TestHash{"8eb208f7e05d987a9b044a8e98c6b087f15a0bfc", "abc"}, - TestHash{"5d0689ef49d2fae572b881b123a85ffa21595f36", "message digest"}, - TestHash{"f71c27109c692c1b56bbdceb5b9d2865b3708dbc", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"12a053384a9c0c88e405a06c27dcf49ada62eb2b", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"b0e20b6e3116640286ed3a87a5713079b21f5189", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - } - for v, _ in test_vectors { - computed := ripemd.hash_160(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://homes.esat.kuleuven.be/~bosselae/ripemd160.html + test_vectors := [?]TestHash { + TestHash{"9c1185a5c5e9fc54612808977ee8f548b2258d31", ""}, + TestHash{"0bdc9d2d256b3ee9daae347be6f4dc835a467ffe", "a"}, + TestHash{"8eb208f7e05d987a9b044a8e98c6b087f15a0bfc", "abc"}, + TestHash{"5d0689ef49d2fae572b881b123a85ffa21595f36", "message digest"}, + TestHash{"f71c27109c692c1b56bbdceb5b9d2865b3708dbc", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"12a053384a9c0c88e405a06c27dcf49ada62eb2b", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"b0e20b6e3116640286ed3a87a5713079b21f5189", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + } + for v, _ in test_vectors { + computed := ripemd.hash_160(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_ripemd_256 :: proc(t: ^testing.T) { - // Test vectors from - // https://homes.esat.kuleuven.be/~bosselae/ripemd160.html - test_vectors := [?]TestHash { - TestHash{"02ba4c4e5f8ecd1877fc52d64d30e37a2d9774fb1e5d026380ae0168e3c5522d", ""}, - TestHash{"f9333e45d857f5d90a91bab70a1eba0cfb1be4b0783c9acfcd883a9134692925", "a"}, - TestHash{"afbd6e228b9d8cbbcef5ca2d03e6dba10ac0bc7dcbe4680e1e42d2e975459b65", "abc"}, - TestHash{"87e971759a1ce47a514d5c914c392c9018c7c46bc14465554afcdf54a5070c0e", "message digest"}, - TestHash{"649d3034751ea216776bf9a18acc81bc7896118a5197968782dd1fd97d8d5133", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"3843045583aac6c8c8d9128573e7a9809afb2a0f34ccc36ea9e72f16f6368e3f", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"5740a408ac16b720b84424ae931cbb1fe363d1d0bf4017f1a89f7ea6de77a0b8", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - } - for v, _ in test_vectors { - computed := ripemd.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://homes.esat.kuleuven.be/~bosselae/ripemd160.html + test_vectors := [?]TestHash { + TestHash{"02ba4c4e5f8ecd1877fc52d64d30e37a2d9774fb1e5d026380ae0168e3c5522d", ""}, + TestHash{"f9333e45d857f5d90a91bab70a1eba0cfb1be4b0783c9acfcd883a9134692925", "a"}, + TestHash{"afbd6e228b9d8cbbcef5ca2d03e6dba10ac0bc7dcbe4680e1e42d2e975459b65", "abc"}, + TestHash{"87e971759a1ce47a514d5c914c392c9018c7c46bc14465554afcdf54a5070c0e", "message digest"}, + TestHash{"649d3034751ea216776bf9a18acc81bc7896118a5197968782dd1fd97d8d5133", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"3843045583aac6c8c8d9128573e7a9809afb2a0f34ccc36ea9e72f16f6368e3f", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"5740a408ac16b720b84424ae931cbb1fe363d1d0bf4017f1a89f7ea6de77a0b8", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + } + for v, _ in test_vectors { + computed := ripemd.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_ripemd_320 :: proc(t: ^testing.T) { - // Test vectors from - // https://homes.esat.kuleuven.be/~bosselae/ripemd160.html - test_vectors := [?]TestHash { - TestHash{"22d65d5661536cdc75c1fdf5c6de7b41b9f27325ebc61e8557177d705a0ec880151c3a32a00899b8", ""}, - TestHash{"ce78850638f92658a5a585097579926dda667a5716562cfcf6fbe77f63542f99b04705d6970dff5d", "a"}, - TestHash{"de4c01b3054f8930a79d09ae738e92301e5a17085beffdc1b8d116713e74f82fa942d64cdbc4682d", "abc"}, - TestHash{"3a8e28502ed45d422f68844f9dd316e7b98533fa3f2a91d29f84d425c88d6b4eff727df66a7c0197", "message digest"}, - TestHash{"cabdb1810b92470a2093aa6bce05952c28348cf43ff60841975166bb40ed234004b8824463e6b009", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"d034a7950cf722021ba4b84df769a5de2060e259df4c9bb4a4268c0e935bbc7470a969c9d072a1ac", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"ed544940c86d67f250d232c30b7b3e5770e0c60c8cb9a4cafe3b11388af9920e1b99230b843c86a4", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - } - for v, _ in test_vectors { - computed := ripemd.hash_320(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + // Test vectors from + // https://homes.esat.kuleuven.be/~bosselae/ripemd160.html + test_vectors := [?]TestHash { + TestHash{"22d65d5661536cdc75c1fdf5c6de7b41b9f27325ebc61e8557177d705a0ec880151c3a32a00899b8", ""}, + TestHash{"ce78850638f92658a5a585097579926dda667a5716562cfcf6fbe77f63542f99b04705d6970dff5d", "a"}, + TestHash{"de4c01b3054f8930a79d09ae738e92301e5a17085beffdc1b8d116713e74f82fa942d64cdbc4682d", "abc"}, + TestHash{"3a8e28502ed45d422f68844f9dd316e7b98533fa3f2a91d29f84d425c88d6b4eff727df66a7c0197", "message digest"}, + TestHash{"cabdb1810b92470a2093aa6bce05952c28348cf43ff60841975166bb40ed234004b8824463e6b009", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"d034a7950cf722021ba4b84df769a5de2060e259df4c9bb4a4268c0e935bbc7470a969c9d072a1ac", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"ed544940c86d67f250d232c30b7b3e5770e0c60c8cb9a4cafe3b11388af9920e1b99230b843c86a4", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + } + for v, _ in test_vectors { + computed := ripemd.hash_320(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_tiger_128 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"3293ac630c13f0245f92bbb1766e1616", ""}, - TestHash{"77befbef2e7ef8ab2ec8f93bf587a7fc", "a"}, - TestHash{"2aab1484e8c158f2bfb8c5ff41b57a52", "abc"}, - TestHash{"d981f8cb78201a950dcf3048751e441c", "message digest"}, - TestHash{"1714a472eee57d30040412bfcc55032a", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"0f7bf9a19b9c58f2b7610df7e84f0ac3", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"8dcea680a17583ee502ba38a3c368651", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - TestHash{"1c14795529fd9f207a958f84c52f11e8", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - TestHash{"6d12a41e72e644f017b6f0e2f7b44c62", "The quick brown fox jumps over the lazy dog"}, - } - for v, _ in test_vectors { - computed := tiger.hash_128(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"3293ac630c13f0245f92bbb1766e1616", ""}, + TestHash{"77befbef2e7ef8ab2ec8f93bf587a7fc", "a"}, + TestHash{"2aab1484e8c158f2bfb8c5ff41b57a52", "abc"}, + TestHash{"d981f8cb78201a950dcf3048751e441c", "message digest"}, + TestHash{"1714a472eee57d30040412bfcc55032a", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"0f7bf9a19b9c58f2b7610df7e84f0ac3", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"8dcea680a17583ee502ba38a3c368651", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + TestHash{"1c14795529fd9f207a958f84c52f11e8", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + TestHash{"6d12a41e72e644f017b6f0e2f7b44c62", "The quick brown fox jumps over the lazy dog"}, + } + for v, _ in test_vectors { + computed := tiger.hash_128(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_tiger_160 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"3293ac630c13f0245f92bbb1766e16167a4e5849", ""}, - TestHash{"77befbef2e7ef8ab2ec8f93bf587a7fc613e247f", "a"}, - TestHash{"2aab1484e8c158f2bfb8c5ff41b57a525129131c", "abc"}, - TestHash{"d981f8cb78201a950dcf3048751e441c517fca1a", "message digest"}, - TestHash{"1714a472eee57d30040412bfcc55032a0b11602f", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"0f7bf9a19b9c58f2b7610df7e84f0ac3a71c631e", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"8dcea680a17583ee502ba38a3c368651890ffbcc", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - TestHash{"1c14795529fd9f207a958f84c52f11e887fa0cab", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - TestHash{"6d12a41e72e644f017b6f0e2f7b44c6285f06dd5", "The quick brown fox jumps over the lazy dog"}, - } - for v, _ in test_vectors { - computed := tiger.hash_160(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"3293ac630c13f0245f92bbb1766e16167a4e5849", ""}, + TestHash{"77befbef2e7ef8ab2ec8f93bf587a7fc613e247f", "a"}, + TestHash{"2aab1484e8c158f2bfb8c5ff41b57a525129131c", "abc"}, + TestHash{"d981f8cb78201a950dcf3048751e441c517fca1a", "message digest"}, + TestHash{"1714a472eee57d30040412bfcc55032a0b11602f", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"0f7bf9a19b9c58f2b7610df7e84f0ac3a71c631e", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"8dcea680a17583ee502ba38a3c368651890ffbcc", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + TestHash{"1c14795529fd9f207a958f84c52f11e887fa0cab", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + TestHash{"6d12a41e72e644f017b6f0e2f7b44c6285f06dd5", "The quick brown fox jumps over the lazy dog"}, + } + for v, _ in test_vectors { + computed := tiger.hash_160(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_tiger_192 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"3293ac630c13f0245f92bbb1766e16167a4e58492dde73f3", ""}, - TestHash{"77befbef2e7ef8ab2ec8f93bf587a7fc613e247f5f247809", "a"}, - TestHash{"2aab1484e8c158f2bfb8c5ff41b57a525129131c957b5f93", "abc"}, - TestHash{"d981f8cb78201a950dcf3048751e441c517fca1aa55a29f6", "message digest"}, - TestHash{"1714a472eee57d30040412bfcc55032a0b11602ff37beee9", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"0f7bf9a19b9c58f2b7610df7e84f0ac3a71c631e7b53f78e", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"8dcea680a17583ee502ba38a3c368651890ffbccdc49a8cc", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - TestHash{"1c14795529fd9f207a958f84c52f11e887fa0cabdfd91bfd", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - TestHash{"6d12a41e72e644f017b6f0e2f7b44c6285f06dd5d2c5b075", "The quick brown fox jumps over the lazy dog"}, - } - for v, _ in test_vectors { - computed := tiger.hash_192(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"3293ac630c13f0245f92bbb1766e16167a4e58492dde73f3", ""}, + TestHash{"77befbef2e7ef8ab2ec8f93bf587a7fc613e247f5f247809", "a"}, + TestHash{"2aab1484e8c158f2bfb8c5ff41b57a525129131c957b5f93", "abc"}, + TestHash{"d981f8cb78201a950dcf3048751e441c517fca1aa55a29f6", "message digest"}, + TestHash{"1714a472eee57d30040412bfcc55032a0b11602ff37beee9", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"0f7bf9a19b9c58f2b7610df7e84f0ac3a71c631e7b53f78e", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"8dcea680a17583ee502ba38a3c368651890ffbccdc49a8cc", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + TestHash{"1c14795529fd9f207a958f84c52f11e887fa0cabdfd91bfd", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + TestHash{"6d12a41e72e644f017b6f0e2f7b44c6285f06dd5d2c5b075", "The quick brown fox jumps over the lazy dog"}, + } + for v, _ in test_vectors { + computed := tiger.hash_192(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_tiger2_128 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"4441be75f6018773c206c22745374b92", ""}, - TestHash{"976abff8062a2e9dcea3a1ace966ed9c", "The quick brown fox jumps over the lazy dog"}, - TestHash{"09c11330283a27efb51930aa7dc1ec62", "The quick brown fox jumps over the lazy cog"}, - } - for v, _ in test_vectors { - computed := tiger2.hash_128(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"4441be75f6018773c206c22745374b92", ""}, + TestHash{"976abff8062a2e9dcea3a1ace966ed9c", "The quick brown fox jumps over the lazy dog"}, + TestHash{"09c11330283a27efb51930aa7dc1ec62", "The quick brown fox jumps over the lazy cog"}, + } + for v, _ in test_vectors { + computed := tiger2.hash_128(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_tiger2_160 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"4441be75f6018773c206c22745374b924aa8313f", ""}, - TestHash{"976abff8062a2e9dcea3a1ace966ed9c19cb8555", "The quick brown fox jumps over the lazy dog"}, - TestHash{"09c11330283a27efb51930aa7dc1ec624ff738a8", "The quick brown fox jumps over the lazy cog"}, - } - for v, _ in test_vectors { - computed := tiger2.hash_160(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"4441be75f6018773c206c22745374b924aa8313f", ""}, + TestHash{"976abff8062a2e9dcea3a1ace966ed9c19cb8555", "The quick brown fox jumps over the lazy dog"}, + TestHash{"09c11330283a27efb51930aa7dc1ec624ff738a8", "The quick brown fox jumps over the lazy cog"}, + } + for v, _ in test_vectors { + computed := tiger2.hash_160(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_tiger2_192 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"4441be75f6018773c206c22745374b924aa8313fef919f41", ""}, - TestHash{"976abff8062a2e9dcea3a1ace966ed9c19cb85558b4976d8", "The quick brown fox jumps over the lazy dog"}, - TestHash{"09c11330283a27efb51930aa7dc1ec624ff738a8d9bdd3df", "The quick brown fox jumps over the lazy cog"}, - } - for v, _ in test_vectors { - computed := tiger2.hash_192(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"4441be75f6018773c206c22745374b924aa8313fef919f41", ""}, + TestHash{"976abff8062a2e9dcea3a1ace966ed9c19cb85558b4976d8", "The quick brown fox jumps over the lazy dog"}, + TestHash{"09c11330283a27efb51930aa7dc1ec624ff738a8d9bdd3df", "The quick brown fox jumps over the lazy cog"}, + } + for v, _ in test_vectors { + computed := tiger2.hash_192(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_sm3 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"1ab21d8355cfa17f8e61194831e81a8f22bec8c728fefb747ed035eb5082aa2b", ""}, - TestHash{"66c7f0f462eeedd9d1f2d46bdc10e4e24167c4875cf2f7a2297da02b8f4ba8e0", "abc"}, - TestHash{"debe9ff92275b8a138604889c18e5a4d6fdb70e5387e5765293dcba39c0c5732", "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"}, - TestHash{"5fdfe814b8573ca021983970fc79b2218c9570369b4859684e2e4c3fc76cb8ea", "The quick brown fox jumps over the lazy dog"}, - TestHash{"ca27d14a42fc04c1e5ecf574a95a8c2d70ecb5805e9b429026ccac8f28b20098", "The quick brown fox jumps over the lazy cog"}, - } - for v, _ in test_vectors { - computed := sm3.hash(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"1ab21d8355cfa17f8e61194831e81a8f22bec8c728fefb747ed035eb5082aa2b", ""}, + TestHash{"66c7f0f462eeedd9d1f2d46bdc10e4e24167c4875cf2f7a2297da02b8f4ba8e0", "abc"}, + TestHash{"debe9ff92275b8a138604889c18e5a4d6fdb70e5387e5765293dcba39c0c5732", "abcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"}, + TestHash{"5fdfe814b8573ca021983970fc79b2218c9570369b4859684e2e4c3fc76cb8ea", "The quick brown fox jumps over the lazy dog"}, + TestHash{"ca27d14a42fc04c1e5ecf574a95a8c2d70ecb5805e9b429026ccac8f28b20098", "The quick brown fox jumps over the lazy cog"}, + } + for v, _ in test_vectors { + computed := sm3.hash(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_jh_224 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"2c99df889b019309051c60fecc2bd285a774940e43175b76b2626630", ""}, - TestHash{"e715f969fb61b203a97e494aab92d91a9cec52f0933436b0d63bf722", "a"}, - TestHash{"c2b1967e635bd55b6a4d36f863ac4a877be302251d68692873007281", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := jh.hash_224(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"2c99df889b019309051c60fecc2bd285a774940e43175b76b2626630", ""}, + TestHash{"e715f969fb61b203a97e494aab92d91a9cec52f0933436b0d63bf722", "a"}, + TestHash{"c2b1967e635bd55b6a4d36f863ac4a877be302251d68692873007281", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := jh.hash_224(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_jh_256 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"46e64619c18bb0a92a5e87185a47eef83ca747b8fcc8e1412921357e326df434", ""}, - TestHash{"d52c0c130a1bc0ae5136375637a52773e150c71efe1c968df8956f6745b05386", "a"}, - TestHash{"fc4214867025a8af94c614353b3553b10e561ae749fc18c40e5fd44a7a4ecd1b", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := jh.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"46e64619c18bb0a92a5e87185a47eef83ca747b8fcc8e1412921357e326df434", ""}, + TestHash{"d52c0c130a1bc0ae5136375637a52773e150c71efe1c968df8956f6745b05386", "a"}, + TestHash{"fc4214867025a8af94c614353b3553b10e561ae749fc18c40e5fd44a7a4ecd1b", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := jh.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_jh_384 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"2fe5f71b1b3290d3c017fb3c1a4d02a5cbeb03a0476481e25082434a881994b0ff99e078d2c16b105ad069b569315328", ""}, - TestHash{"77de897ca4fd5dadfbcbd1d8d4ea3c3c1426855e38661325853e92b069f3fe156729f6bbb9a5892c7c18a77f1cb9d0bb", "a"}, - TestHash{"6f73d9b9b8ed362f8180fb26020725b40bd6ca75b3b947405f26c4c37a885ce028876dc42e379d2faf6146fed3ea0e42", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := jh.hash_384(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"2fe5f71b1b3290d3c017fb3c1a4d02a5cbeb03a0476481e25082434a881994b0ff99e078d2c16b105ad069b569315328", ""}, + TestHash{"77de897ca4fd5dadfbcbd1d8d4ea3c3c1426855e38661325853e92b069f3fe156729f6bbb9a5892c7c18a77f1cb9d0bb", "a"}, + TestHash{"6f73d9b9b8ed362f8180fb26020725b40bd6ca75b3b947405f26c4c37a885ce028876dc42e379d2faf6146fed3ea0e42", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := jh.hash_384(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_jh_512 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"90ecf2f76f9d2c8017d979ad5ab96b87d58fc8fc4b83060f3f900774faa2c8fabe69c5f4ff1ec2b61d6b316941cedee117fb04b1f4c5bc1b919ae841c50eec4f", ""}, - TestHash{"f12c87e986daff17c481c81a99a39b603ca6bafcd320c5735523b97cb9a26f7681bad62ffad9aad0e21160a05f773fb0d1434ca4cbcb0483f480a171ada1561b", "a"}, - TestHash{"bafb8e710b35eabeb1a48220c4b0987c2c985b6e73b7b31d164bfb9d67c94d99d7bc43b474a25e647cd6cc36334b6a00a5f2a85fae74907fd2885c6168132fe7", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := jh.hash_512(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"90ecf2f76f9d2c8017d979ad5ab96b87d58fc8fc4b83060f3f900774faa2c8fabe69c5f4ff1ec2b61d6b316941cedee117fb04b1f4c5bc1b919ae841c50eec4f", ""}, + TestHash{"f12c87e986daff17c481c81a99a39b603ca6bafcd320c5735523b97cb9a26f7681bad62ffad9aad0e21160a05f773fb0d1434ca4cbcb0483f480a171ada1561b", "a"}, + TestHash{"bafb8e710b35eabeb1a48220c4b0987c2c985b6e73b7b31d164bfb9d67c94d99d7bc43b474a25e647cd6cc36334b6a00a5f2a85fae74907fd2885c6168132fe7", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := jh.hash_512(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_groestl_224 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"f2e180fb5947be964cd584e22e496242c6a329c577fc4ce8c36d34c3", ""}, - TestHash{"2dfa5bd326c23c451b1202d99e6cee98a98c45927e1a31077f538712", "a"}, - TestHash{"c8a3e7274d599900ae673419683c3626a2e49ed57308ed2687508bef", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := groestl.hash_224(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"f2e180fb5947be964cd584e22e496242c6a329c577fc4ce8c36d34c3", ""}, + TestHash{"2dfa5bd326c23c451b1202d99e6cee98a98c45927e1a31077f538712", "a"}, + TestHash{"c8a3e7274d599900ae673419683c3626a2e49ed57308ed2687508bef", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := groestl.hash_224(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_groestl_256 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"1a52d11d550039be16107f9c58db9ebcc417f16f736adb2502567119f0083467", ""}, - TestHash{"3645c245bb31223ad93c80885b719aa40b4bed0a9d9d6e7c11fe99e59ca350b5", "a"}, - TestHash{"2679d98913bee62e57fdbdde97ddb328373548c6b24fc587cc3d08f2a02a529c", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := groestl.hash_256(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"1a52d11d550039be16107f9c58db9ebcc417f16f736adb2502567119f0083467", ""}, + TestHash{"3645c245bb31223ad93c80885b719aa40b4bed0a9d9d6e7c11fe99e59ca350b5", "a"}, + TestHash{"2679d98913bee62e57fdbdde97ddb328373548c6b24fc587cc3d08f2a02a529c", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := groestl.hash_256(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_groestl_384 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"ac353c1095ace21439251007862d6c62f829ddbe6de4f78e68d310a9205a736d8b11d99bffe448f57a1cfa2934f044a5", ""}, - TestHash{"13fce7bd9fc69b67cc12c77e765a0a97794c585f89df39fbff32408e060d7d9225c7e80fd87da647686888bda896c342", "a"}, - TestHash{"1c446cd70a6de52c9db386f5305aae029fe5a4120bc6230b7cd3a5e1ef1949cc8e6d2548c24cd7347b5ba512628a62f6", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := groestl.hash_384(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"ac353c1095ace21439251007862d6c62f829ddbe6de4f78e68d310a9205a736d8b11d99bffe448f57a1cfa2934f044a5", ""}, + TestHash{"13fce7bd9fc69b67cc12c77e765a0a97794c585f89df39fbff32408e060d7d9225c7e80fd87da647686888bda896c342", "a"}, + TestHash{"1c446cd70a6de52c9db386f5305aae029fe5a4120bc6230b7cd3a5e1ef1949cc8e6d2548c24cd7347b5ba512628a62f6", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := groestl.hash_384(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_groestl_512 :: proc(t: ^testing.T) { - test_vectors := [?]TestHash { - TestHash{"6d3ad29d279110eef3adbd66de2a0345a77baede1557f5d099fce0c03d6dc2ba8e6d4a6633dfbd66053c20faa87d1a11f39a7fbe4a6c2f009801370308fc4ad8", ""}, - TestHash{"9ef345a835ee35d6d0d462ce45f722d84b5ca41fde9c81a98a22cfb4f7425720511b03a258cdc055bf8e9179dc9bdb5d88bed906c71125d4cf0cd39d3d7bebc7", "a"}, - TestHash{"862849fd911852cd54beefa88759db4cead0ef8e36aaf15398303c5c4cbc016d9b4c42b32081cbdcba710d2693e7663d244fae116ec29ffb40168baf44f944e7", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, - } - for v, _ in test_vectors { - computed := groestl.hash_512(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors := [?]TestHash { + TestHash{"6d3ad29d279110eef3adbd66de2a0345a77baede1557f5d099fce0c03d6dc2ba8e6d4a6633dfbd66053c20faa87d1a11f39a7fbe4a6c2f009801370308fc4ad8", ""}, + TestHash{"9ef345a835ee35d6d0d462ce45f722d84b5ca41fde9c81a98a22cfb4f7425720511b03a258cdc055bf8e9179dc9bdb5d88bed906c71125d4cf0cd39d3d7bebc7", "a"}, + TestHash{"862849fd911852cd54beefa88759db4cead0ef8e36aaf15398303c5c4cbc016d9b4c42b32081cbdcba710d2693e7663d244fae116ec29ffb40168baf44f944e7", "12345678901234567890123456789012345678901234567890123456789012345678901234567890"}, + } + for v, _ in test_vectors { + computed := groestl.hash_512(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_haval_128 :: proc(t: ^testing.T) { - test_vectors_3 := [?]TestHash { - TestHash{"c68f39913f901f3ddf44c707357a7d70", ""}, - TestHash{"0cd40739683e15f01ca5dbceef4059f1", "a"}, - TestHash{"9e40ed883fb63e985d299b40cda2b8f2", "abc"}, - TestHash{"3caf4a79e81adcd6d1716bcc1cef4573", "message digest"}, - TestHash{"dc502247fb3eb8376109eda32d361d82", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"44068770868768964d1f2c3bff4aa3d8", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"de5eb3f7d9eb08fae7a07d68e3047ec6", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - } - for v, _ in test_vectors_3 { - computed := haval.hash_128_3(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_4 := [?]TestHash { - TestHash{"ee6bbf4d6a46a679b3a856c88538bb98", ""}, - TestHash{"5cd07f03330c3b5020b29ba75911e17d", "a"}, - } - for v, _ in test_vectors_4 { - computed := haval.hash_128_4(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_5 := [?]TestHash { - TestHash{"184b8482a0c050dca54b59c7f05bf5dd", ""}, - TestHash{"f23fbe704be8494bfa7a7fb4f8ab09e5", "a"}, - } - for v, _ in test_vectors_5 { - computed := haval.hash_128_5(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors_3 := [?]TestHash { + TestHash{"c68f39913f901f3ddf44c707357a7d70", ""}, + TestHash{"0cd40739683e15f01ca5dbceef4059f1", "a"}, + TestHash{"9e40ed883fb63e985d299b40cda2b8f2", "abc"}, + TestHash{"3caf4a79e81adcd6d1716bcc1cef4573", "message digest"}, + TestHash{"dc502247fb3eb8376109eda32d361d82", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"44068770868768964d1f2c3bff4aa3d8", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"de5eb3f7d9eb08fae7a07d68e3047ec6", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + } + for v, _ in test_vectors_3 { + computed := haval.hash_128_3(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_4 := [?]TestHash { + TestHash{"ee6bbf4d6a46a679b3a856c88538bb98", ""}, + TestHash{"5cd07f03330c3b5020b29ba75911e17d", "a"}, + } + for v, _ in test_vectors_4 { + computed := haval.hash_128_4(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_5 := [?]TestHash { + TestHash{"184b8482a0c050dca54b59c7f05bf5dd", ""}, + TestHash{"f23fbe704be8494bfa7a7fb4f8ab09e5", "a"}, + } + for v, _ in test_vectors_5 { + computed := haval.hash_128_5(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_haval_160 :: proc(t: ^testing.T) { - test_vectors_3 := [?]TestHash { - TestHash{"d353c3ae22a25401d257643836d7231a9a95f953", ""}, - TestHash{"4da08f514a7275dbc4cece4a347385983983a830", "a"}, - TestHash{"b21e876c4d391e2a897661149d83576b5530a089", "abc"}, - TestHash{"43a47f6f1c016207f08be8115c0977bf155346da", "message digest"}, - TestHash{"eba9fa6050f24c07c29d1834a60900ea4e32e61b", "abcdefghijklmnopqrstuvwxyz"}, - TestHash{"c30bce448cf8cfe957c141e90c0a063497cdfeeb", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - TestHash{"97dc988d97caae757be7523c4e8d4ea63007a4b9", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, - } - for v, _ in test_vectors_3 { - computed := haval.hash_160_3(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_4 := [?]TestHash { - TestHash{"1d33aae1be4146dbaaca0b6e70d7a11f10801525", ""}, - TestHash{"e0a5be29627332034d4dd8a910a1a0e6fe04084d", "a"}, - } - for v, _ in test_vectors_4 { - computed := haval.hash_160_4(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_5 := [?]TestHash { - TestHash{"255158cfc1eed1a7be7c55ddd64d9790415b933b", ""}, - TestHash{"f5147df7abc5e3c81b031268927c2b5761b5a2b5", "a"}, - } - for v, _ in test_vectors_5 { - computed := haval.hash_160_5(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors_3 := [?]TestHash { + TestHash{"d353c3ae22a25401d257643836d7231a9a95f953", ""}, + TestHash{"4da08f514a7275dbc4cece4a347385983983a830", "a"}, + TestHash{"b21e876c4d391e2a897661149d83576b5530a089", "abc"}, + TestHash{"43a47f6f1c016207f08be8115c0977bf155346da", "message digest"}, + TestHash{"eba9fa6050f24c07c29d1834a60900ea4e32e61b", "abcdefghijklmnopqrstuvwxyz"}, + TestHash{"c30bce448cf8cfe957c141e90c0a063497cdfeeb", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + TestHash{"97dc988d97caae757be7523c4e8d4ea63007a4b9", "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"}, + } + for v, _ in test_vectors_3 { + computed := haval.hash_160_3(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_4 := [?]TestHash { + TestHash{"1d33aae1be4146dbaaca0b6e70d7a11f10801525", ""}, + TestHash{"e0a5be29627332034d4dd8a910a1a0e6fe04084d", "a"}, + } + for v, _ in test_vectors_4 { + computed := haval.hash_160_4(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_5 := [?]TestHash { + TestHash{"255158cfc1eed1a7be7c55ddd64d9790415b933b", ""}, + TestHash{"f5147df7abc5e3c81b031268927c2b5761b5a2b5", "a"}, + } + for v, _ in test_vectors_5 { + computed := haval.hash_160_5(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_haval_192 :: proc(t: ^testing.T) { - test_vectors_3 := [?]TestHash { - TestHash{"e9c48d7903eaf2a91c5b350151efcb175c0fc82de2289a4e", ""}, - TestHash{"b359c8835647f5697472431c142731ff6e2cddcacc4f6e08", "a"}, - } - for v, _ in test_vectors_3 { - computed := haval.hash_192_3(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_4 := [?]TestHash { - TestHash{"4a8372945afa55c7dead800311272523ca19d42ea47b72da", ""}, - TestHash{"856c19f86214ea9a8a2f0c4b758b973cce72a2d8ff55505c", "a"}, - } - for v, _ in test_vectors_4 { - computed := haval.hash_192_4(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_5 := [?]TestHash { - TestHash{"4839d0626f95935e17ee2fc4509387bbe2cc46cb382ffe85", ""}, - TestHash{"5ffa3b3548a6e2cfc06b7908ceb5263595df67cf9c4b9341", "a"}, - } - for v, _ in test_vectors_5 { - computed := haval.hash_192_5(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors_3 := [?]TestHash { + TestHash{"e9c48d7903eaf2a91c5b350151efcb175c0fc82de2289a4e", ""}, + TestHash{"b359c8835647f5697472431c142731ff6e2cddcacc4f6e08", "a"}, + } + for v, _ in test_vectors_3 { + computed := haval.hash_192_3(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_4 := [?]TestHash { + TestHash{"4a8372945afa55c7dead800311272523ca19d42ea47b72da", ""}, + TestHash{"856c19f86214ea9a8a2f0c4b758b973cce72a2d8ff55505c", "a"}, + } + for v, _ in test_vectors_4 { + computed := haval.hash_192_4(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_5 := [?]TestHash { + TestHash{"4839d0626f95935e17ee2fc4509387bbe2cc46cb382ffe85", ""}, + TestHash{"5ffa3b3548a6e2cfc06b7908ceb5263595df67cf9c4b9341", "a"}, + } + for v, _ in test_vectors_5 { + computed := haval.hash_192_5(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_haval_224 :: proc(t: ^testing.T) { - test_vectors_3 := [?]TestHash { - TestHash{"c5aae9d47bffcaaf84a8c6e7ccacd60a0dd1932be7b1a192b9214b6d", ""}, - TestHash{"731814ba5605c59b673e4caae4ad28eeb515b3abc2b198336794e17b", "a"}, - } - for v, _ in test_vectors_3 { - computed := haval.hash_224_3(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_4 := [?]TestHash { - TestHash{"3e56243275b3b81561750550e36fcd676ad2f5dd9e15f2e89e6ed78e", ""}, - TestHash{"742f1dbeeaf17f74960558b44f08aa98bdc7d967e6c0ab8f799b3ac1", "a"}, - } - for v, _ in test_vectors_4 { - computed := haval.hash_224_4(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_5 := [?]TestHash { - TestHash{"4a0513c032754f5582a758d35917ac9adf3854219b39e3ac77d1837e", ""}, - TestHash{"67b3cb8d4068e3641fa4f156e03b52978b421947328bfb9168c7655d", "a"}, - } - for v, _ in test_vectors_5 { - computed := haval.hash_224_5(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors_3 := [?]TestHash { + TestHash{"c5aae9d47bffcaaf84a8c6e7ccacd60a0dd1932be7b1a192b9214b6d", ""}, + TestHash{"731814ba5605c59b673e4caae4ad28eeb515b3abc2b198336794e17b", "a"}, + } + for v, _ in test_vectors_3 { + computed := haval.hash_224_3(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_4 := [?]TestHash { + TestHash{"3e56243275b3b81561750550e36fcd676ad2f5dd9e15f2e89e6ed78e", ""}, + TestHash{"742f1dbeeaf17f74960558b44f08aa98bdc7d967e6c0ab8f799b3ac1", "a"}, + } + for v, _ in test_vectors_4 { + computed := haval.hash_224_4(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_5 := [?]TestHash { + TestHash{"4a0513c032754f5582a758d35917ac9adf3854219b39e3ac77d1837e", ""}, + TestHash{"67b3cb8d4068e3641fa4f156e03b52978b421947328bfb9168c7655d", "a"}, + } + for v, _ in test_vectors_5 { + computed := haval.hash_224_5(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } @(test) test_haval_256 :: proc(t: ^testing.T) { - test_vectors_3 := [?]TestHash { - TestHash{"4f6938531f0bc8991f62da7bbd6f7de3fad44562b8c6f4ebf146d5b4e46f7c17", ""}, - TestHash{"47c838fbb4081d9525a0ff9b1e2c05a98f625714e72db289010374e27db021d8", "a"}, - } - for v, _ in test_vectors_3 { - computed := haval.hash_256_3(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_4 := [?]TestHash { - TestHash{"c92b2e23091e80e375dadce26982482d197b1a2521be82da819f8ca2c579b99b", ""}, - TestHash{"e686d2394a49b44d306ece295cf9021553221db132b36cc0ff5b593d39295899", "a"}, - } - for v, _ in test_vectors_4 { - computed := haval.hash_256_4(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } - test_vectors_5 := [?]TestHash { - TestHash{"be417bb4dd5cfb76c7126f4f8eeb1553a449039307b1a3cd451dbfdc0fbbe330", ""}, - TestHash{"de8fd5ee72a5e4265af0a756f4e1a1f65c9b2b2f47cf17ecf0d1b88679a3e22f", "a"}, - } - for v, _ in test_vectors_5 { - computed := haval.hash_256_5(v.str) - computed_str := hex_string(computed[:]) - expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) - } + test_vectors_3 := [?]TestHash { + TestHash{"4f6938531f0bc8991f62da7bbd6f7de3fad44562b8c6f4ebf146d5b4e46f7c17", ""}, + TestHash{"47c838fbb4081d9525a0ff9b1e2c05a98f625714e72db289010374e27db021d8", "a"}, + } + for v, _ in test_vectors_3 { + computed := haval.hash_256_3(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_4 := [?]TestHash { + TestHash{"c92b2e23091e80e375dadce26982482d197b1a2521be82da819f8ca2c579b99b", ""}, + TestHash{"e686d2394a49b44d306ece295cf9021553221db132b36cc0ff5b593d39295899", "a"}, + } + for v, _ in test_vectors_4 { + computed := haval.hash_256_4(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } + test_vectors_5 := [?]TestHash { + TestHash{"be417bb4dd5cfb76c7126f4f8eeb1553a449039307b1a3cd451dbfdc0fbbe330", ""}, + TestHash{"de8fd5ee72a5e4265af0a756f4e1a1f65c9b2b2f47cf17ecf0d1b88679a3e22f", "a"}, + } + for v, _ in test_vectors_5 { + computed := haval.hash_256_5(v.str) + computed_str := hex_string(computed[:]) + expect(t, computed_str == v.hash, fmt.tprintf("Expected: %s for input of %s, but got %s instead", v.hash, v.str, computed_str)) + } } diff --git a/tests/core/encoding/json/test_core_json.odin b/tests/core/encoding/json/test_core_json.odin index 4f415c008..6f2e8c35a 100644 --- a/tests/core/encoding/json/test_core_json.odin +++ b/tests/core/encoding/json/test_core_json.odin @@ -12,18 +12,15 @@ when ODIN_TEST { log :: testing.log } else { expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) TEST_count += 1 if !condition { TEST_fail += 1 - fmt.println(message) + fmt.printf("[%v] %v\n", loc, message) return } - fmt.println(" PASS") } log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) + fmt.printf("[%v] LOG:\n\t%v\n", loc, v) } } @@ -33,7 +30,7 @@ main :: proc() { parse_json(&t) marshal_json(&t) - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("\n%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) } @test diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin index 5cb59e001..f9f7a2992 100644 --- a/tests/core/encoding/xml/test_core_xml.odin +++ b/tests/core/encoding/xml/test_core_xml.odin @@ -173,23 +173,20 @@ TESTS :: []TEST{ } when ODIN_TEST { - expect :: testing.expect - log :: testing.log + expect :: testing.expect + log :: testing.log } else { - expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) - TEST_count += 1 - if !condition { - TEST_fail += 1 - fmt.println(message) - return - } - fmt.println(" PASS") - } - log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) - } + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] LOG:\n\t%v\n", loc, v) + } } test_file_path :: proc(filename: string) -> (path: string) { @@ -229,7 +226,7 @@ doc_to_string :: proc(doc: ^xml.Document) -> (result: string) { written += wprintf(writer, "[DOCTYPE] %v\n", doc.doctype.ident) if len(doc.doctype.rest) > 0 { - wprintf(writer, "\t%v\n", doc.doctype.rest) + wprintf(writer, "\t%v\n", doc.doctype.rest) } } @@ -238,9 +235,9 @@ doc_to_string :: proc(doc: ^xml.Document) -> (result: string) { } if doc.root != nil { - wprintln(writer, " --- ") - print_element(writer, doc.root) - wprintln(writer, " --- ") + wprintln(writer, " --- ") + print_element(writer, doc.root) + wprintln(writer, " --- ") } return written, .None @@ -293,7 +290,7 @@ run_tests :: proc(t: ^testing.T) { for test in TESTS { path := test_file_path(test.filename) - printf("\nTrying to parse %v\n\n", path) + log(t, fmt.tprintf("Trying to parse %v", path)) doc, err := xml.parse(path, test.options, Silent) defer xml.destroy(doc) @@ -323,7 +320,7 @@ run_tests :: proc(t: ^testing.T) { } main :: proc() { - t := testing.T{} + t := testing.T{} track: mem.Tracking_Allocator mem.tracking_allocator_init(&track, context.allocator) @@ -338,5 +335,5 @@ main :: proc() { } } - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("\n%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) } \ No newline at end of file diff --git a/tests/core/hash/test_core_hash.odin b/tests/core/hash/test_core_hash.odin index 8baa604b6..b81af01a5 100644 --- a/tests/core/hash/test_core_hash.odin +++ b/tests/core/hash/test_core_hash.odin @@ -14,18 +14,15 @@ when ODIN_TEST { log :: testing.log } else { expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) TEST_count += 1 if !condition { TEST_fail += 1 - fmt.println(" FAIL:", message) + fmt.printf("[%v] %v", loc, message) return } - fmt.println(" PASS") } log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) + fmt.printf("[%v] LOG:\n\t%v\n", loc, v) } } @@ -34,7 +31,7 @@ main :: proc() { test_benchmark_runner(&t) test_xxhash_vectors(&t) test_crc64_vectors(&t) - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("\n%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) } /* diff --git a/tests/core/image/test_core_image.odin b/tests/core/image/test_core_image.odin index 155b69298..48d9ca1d2 100644 --- a/tests/core/image/test_core_image.odin +++ b/tests/core/image/test_core_image.odin @@ -25,38 +25,49 @@ import "core:time" import "core:runtime" -WRITE_PPM_ON_FAIL :: #config(WRITE_PPM_ON_FAIL, false) -TEST_SUITE_PATH :: "assets/PNG" +WRITE_PPM_ON_FAIL :: #config(WRITE_PPM_ON_FAIL, false) +TEST_FILE_PATH_PREFIX :: "tests/core/assets/PNG" TEST_count := 0 TEST_fail := 0 when ODIN_TEST { - expect :: testing.expect - log :: testing.log + expect :: testing.expect + log :: testing.log } else { - expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) - TEST_count += 1 - if !condition { - TEST_fail += 1 - fmt.println(message) - return - } - fmt.println(" PASS") - } - log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) - } + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] LOG:\n\t%v\n", loc, v) + } } + I_Error :: image.Error main :: proc() { t := testing.T{} png_test(&t) - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("\n%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) +} + +test_file_path :: proc(filename: string, extension := "png") -> (path: string) { + + path = fmt.tprintf("%v%v/%v.%v", ODIN_ROOT, TEST_FILE_PATH_PREFIX, filename, extension) + temp := transmute([]u8)path + + for r, i in path { + if r == '\\' { + temp[i] = '/' + } + } + return path } PNG_Test :: struct { @@ -1461,7 +1472,7 @@ run_png_suite :: proc(t: ^testing.T, suite: []PNG_Test) -> (subtotal: int) { context = runtime.default_context() for file in suite { - test_file := fmt.tprintf("%v/%v.png", TEST_SUITE_PATH, file.file) + test_file := test_file_path(file.file) img: ^png.Image err: png.Error diff --git a/tests/core/odin/test_parser.odin b/tests/core/odin/test_parser.odin index 90d913d10..53711d3ec 100644 --- a/tests/core/odin/test_parser.odin +++ b/tests/core/odin/test_parser.odin @@ -10,34 +10,29 @@ TEST_count := 0 TEST_fail := 0 when ODIN_TEST { - expect :: testing.expect - log :: testing.log + expect :: testing.expect + log :: testing.log } else { - expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) - TEST_count += 1 - if !condition { - TEST_fail += 1 - fmt.println(message) - return - } - fmt.println(" PASS") - } - log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) - } + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] LOG:\n\t%v\n", loc, v) + } } - main :: proc() { - t := testing.T{} - test_parse_demo(&t) + t := testing.T{} + test_parse_demo(&t) - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("\n%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) } - @test test_parse_demo :: proc(t: ^testing.T) { pkg, ok := parser.parse_package_from_path("examples/demo") diff --git a/tests/core/strings/test_core_strings.odin b/tests/core/strings/test_core_strings.odin index fc1518349..c5436f5c1 100644 --- a/tests/core/strings/test_core_strings.odin +++ b/tests/core/strings/test_core_strings.odin @@ -8,56 +8,53 @@ TEST_count := 0 TEST_fail := 0 when ODIN_TEST { - expect :: testing.expect - log :: testing.log + expect :: testing.expect + log :: testing.log } else { - expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { - fmt.printf("[%v] ", loc) - TEST_count += 1 - if !condition { - TEST_fail += 1 - fmt.println(message) - return - } - fmt.println(" PASS") - } - log :: proc(t: ^testing.T, v: any, loc := #caller_location) { - fmt.printf("[%v] ", loc) - fmt.printf("log: %v\n", v) - } + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] LOG:\n\t%v\n", loc, v) + } } main :: proc() { - t := testing.T{} - test_index_any_small_string_not_found(&t) - test_index_any_larger_string_not_found(&t) - test_index_any_small_string_found(&t) - test_index_any_larger_string_found(&t) + t := testing.T{} + test_index_any_small_string_not_found(&t) + test_index_any_larger_string_not_found(&t) + test_index_any_small_string_found(&t) + test_index_any_larger_string_found(&t) - fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + fmt.printf("\n%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) } @test test_index_any_small_string_not_found :: proc(t: ^testing.T) { - index := strings.index_any(".", "/:\"") - log(t, index) - expect(t, index == -1, "index_any should be negative") + index := strings.index_any(".", "/:\"") + log(t, index) + expect(t, index == -1, "index_any should be negative") } @test test_index_any_larger_string_not_found :: proc(t: ^testing.T) { - index := strings.index_any("aaaaaaaa.aaaaaaaa", "/:\"") - expect(t, index == -1, "index_any should be negative") + index := strings.index_any("aaaaaaaa.aaaaaaaa", "/:\"") + expect(t, index == -1, "index_any should be negative") } @test test_index_any_small_string_found :: proc(t: ^testing.T) { - index := strings.index_any(".", "/:.\"") - expect(t, index == 0, "index_any should be 0") + index := strings.index_any(".", "/:.\"") + expect(t, index == 0, "index_any should be 0") } @test test_index_any_larger_string_found :: proc(t: ^testing.T) { - index := strings.index_any("aaaaaaaa:aaaaaaaa", "/:\"") - expect(t, index == 8, "index_any should be 8") + index := strings.index_any("aaaaaaaa:aaaaaaaa", "/:\"") + expect(t, index == 8, "index_any should be 8") } From 7ec88d24302dcdea38ac09996a2279f4de4f6a25 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 5 Dec 2021 21:06:33 +0100 Subject: [PATCH 14/79] [xml] Add option. --- .../entity/example/entity_example.odin | 4 +- core/encoding/xml/example/xml_example.odin | 53 +++++++++++-------- core/encoding/xml/xml_reader.odin | 26 +++++++-- tests/core/encoding/xml/test_core_xml.odin | 10 ++-- 4 files changed, 59 insertions(+), 34 deletions(-) diff --git a/core/encoding/entity/example/entity_example.odin b/core/encoding/entity/example/entity_example.odin index 882203f48..6fc377f9d 100644 --- a/core/encoding/entity/example/entity_example.odin +++ b/core/encoding/entity/example/entity_example.odin @@ -64,8 +64,8 @@ main :: proc() { mem.tracking_allocator_init(&track, context.allocator) context.allocator = mem.tracking_allocator(&track) - _main() - //_entities() + // _main() + _entities() if len(track.allocation_map) > 0 { println() diff --git a/core/encoding/xml/example/xml_example.odin b/core/encoding/xml/example/xml_example.odin index 085252e92..daa3c5dab 100644 --- a/core/encoding/xml/example/xml_example.odin +++ b/core/encoding/xml/example/xml_example.odin @@ -2,35 +2,40 @@ package xml_example import "core:encoding/xml" import "core:os" -import "core:path" import "core:mem" import "core:fmt" - -/* - Silent error handler for the parser. -*/ -Error_Handler :: proc(pos: xml.Pos, fmt: string, args: ..any) {} - -OPTIONS :: xml.Options{ flags = { .Ignore_Unsupported, }, expected_doctype = "unicode", } +import "core:time" +import "core:strings" +import "core:hash" example :: proc() { using fmt - filename := path.join(ODIN_ROOT, "tests", "core", "assets", "XML", "unicode.xml") - defer delete(filename) + doc: ^xml.Document + err: xml.Error - doc, err := xml.parse(filename, OPTIONS, Error_Handler) + DOC :: #load("../../../../tests/core/assets/XML/unicode.xml") + + parse_duration: time.Duration + { + time.SCOPED_TICK_DURATION(&parse_duration) + doc, err = xml.parse(DOC, xml.Options{flags={.Ignore_Unsupported}}) + } defer xml.destroy(doc) + ms := time.duration_milliseconds(parse_duration) + speed := (f64(1000.0) / ms) * f64(len(DOC)) / 1_024.0 / 1_024.0 + fmt.printf("Parse time: %v bytes in %.2f ms (%.2f MiB/s).\n", len(DOC), ms, speed) + if err != .None { printf("Load/Parse error: %v\n", err) if err == .File_Error { - printf("\"%v\" not found. Did you run \"tests\\download_assets.py\"?", filename) + println("\"unicode.xml\" not found. Did you run \"tests\\download_assets.py\"?") } os.exit(1) } - printf("\"%v\" loaded and parsed.\n", filename) + println("\"unicode.xml\" loaded and parsed.") charlist, charlist_ok := xml.find_child_by_ident(doc.root, "charlist") if !charlist_ok { @@ -40,17 +45,19 @@ example :: proc() { printf("Found `` with %v children.\n", len(charlist.children)) - for char in charlist.children { - if char.ident != "character" { - eprintf("Expected ``, got `<%v>`\n", char.ident) - os.exit(1) - } + crc32 := doc_hash(doc) + printf("[%v] CRC32: 0x%08x\n", "🎉" if crc32 == 0xcaa042b9 else "🤬", crc32) +} - if _, ok := xml.find_attribute_val_by_key(char, "dec"); !ok { - eprintln("`` attribute not found.") - os.exit(1) - } - } +doc_hash :: proc(doc: ^xml.Document, print := false) -> (crc32: u32) { + buf: strings.Builder + defer strings.destroy_builder(&buf) + w := strings.to_writer(&buf) + + xml.print(w, doc) + tree := strings.to_string(buf) + if print { fmt.println(tree) } + return hash.crc32(transmute([]u8)tree) } main :: proc() { diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 6f49b8e08..b169bd57a 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -71,6 +71,12 @@ Option_Flag :: enum { This option decodes them when encountered. */ Decode_SGML_Entities, + + /* + If a tag body has a comment, it will be stripped unless this option is given. + */ + Keep_Tag_Body_Comments, + } Option_Flags :: bit_set[Option_Flag; u8] @@ -413,15 +419,29 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err /* This should be a tag's body text. */ - body_text := scan_string(t, t.offset) or_return + body_text := scan_string(t, t.offset) or_return + needs_processing := .Unbox_CDATA in opts.flags + needs_processing |= .Decode_SGML_Entities in opts.flags - decode_opts := entity.XML_Decode_Options{ .Comment_Strip } + if !needs_processing { + element.value = strings.intern_get(&doc.intern, body_text) + continue + } + + decode_opts := entity.XML_Decode_Options{} + if .Keep_Tag_Body_Comments not_in opts.flags { + decode_opts += { .Comment_Strip } + } if .Decode_SGML_Entities not_in opts.flags { decode_opts += { .No_Entity_Decode } } + if .Unbox_CDATA in opts.flags { - decode_opts += { .Unbox_CDATA, .Decode_CDATA } + decode_opts += { .Unbox_CDATA } + if .Decode_SGML_Entities in opts.flags { + decode_opts += { .Decode_CDATA } + } } decoded, decode_err := entity.decode_xml(body_text, decode_opts) diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin index f9f7a2992..7669afe97 100644 --- a/tests/core/encoding/xml/test_core_xml.odin +++ b/tests/core/encoding/xml/test_core_xml.odin @@ -8,9 +8,7 @@ import "core:io" import "core:fmt" import "core:hash" -Silent :: proc(pos: xml.Pos, fmt: string, args: ..any) { - // Custom (silent) error handler. -} +Silent :: proc(pos: xml.Pos, format: string, args: ..any) {} OPTIONS :: xml.Options{ flags = { .Ignore_Unsupported, .Intern_Comments, }, expected_doctype = "", @@ -75,7 +73,7 @@ TESTS :: []TEST{ }, expected_doctype = "恥ずべきフクロウ", }, - crc32 = 0x6d38ac58, + crc32 = 0xad31d8e8, }, { @@ -131,7 +129,7 @@ TESTS :: []TEST{ }, expected_doctype = "html", }, - crc32 = 0xdb4a1e79, + crc32 = 0x573c1033, }, { @@ -306,7 +304,7 @@ run_tests :: proc(t: ^testing.T) { expect(t, err == test.err, err_msg) failed |= crc32 != test.crc32 - err_msg = tprintf("Expected CRC 0x%08x, got 0x%08x", test.crc32, crc32) + err_msg = tprintf("Expected CRC 0x%08x, got 0x%08x, with options %v", test.crc32, crc32, test.options) expect(t, crc32 == test.crc32, err_msg) if failed { From 6e61abc7d06f22129f93110a9f652c3eec21f0c6 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Mon, 6 Dec 2021 12:04:59 +0100 Subject: [PATCH 15/79] [xml] Initial optimization. --- core/encoding/xml/xml_reader.odin | 68 +++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index b169bd57a..0315b0e05 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -26,6 +26,7 @@ package xml Jeroen van Rijn: Initial implementation. */ +import "core:bytes" import "core:strings" import "core:encoding/entity" import "core:mem" @@ -39,6 +40,12 @@ DEFAULT_Options :: Options{ } Option_Flag :: enum { + /* + If the caller says that input may be modified, we can perform in-situ parsing. + If this flag isn't provided, the XML parser first duplicates the input so that it can. + */ + Input_May_Be_Modified, + /* Document MUST start with ` (doc: ^Document, err: Error) { + data := data context.allocator = allocator opts := validate_options(options) or_return + /* + If `.Input_May_Be_Modified` is not specified, we duplicate the input so that we can modify it in-place. + */ + if .Input_May_Be_Modified not_in opts.flags { + data = bytes.clone(data) + } + t := &Tokenizer{} init(t, string(data), path, error_handler) doc = new(Document) doc.allocator = allocator doc.tokenizer = t + doc.input = data - strings.intern_init(&doc.intern, allocator, allocator) + // strings.intern_init(&doc.intern, allocator, allocator) - err = .Unexpected_Token - element, parent: ^Element + err = .Unexpected_Token + element, parent: ^Element tag_is_open := false @@ -292,8 +313,7 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err case: if .Error_on_Unsupported in opts.flags { error(t, t.offset, "Unhandled: (doc: ^Document, err: Error) { context.allocator = allocator + options := options data, data_ok := os.read_entire_file(filename) - defer delete(data) - if !data_ok { return {}, .File_Error } + options.flags += { .Input_May_Be_Modified } + return parse_from_slice(data, options, filename, error_handler, allocator) } @@ -499,10 +517,16 @@ destroy :: proc(doc: ^Document) { if doc == nil { return } free_element(doc.root) - strings.intern_destroy(&doc.intern) delete(doc.prolog) delete(doc.comments) + delete(doc.input) + + for s in doc.strings_to_free { + delete(s) + } + delete(doc.strings_to_free) + free(doc) } @@ -538,8 +562,8 @@ parse_attribute :: proc(doc: ^Document) -> (attr: Attr, offset: int, err: Error) _ = expect(t, .Eq) or_return value := expect(t, .String) or_return - attr.key = strings.intern_get(&doc.intern, key.text) - attr.val = strings.intern_get(&doc.intern, value.text) + attr.key = key.text + attr.val = value.text err = .None return @@ -651,7 +675,7 @@ parse_doctype :: proc(doc: ^Document) -> (err: Error) { t := doc.tokenizer tok := expect(t, .Ident) or_return - doc.doctype.ident = strings.intern_get(&doc.intern, tok.text) + doc.doctype.ident = tok.text skip_whitespace(t) offset := t.offset @@ -660,6 +684,6 @@ parse_doctype :: proc(doc: ^Document) -> (err: Error) { /* -1 because the current offset is that of the closing tag, so the rest of the DOCTYPE tag ends just before it. */ - doc.doctype.rest = strings.intern_get(&doc.intern, string(t.src[offset : t.offset - 1])) + doc.doctype.rest = string(t.src[offset : t.offset - 1]) return .None } \ No newline at end of file From fdd24f787fd6df5c1f3339d6317c76a1bafe60f5 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Mon, 18 Apr 2022 23:28:34 +0200 Subject: [PATCH 16/79] [image/tga] Writer for RGB(A) 8-bit images. --- core/image/common.odin | 14 ++++++ core/image/tga/tga.odin | 103 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 core/image/tga/tga.odin diff --git a/core/image/common.odin b/core/image/common.odin index 2e7bca17e..8c77ec48a 100644 --- a/core/image/common.odin +++ b/core/image/common.odin @@ -320,6 +320,20 @@ QOI_Info :: struct { header: QOI_Header, } +TGA_Header :: struct #packed { + id_length: u8, + color_map_type: u8, + data_type_code: u8, + color_map_origin: u16le, + color_map_length: u16le, + color_map_depth: u8, + origin: [2]u16le, + dimensions: [2]u16le, + bits_per_pixel: u8, + image_descriptor: u8, +} +#assert(size_of(TGA_Header) == 18) + // Function to help with image buffer calculations compute_buffer_size :: proc(width, height, channels, depth: int, extra_row_bytes := int(0)) -> (size: int) { size = ((((channels * width * depth) + 7) >> 3) + extra_row_bytes) * height diff --git a/core/image/tga/tga.odin b/core/image/tga/tga.odin new file mode 100644 index 000000000..3c860cb62 --- /dev/null +++ b/core/image/tga/tga.odin @@ -0,0 +1,103 @@ +/* + Copyright 2022 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + List of contributors: + Jeroen van Rijn: Initial implementation. +*/ + + +// package tga implements a TGA image writer for 8-bit RGB and RGBA images. +package tga + +import "core:mem" +import "core:image" +import "core:compress" +import "core:bytes" +import "core:os" + +Error :: image.Error +General :: compress.General_Error +Image :: image.Image +Options :: image.Options + +RGB_Pixel :: image.RGB_Pixel +RGBA_Pixel :: image.RGBA_Pixel + +save_to_memory :: proc(output: ^bytes.Buffer, img: ^Image, options := Options{}, allocator := context.allocator) -> (err: Error) { + context.allocator = allocator + + if img == nil { + return .Invalid_Input_Image + } + + if output == nil { + return .Invalid_Output + } + + pixels := img.width * img.height + if pixels == 0 || pixels > image.MAX_DIMENSIONS || img.width > 65535 || img.height > 65535 { + return .Invalid_Input_Image + } + + // Our TGA writer supports only 8-bit images with 3 or 4 channels. + if img.depth != 8 || img.channels < 3 || img.channels > 4 { + return .Invalid_Input_Image + } + + if img.channels * pixels != len(img.pixels.buf) { + return .Invalid_Input_Image + } + + written := 0 + + // Calculate and allocate necessary space. + necessary := pixels * img.channels + size_of(image.TGA_Header) + + if !resize(&output.buf, necessary) { + return General.Resize_Failed + } + + header := image.TGA_Header{ + data_type_code = 0x02, // Color, uncompressed. + dimensions = {u16le(img.width), u16le(img.height)}, + bits_per_pixel = u8(img.depth * img.channels), + image_descriptor = 1 << 5, // Origin is top left. + } + header_bytes := transmute([size_of(image.TGA_Header)]u8)header + + copy(output.buf[written:], header_bytes[:]) + written += size_of(image.TGA_Header) + + /* + Encode loop starts here. + */ + if img.channels == 3 { + pix := mem.slice_data_cast([]RGB_Pixel, img.pixels.buf[:]) + out := mem.slice_data_cast([]RGB_Pixel, output.buf[written:]) + for p, i in pix { + out[i] = p.bgr + } + } else if img.channels == 4 { + pix := mem.slice_data_cast([]RGBA_Pixel, img.pixels.buf[:]) + out := mem.slice_data_cast([]RGBA_Pixel, output.buf[written:]) + for p, i in pix { + out[i] = p.bgra + } + } + return nil +} + +save_to_file :: proc(output: string, img: ^Image, options := Options{}, allocator := context.allocator) -> (err: Error) { + context.allocator = allocator + + out := &bytes.Buffer{} + defer bytes.buffer_destroy(out) + + save_to_memory(out, img, options) or_return + write_ok := os.write_entire_file(output, out.buf[:]) + + return nil if write_ok else General.Cannot_Open_File +} + +save :: proc{save_to_memory, save_to_file} \ No newline at end of file From aa4eb35671c519c520bd25ff90837c1d70558c6b Mon Sep 17 00:00:00 2001 From: hikari Date: Tue, 19 Apr 2022 05:58:22 +0300 Subject: [PATCH 17/79] sys/windows: add some procedures --- core/sys/windows/kernel32.odin | 7 +++++++ core/sys/windows/user32.odin | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/core/sys/windows/kernel32.odin b/core/sys/windows/kernel32.odin index 735e065e2..cb90f71da 100644 --- a/core/sys/windows/kernel32.odin +++ b/core/sys/windows/kernel32.odin @@ -62,6 +62,13 @@ foreign kernel32 { GetCurrentProcessId :: proc() -> DWORD --- GetCurrentThread :: proc() -> HANDLE --- GetCurrentThreadId :: proc() -> DWORD --- + GetProcessTimes :: proc( + hProcess: HANDLE, + lpCreationTime: LPFILETIME, + lpExitTime: LPFILETIME, + lpKernelTime: LPFILETIME, + lpUserTime: LPFILETIME, + ) -> BOOL --- GetStdHandle :: proc(which: DWORD) -> HANDLE --- ExitProcess :: proc(uExitCode: c_uint) -> ! --- DeviceIoControl :: proc( diff --git a/core/sys/windows/user32.odin b/core/sys/windows/user32.odin index 2316d3363..dd45df42a 100644 --- a/core/sys/windows/user32.odin +++ b/core/sys/windows/user32.odin @@ -60,6 +60,12 @@ foreign user32 { DestroyWindow :: proc(hWnd: HWND) -> BOOL --- ShowWindow :: proc(hWnd: HWND, nCmdShow: c_int) -> BOOL --- + BringWindowToTop :: proc(hWnd: HWND) -> BOOL --- + GetTopWindow :: proc(hWnd: HWND) -> HWND --- + SetForegroundWindow :: proc(hWnd: HWND) -> BOOL --- + GetForegroundWindow :: proc() -> HWND --- + SetActiveWindow :: proc(hWnd: HWND) -> HWND --- + GetActiveWindow :: proc() -> HWND --- GetMessageA :: proc(lpMsg: ^MSG, hWnd: HWND, wMsgFilterMin: UINT, wMsgFilterMax: UINT) -> BOOL --- GetMessageW :: proc(lpMsg: ^MSG, hWnd: HWND, wMsgFilterMin: UINT, wMsgFilterMax: UINT) -> BOOL --- From ded8342f3fdb176d6c264057f3bdcb890bfcfad7 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Tue, 19 Apr 2022 20:46:33 +0800 Subject: [PATCH 18/79] Reduce allocations --- core/container/lru/lru_cache.odin | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/core/container/lru/lru_cache.odin b/core/container/lru/lru_cache.odin index 81f0142b0..b59f29f0c 100644 --- a/core/container/lru/lru_cache.odin +++ b/core/container/lru/lru_cache.odin @@ -65,20 +65,22 @@ set :: proc(c: ^$C/Cache($Key, $Value), key: Key, value: Value) -> runtime.Alloc return nil } - e := new(Node(Key, Value), c.node_allocator) or_return - e.key = key - e.value = value - + e : ^Node(Key, Value) = nil assert(c.count <= c.capacity) if c.count == c.capacity { - _remove_node(c, c.tail) + e = c.tail + _remove_node(c, e) } else { c.count += 1 + e = new(Node(Key, Value), c.node_allocator) or_return } - _push_front_node(c, e) + e.key = key + e.value = value + _push_front_node(c, e) c.entries[key] = e + return nil } @@ -128,6 +130,7 @@ remove :: proc(c: ^$C/Cache($Key, $Value), key: Key) -> bool { return false } _remove_node(c, e) + free(node, c.node_allocator) c.count -= 1 return true } @@ -153,9 +156,6 @@ _remove_node :: proc(c: ^$C/Cache($Key, $Value), node: ^Node(Key, Value)) { delete_key(&c.entries, node.key) _call_on_remove(c, node) - - free(node, c.node_allocator) - } @(private) From 7654afc2db9e88b1803862310698ca047bf10a01 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 19 Apr 2022 15:01:54 +0200 Subject: [PATCH 19/79] Revert "Update `mem.nil_allocator` to match the same in `runtime`" The change broke JSON unmarshaling. This reverts commit 4484a3433d6c58f1d1c594a4c36317f323cb5102. --- core/mem/allocators.odin | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/core/mem/allocators.odin b/core/mem/allocators.odin index 4954122ed..b8bd9a065 100644 --- a/core/mem/allocators.odin +++ b/core/mem/allocators.odin @@ -6,24 +6,7 @@ import "core:runtime" nil_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, size, alignment: int, old_memory: rawptr, old_size: int, loc := #caller_location) -> ([]byte, Allocator_Error) { - switch mode { - case .Alloc: - return nil, .Out_Of_Memory - case .Free: - return nil, .None - case .Free_All: - return nil, .Mode_Not_Implemented - case .Resize: - if size == 0 { - return nil, .None - } - return nil, .Out_Of_Memory - case .Query_Features: - return nil, .Mode_Not_Implemented - case .Query_Info: - return nil, .Mode_Not_Implemented - } - return nil, .None + return nil, nil } nil_allocator :: proc() -> Allocator { From 323e7a2d02819ef0b7137abbdfee8a0d48f5cecb Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 19 Apr 2022 15:03:09 +0200 Subject: [PATCH 20/79] Add JSON unmarshal test. --- tests/core/encoding/json/test_core_json.odin | 253 +++++++++++++++++++ 1 file changed, 253 insertions(+) diff --git a/tests/core/encoding/json/test_core_json.odin b/tests/core/encoding/json/test_core_json.odin index 285cc04a1..c83710352 100644 --- a/tests/core/encoding/json/test_core_json.odin +++ b/tests/core/encoding/json/test_core_json.odin @@ -31,6 +31,7 @@ main :: proc() { parse_json(&t) marshal_json(&t) + unmarshal_json(&t) fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) if TEST_fail > 0 { @@ -90,3 +91,255 @@ marshal_json :: proc(t: ^testing.T) { expect(t, err == nil, "expected json error to be none") } + +PRODUCTS := ` +{ + "cash": "0", + "products": [ + { + "name": "Cog Cola", + "cost": "3", + "owned": "1", + + "profit": "4", + "seconds": 3, + "multiplier": 1, + "auto_click": false + }, + { + "name": "gingerBeer", + "cost": "9", + "owned": "0", + + "profit": "16", + "seconds": 5, + "multiplier": 1, + "auto_click": false + }, + { + "name": "Coffee", + "cost": "27", + "owned": "0", + + "profit": "64", + "seconds": 7, + "multiplier": 1, + "auto_click": false + }, + { + "name": "Haggis", + "cost": "81", + "owned": "0", + + "profit": "256", + "seconds": 11, + "multiplier": 1, + "auto_click": false + }, + { + "name": "Lasagna", + "cost": "243", + "owned": "0", + + "profit": "1024", + "seconds": 13, + "multiplier": 1, + "auto_click": false + }, + { + "name": "Asparagus", + "cost": "729", + "owned": "0", + + "profit": "4096", + "seconds": 17, + "multiplier": 1, + "auto_click": false + }, + { + "name": "Yorkshire Pudding", + "cost": "2187", + "owned": "0", + + "profit": "16384", + "seconds": 19, + "multiplier": 1, + "auto_click": false + }, + { + "name": "Salmon Wrap", + "cost": "6561", + "owned": "0", + + "profit": "65536", + "seconds": 23, + "multiplier": 1, + "auto_click": false + }, + { + "name": "Poke Bowl", + "cost": "19683", + "owned": "0", + + "profit": "262144", + "seconds": 29, + "multiplier": 1, + "auto_click": false + }, + { + "name": "Chili Con Carne", + "cost": "59049", + "owned": "0", + + "profit": "1048576", + "seconds": 59, + "multiplier": 1, + "auto_click": false + }, + ], +} +` + +original_data := Game_Marshal{ + cash = "0", + products = { + { + name = "Cog Cola", + cost = "3", + owned = "1", + profit = "4", + seconds = 3, + multiplier = 1, + auto_click = false, + }, + { + name = "gingerBeer", + cost = "9", + owned = "0", + profit = "16", + seconds = 5, + multiplier = 1, + auto_click = false, + }, + { + name = "Coffee", + cost = "27", + owned = "0", + profit = "64", + seconds = 7, + multiplier = 1, + auto_click = false, + }, + { + name = "Haggis", + cost = "81", + owned = "0", + profit = "256", + seconds = 11, + multiplier = 1, + auto_click = false, + }, + { + name = "Lasagna", + cost = "243", + owned = "0", + profit = "1024", + seconds = 13, + multiplier = 1, + auto_click = false, + }, + { + name = "Asparagus", + cost = "729", + owned = "0", + profit = "4096", + seconds = 17, + multiplier = 1, + auto_click = false, + }, + { + name = "Yorkshire Pudding", + cost = "2187", + owned = "0", + profit = "16384", + seconds = 19, + multiplier = 1, + auto_click = false, + }, + { + name = "Salmon Wrap", + cost = "6561", + owned = "0", + profit = "65536", + seconds = 23, + multiplier = 1, + auto_click = false, + }, + { + name = "Poke Bowl", + cost = "19683", + owned = "0", + profit = "262144", + seconds = 29, + multiplier = 1, + auto_click = false, + }, + { + name = "Chili Con Carne", + cost = "59049", + owned = "0", + profit = "1048576", + seconds = 59, + multiplier = 1, + auto_click = false, + }, + }, +} + +Product_Marshal :: struct { + name: cstring, + owned: string, + + cost: string, + + profit: string, + seconds: int, + multiplier: int, + + auto_click: bool, +} + +Game_Marshal :: struct { + cash: string, + products: []Product_Marshal, +} + +cleanup :: proc(g: Game_Marshal) { + for p in g.products { + delete(p.name) + delete(p.owned) + delete(p.cost) + delete(p.profit) + } + delete(g.products) + delete(g.cash) +} + +@test +unmarshal_json :: proc(t: ^testing.T) { + g: Game_Marshal + err := json.unmarshal(transmute([]u8)PRODUCTS, &g, json.DEFAULT_SPECIFICATION) + defer cleanup(g) + + expect(t, err == nil, "Expected json error to be nil") + + msg := fmt.tprintf("Expected %v products to have been unmarshaled, got %v", len(original_data.products), len(g.products)) + expect(t, len(g.products) == len(original_data.products), msg) + + msg = fmt.tprintf("Expected cash to have been unmarshaled as %v, got %v", original_data.cash, g.cash) + expect(t, original_data.cash == g.cash, "Cash unmarshaled improperly") + + for p, i in g.products { + expect(t, p == original_data.products[i], "Producted unmarshaled improperly") + } +} \ No newline at end of file From 581255bf23af90b77bb2b6e2671b40e2b565754e Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 19 Apr 2022 20:04:38 +0200 Subject: [PATCH 21/79] Fix unmarshal for escaped strings. --- core/encoding/json/parser.odin | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index c682ec9bd..0b9a1cf33 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -281,6 +281,11 @@ parse_object :: proc(p: ^Parser) -> (value: Value, err: Error) { // IMPORTANT NOTE(bill): unquote_string assumes a mostly valid string unquote_string :: proc(token: Token, spec: Specification, allocator := context.allocator) -> (value: string, err: Error) { + if allocator.data == nil { + // We were called from `unmarshal_count_array`, return early. + return "", nil + } + get_u2_rune :: proc(s: string) -> rune { if len(s) < 4 || s[0] != '\\' || s[1] != 'x' { return -1 From 29b2c0476698d0f4b240e87945cfa278da82b57a Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 19 Apr 2022 20:11:02 +0200 Subject: [PATCH 22/79] Revert "Fix unmarshal for escaped strings." This reverts commit 581255bf23af90b77bb2b6e2671b40e2b565754e. --- core/encoding/json/parser.odin | 5 ----- 1 file changed, 5 deletions(-) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index 0b9a1cf33..c682ec9bd 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -281,11 +281,6 @@ parse_object :: proc(p: ^Parser) -> (value: Value, err: Error) { // IMPORTANT NOTE(bill): unquote_string assumes a mostly valid string unquote_string :: proc(token: Token, spec: Specification, allocator := context.allocator) -> (value: string, err: Error) { - if allocator.data == nil { - // We were called from `unmarshal_count_array`, return early. - return "", nil - } - get_u2_rune :: proc(s: string) -> rune { if len(s) < 4 || s[0] != '\\' || s[1] != 'x' { return -1 From a30b9b17b3a91bc856a037c1e1025e389a8524b3 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 19 Apr 2022 20:32:22 +0200 Subject: [PATCH 23/79] [json/unmarshal] Fix quoted strings. --- core/encoding/json/parser.odin | 6 ++++++ tests/core/encoding/json/test_core_json.odin | 20 +++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index c682ec9bd..7bf88c565 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -354,6 +354,12 @@ unquote_string :: proc(token: Token, spec: Specification, allocator := context.a b := bytes_make(len(s) + 2*utf8.UTF_MAX, 1, allocator) or_return w := copy(b, s[0:i]) + + if len(b) == 0 && allocator.data == nil { + // `unmarshal_count_array` calls us with a nil allocator + return string(b[:w]), nil + } + loop: for i < len(s) { c := s[i] switch { diff --git a/tests/core/encoding/json/test_core_json.odin b/tests/core/encoding/json/test_core_json.odin index c83710352..0e6a6412f 100644 --- a/tests/core/encoding/json/test_core_json.odin +++ b/tests/core/encoding/json/test_core_json.odin @@ -71,7 +71,8 @@ parse_json :: proc(t: ^testing.T) { _, err := json.parse(transmute([]u8)json_data) - expect(t, err == .None, "expected json error to be none") + msg := fmt.tprintf("Expected `json.parse` to return nil, got %v", err) + expect(t, err == nil, msg) } @test @@ -88,8 +89,8 @@ marshal_json :: proc(t: ^testing.T) { } _, err := json.marshal(my_struct) - - expect(t, err == nil, "expected json error to be none") + msg := fmt.tprintf("Expected `json.marshal` to return nil, got %v", err) + expect(t, err == nil, msg) } PRODUCTS := ` @@ -97,7 +98,7 @@ PRODUCTS := ` "cash": "0", "products": [ { - "name": "Cog Cola", + "name": "Cog\nCola", "cost": "3", "owned": "1", @@ -204,7 +205,7 @@ original_data := Game_Marshal{ cash = "0", products = { { - name = "Cog Cola", + name = "Cog\nCola", cost = "3", owned = "1", profit = "4", @@ -331,13 +332,14 @@ unmarshal_json :: proc(t: ^testing.T) { err := json.unmarshal(transmute([]u8)PRODUCTS, &g, json.DEFAULT_SPECIFICATION) defer cleanup(g) - expect(t, err == nil, "Expected json error to be nil") + msg := fmt.tprintf("Expected `json.unmarshal` to return nil, got %v", err) + expect(t, err == nil, msg) - msg := fmt.tprintf("Expected %v products to have been unmarshaled, got %v", len(original_data.products), len(g.products)) + msg = fmt.tprintf("Expected %v products to have been unmarshaled, got %v", len(original_data.products), len(g.products)) expect(t, len(g.products) == len(original_data.products), msg) - msg = fmt.tprintf("Expected cash to have been unmarshaled as %v, got %v", original_data.cash, g.cash) - expect(t, original_data.cash == g.cash, "Cash unmarshaled improperly") + msg = fmt.tprintf("Expected cash to have been unmarshaled as %v, got %v", original_data.cash, g.cash) + expect(t, original_data.cash == g.cash, msg) for p, i in g.products { expect(t, p == original_data.products[i], "Producted unmarshaled improperly") From eee97f7f62bbd65dd03ea3ec8668fef3fcfc685c Mon Sep 17 00:00:00 2001 From: hikari Date: Thu, 21 Apr 2022 20:49:32 +0300 Subject: [PATCH 24/79] strings: add levenshtein_distance procedure --- core/strings/strings.odin | 59 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/core/strings/strings.odin b/core/strings/strings.odin index 8e774b367..87bbb42cf 100644 --- a/core/strings/strings.odin +++ b/core/strings/strings.odin @@ -1809,3 +1809,62 @@ fields_iterator :: proc(s: ^string) -> (field: string, ok: bool) { s^ = s[len(s):] return } + +// `levenshtein_distance` returns the Levenshtein edit distance between 2 strings. +// This is a single-row-version of the Wagner–Fischer algorithm, based on C code by Martin Ettl. +// Note: allocator isn't used if the length of string b in runes is smaller than 70. +levenshtein_distance :: proc(a, b: string, allocator := context.allocator) -> int { + LEVENSHTEIN_DEFAULT_COSTS: []int : { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + } + + m, n := utf8.rune_count_in_string(a), utf8.rune_count_in_string(b) + + if m == 0 do return n + if n == 0 do return m + + costs: []int + + if n + 1 > len(LEVENSHTEIN_DEFAULT_COSTS) { + costs = make([]int, n + 1, allocator) + } else { + costs = LEVENSHTEIN_DEFAULT_COSTS + } + + defer if n + 1 > len(LEVENSHTEIN_DEFAULT_COSTS) { + delete(costs, allocator) + } + + for k in 0..=n { + costs[k] = k + } + + i: int + for c1 in a { + costs[0] = i + 1 + corner := i + j: int + for c2 in b { + upper := costs[j + 1] + if c1 == c2 { + costs[j + 1] = corner + } else { + t := upper if upper < corner else corner + costs[j + 1] = (costs[j] if costs[j] < t else t) + 1 + } + + corner = upper + j += 1 + } + + i += 1 + } + + return costs[n] +} From 591732f347c094e706471994996fcffa44687e89 Mon Sep 17 00:00:00 2001 From: hikari Date: Thu, 21 Apr 2022 20:58:50 +0300 Subject: [PATCH 25/79] strings: levenshtein_distance: remove costs calculation for default array --- core/strings/strings.odin | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/strings/strings.odin b/core/strings/strings.odin index 87bbb42cf..3b33a5dc8 100644 --- a/core/strings/strings.odin +++ b/core/strings/strings.odin @@ -1833,6 +1833,9 @@ levenshtein_distance :: proc(a, b: string, allocator := context.allocator) -> in if n + 1 > len(LEVENSHTEIN_DEFAULT_COSTS) { costs = make([]int, n + 1, allocator) + for k in 0..=n { + costs[k] = k + } } else { costs = LEVENSHTEIN_DEFAULT_COSTS } @@ -1841,10 +1844,6 @@ levenshtein_distance :: proc(a, b: string, allocator := context.allocator) -> in delete(costs, allocator) } - for k in 0..=n { - costs[k] = k - } - i: int for c1 in a { costs[0] = i + 1 From d8f0da164b92f52eda045d77e5f6abb5c5146f2a Mon Sep 17 00:00:00 2001 From: hikari Date: Thu, 21 Apr 2022 21:15:11 +0300 Subject: [PATCH 26/79] strings: levenshtein_distance: improve potential caching --- core/strings/strings.odin | 43 ++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/core/strings/strings.odin b/core/strings/strings.odin index 3b33a5dc8..f876aab3d 100644 --- a/core/strings/strings.odin +++ b/core/strings/strings.odin @@ -15,7 +15,7 @@ clone :: proc(s: string, allocator := context.allocator, loc := #caller_location } // returns a clone of the string `s` allocated using the `allocator` as a cstring -// a nul byte is appended to the clone, to make the cstring safe +// a nul byte is appended to the clone, to make the cstring safe clone_to_cstring :: proc(s: string, allocator := context.allocator, loc := #caller_location) -> cstring { c := make([]byte, len(s)+1, allocator, loc) copy(c, s) @@ -37,7 +37,7 @@ string_from_nul_terminated_ptr :: proc(ptr: ^byte, len: int) -> string { return s } -// returns the raw ^byte start of the string `str` +// returns the raw ^byte start of the string `str` ptr_from_string :: proc(str: string) -> ^byte { d := transmute(mem.Raw_String)str return d.data @@ -969,7 +969,7 @@ count :: proc(s, substr: string) -> int { repeats the string `s` multiple `count` times and returns the allocated string panics when `count` is below 0 - strings.repeat("abc", 2) -> "abcabc" + strings.repeat("abc", 2) -> "abcabc" */ repeat :: proc(s: string, count: int, allocator := context.allocator) -> string { if count < 0 { @@ -1378,7 +1378,7 @@ split_multi :: proc(s: string, substrs: []string, allocator := context.allocator // skip when no results if substrings_found < 1 { - return + return } buf = make([]string, substrings_found + 1, allocator) @@ -1812,16 +1812,35 @@ fields_iterator :: proc(s: ^string) -> (field: string, ok: bool) { // `levenshtein_distance` returns the Levenshtein edit distance between 2 strings. // This is a single-row-version of the Wagner–Fischer algorithm, based on C code by Martin Ettl. -// Note: allocator isn't used if the length of string b in runes is smaller than 70. +// Note: allocator isn't used if the length of string b in runes is smaller than 256. levenshtein_distance :: proc(a, b: string, allocator := context.allocator) -> int { LEVENSHTEIN_DEFAULT_COSTS: []int : { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, - 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, - 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, - 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, - 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, + 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, + 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, + 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, + 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, + 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, + 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, + 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, + 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, + 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, + 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, + 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, + 250, 251, 252, 253, 254, 255, } m, n := utf8.rune_count_in_string(a), utf8.rune_count_in_string(b) From 71b1cce517e421fd7eeff4d5ac15e5165dd318c0 Mon Sep 17 00:00:00 2001 From: hikari Date: Thu, 21 Apr 2022 21:19:11 +0300 Subject: [PATCH 27/79] strings: levenshtein_distance: 64 is actually faster than 256 --- core/strings/strings.odin | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/core/strings/strings.odin b/core/strings/strings.odin index f876aab3d..6e01f5c8a 100644 --- a/core/strings/strings.odin +++ b/core/strings/strings.odin @@ -1812,7 +1812,7 @@ fields_iterator :: proc(s: ^string) -> (field: string, ok: bool) { // `levenshtein_distance` returns the Levenshtein edit distance between 2 strings. // This is a single-row-version of the Wagner–Fischer algorithm, based on C code by Martin Ettl. -// Note: allocator isn't used if the length of string b in runes is smaller than 256. +// Note: allocator isn't used if the length of string b in runes is smaller than 64. levenshtein_distance :: proc(a, b: string, allocator := context.allocator) -> int { LEVENSHTEIN_DEFAULT_COSTS: []int : { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, @@ -1821,26 +1821,7 @@ levenshtein_distance :: proc(a, b: string, allocator := context.allocator) -> in 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, - 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, - 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, - 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, - 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, - 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, - 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, - 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, - 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, - 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, - 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, - 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, - 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, - 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, - 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, - 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, - 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, - 250, 251, 252, 253, 254, 255, + 60, 61, 62, 63, } m, n := utf8.rune_count_in_string(a), utf8.rune_count_in_string(b) From f0267536929c5909a8098baf95cb5eb1a4fa6522 Mon Sep 17 00:00:00 2001 From: hikari Date: Thu, 21 Apr 2022 21:19:43 +0300 Subject: [PATCH 28/79] strings: levenshtein_distance: remove `do` --- core/strings/strings.odin | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/strings/strings.odin b/core/strings/strings.odin index 6e01f5c8a..a3d9fa93e 100644 --- a/core/strings/strings.odin +++ b/core/strings/strings.odin @@ -1826,8 +1826,12 @@ levenshtein_distance :: proc(a, b: string, allocator := context.allocator) -> in m, n := utf8.rune_count_in_string(a), utf8.rune_count_in_string(b) - if m == 0 do return n - if n == 0 do return m + if m == 0 { + return n + } + if n == 0 { + return m + } costs: []int From e799476f90537ca173205a64da1fbf87b894b42e Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 22 Apr 2022 16:55:47 +0200 Subject: [PATCH 29/79] [compress/shoco] Add short string compressor. --- core/compress/shoco/model.odin | 148 +++++++++ core/compress/shoco/shoco.odin | 318 ++++++++++++++++++++ examples/all/all_main.odin | 2 + tests/core/assets/Shoco/LICENSE | 26 ++ tests/core/assets/Shoco/LICENSE.shoco | Bin 0 -> 1269 bytes tests/core/assets/Shoco/README.md | 95 ++++++ tests/core/assets/Shoco/README.md.shoco | Bin 0 -> 2227 bytes tests/core/compress/test_core_compress.odin | 57 +++- 8 files changed, 645 insertions(+), 1 deletion(-) create mode 100644 core/compress/shoco/model.odin create mode 100644 core/compress/shoco/shoco.odin create mode 100644 tests/core/assets/Shoco/LICENSE create mode 100644 tests/core/assets/Shoco/LICENSE.shoco create mode 100644 tests/core/assets/Shoco/README.md create mode 100644 tests/core/assets/Shoco/README.md.shoco diff --git a/core/compress/shoco/model.odin b/core/compress/shoco/model.odin new file mode 100644 index 000000000..49e3dd97e --- /dev/null +++ b/core/compress/shoco/model.odin @@ -0,0 +1,148 @@ +/* + This file was generated, so don't edit this by hand. + Transliterated from https://github.com/Ed-von-Schleck/shoco/blob/master/shoco_model.h, + which is an English word model. +*/ + +// package shoco is an implementation of the shoco short string compressor +package shoco + +DEFAULT_MODEL :: Shoco_Model { + min_char = 39, + max_char = 122, + characters_by_id = { + 'e', 'a', 'i', 'o', 't', 'h', 'n', 'r', 's', 'l', 'u', 'c', 'w', 'm', 'd', 'b', 'p', 'f', 'g', 'v', 'y', 'k', '-', 'H', 'M', 'T', '\'', 'B', 'x', 'I', 'W', 'L', + }, + ids_by_character = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 26, -1, -1, -1, -1, -1, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, -1, -1, -1, -1, -1, 23, 29, -1, -1, 31, 24, -1, -1, -1, -1, -1, -1, 25, -1, -1, 30, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 15, 11, 14, 0, 17, 18, 5, 2, -1, 21, 9, 13, 6, 3, 16, -1, 7, 8, 4, 10, 19, 12, 28, 20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }, + successors_by_bigram = { + {7, 4, 12, -1, 6, -1, 1, 0, 3, 5, -1, 9, -1, 8, 2, -1, 15, 14, -1, 10, 11, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1}, + {-1, -1, 6, -1, 1, -1, 0, 3, 2, 4, 15, 11, -1, 9, 5, 10, 13, -1, 12, 8, 7, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {9, 11, -1, 4, 2, -1, 0, 8, 1, 5, -1, 6, -1, 3, 7, 15, -1, 12, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {-1, -1, 14, 7, 5, -1, 1, 2, 8, 9, 0, 15, 6, 4, 11, -1, 12, 3, -1, 10, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {2, 4, 3, 1, 5, 0, -1, 6, 10, 9, 7, 12, 11, -1, -1, -1, -1, 13, -1, -1, 8, -1, 15, -1, -1, -1, 14, -1, -1, -1, -1, -1}, + {0, 1, 2, 3, 4, -1, -1, 5, 9, 10, 6, -1, -1, 8, 15, 11, -1, 14, -1, -1, 7, -1, 13, -1, -1, -1, 12, -1, -1, -1, -1, -1}, + {2, 8, 7, 4, 3, -1, 9, -1, 6, 11, -1, 5, -1, -1, 0, -1, -1, 14, 1, 15, 10, 12, -1, -1, -1, -1, 13, -1, -1, -1, -1, -1}, + {0, 3, 1, 2, 6, -1, 9, 8, 4, 12, 13, 10, -1, 11, 7, -1, -1, 15, 14, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {0, 6, 3, 4, 1, 2, -1, -1, 5, 10, 7, 9, 11, 12, -1, -1, 8, 14, -1, -1, 15, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {0, 6, 2, 5, 9, -1, -1, -1, 10, 1, 8, -1, 12, 14, 4, -1, 15, 7, -1, 13, 3, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {8, 10, 9, 15, 1, -1, 4, 0, 3, 2, -1, 6, -1, 12, 11, 13, 7, 14, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {1, 3, 6, 0, 4, 2, -1, 7, 13, 8, 9, 11, -1, -1, 15, -1, -1, -1, -1, -1, 10, 5, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {3, 0, 1, 4, -1, 2, 5, 6, 7, 8, -1, 14, -1, -1, 9, 15, -1, 12, -1, -1, -1, 10, 11, -1, -1, -1, 13, -1, -1, -1, -1, -1}, + {0, 1, 3, 2, 15, -1, 12, -1, 7, 14, 4, -1, -1, 9, -1, 8, 5, 10, -1, -1, 6, -1, 13, -1, -1, -1, 11, -1, -1, -1, -1, -1}, + {0, 3, 1, 2, -1, -1, 12, 6, 4, 9, 7, -1, -1, 14, 8, -1, -1, 15, 11, 13, 5, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {0, 5, 7, 2, 10, 13, -1, 6, 8, 1, 3, -1, -1, 14, 15, 11, -1, -1, -1, 12, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {0, 2, 6, 3, 7, 10, -1, 1, 9, 4, 8, -1, -1, 15, -1, 12, 5, -1, -1, -1, 11, -1, 13, -1, -1, -1, 14, -1, -1, -1, -1, -1}, + {1, 3, 4, 0, 7, -1, 12, 2, 11, 8, 6, 13, -1, -1, -1, -1, -1, 5, -1, -1, 10, 15, 9, -1, -1, -1, 14, -1, -1, -1, -1, -1}, + {1, 3, 5, 2, 13, 0, 9, 4, 7, 6, 8, -1, -1, 15, -1, 11, -1, -1, 10, -1, 14, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {0, 2, 1, 3, -1, -1, -1, 6, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {1, 11, 4, 0, 3, -1, 13, 12, 2, 7, -1, -1, 15, 10, 5, 8, 14, -1, -1, -1, -1, -1, 9, -1, -1, -1, 6, -1, -1, -1, -1, -1}, + {0, 9, 2, 14, 15, 4, 1, 13, 3, 5, -1, -1, 10, -1, -1, -1, -1, 6, 12, -1, 7, -1, 8, -1, -1, -1, 11, -1, -1, -1, -1, -1}, + {-1, 2, 14, -1, 1, 5, 8, 7, 4, 12, -1, 6, 9, 11, 13, 3, 10, 15, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {0, 1, 3, 2, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {4, 3, 1, 5, -1, -1, -1, 0, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {2, 8, 4, 1, -1, 0, -1, 6, -1, -1, 5, -1, 7, -1, -1, -1, -1, -1, -1, -1, 10, -1, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1}, + {12, 5, -1, -1, 1, -1, -1, 7, 0, 3, -1, 2, -1, 4, 6, -1, -1, -1, -1, 8, -1, -1, 15, -1, 13, 9, -1, -1, -1, -1, -1, 11}, + {1, 3, 2, 4, -1, -1, -1, 5, -1, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, -1}, + {5, 3, 4, 12, 1, 6, -1, -1, -1, -1, 8, 2, -1, -1, -1, -1, 0, 9, -1, -1, 11, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1}, + {-1, -1, -1, -1, 0, -1, 1, 12, 3, -1, -1, -1, -1, 5, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, 6, -1, 10}, + {2, 3, 1, 4, -1, 0, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 7, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1}, + {5, 1, 3, 0, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, 9, -1, -1, 6, -1, 7}, + }, + successors_reversed = { + {'s', 't', 'c', 'l', 'm', 'a', 'd', 'r', 'v', 'T', 'A', 'L', 'e', 'M', 'Y', '-'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'-', 't', 'a', 'b', 's', 'h', 'c', 'r', 'n', 'w', 'p', 'm', 'l', 'd', 'i', 'f'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'u', 'e', 'i', 'a', 'o', 'r', 'y', 'l', 'I', 'E', 'R', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'e', 'a', 'o', 'i', 'u', 'A', 'y', 'E', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'t', 'n', 'f', 's', '\'', 'm', 'I', 'N', 'A', 'E', 'L', 'Z', 'r', 'V', 'R', 'C'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'o', 'a', 'y', 'i', 'u', 'e', 'I', 'L', 'D', '\'', 'E', 'Y', '\x00', '\x00', '\x00', '\x00'}, + {'r', 'i', 'y', 'a', 'e', 'o', 'u', 'Y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'h', 'o', 'e', 'E', 'i', 'u', 'r', 'w', 'a', 'H', 'y', 'R', 'Z', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'h', 'i', 'e', 'a', 'o', 'r', 'I', 'y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'n', 't', 's', 'r', 'l', 'd', 'i', 'y', 'v', 'm', 'b', 'c', 'g', 'p', 'k', 'u'}, + {'e', 'l', 'o', 'u', 'y', 'a', 'r', 'i', 's', 'j', 't', 'b', 'v', 'h', 'm', 'd'}, + {'o', 'e', 'h', 'a', 't', 'k', 'i', 'r', 'l', 'u', 'y', 'c', 'q', 's', '-', 'd'}, + {'e', 'i', 'o', 'a', 's', 'y', 'r', 'u', 'd', 'l', '-', 'g', 'n', 'v', 'm', 'f'}, + {'r', 'n', 'd', 's', 'a', 'l', 't', 'e', 'm', 'c', 'v', 'y', 'i', 'x', 'f', 'p'}, + {'o', 'e', 'r', 'a', 'i', 'f', 'u', 't', 'l', '-', 'y', 's', 'n', 'c', '\'', 'k'}, + {'h', 'e', 'o', 'a', 'r', 'i', 'l', 's', 'u', 'n', 'g', 'b', '-', 't', 'y', 'm'}, + {'e', 'a', 'i', 'o', 't', 'r', 'u', 'y', 'm', 's', 'l', 'b', '\'', '-', 'f', 'd'}, + {'n', 's', 't', 'm', 'o', 'l', 'c', 'd', 'r', 'e', 'g', 'a', 'f', 'v', 'z', 'b'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'e', 'n', 'i', 's', 'h', 'l', 'f', 'y', '-', 'a', 'w', '\'', 'g', 'r', 'o', 't'}, + {'e', 'l', 'i', 'y', 'd', 'o', 'a', 'f', 'u', 't', 's', 'k', 'w', 'v', 'm', 'p'}, + {'e', 'a', 'o', 'i', 'u', 'p', 'y', 's', 'b', 'm', 'f', '\'', 'n', '-', 'l', 't'}, + {'d', 'g', 'e', 't', 'o', 'c', 's', 'i', 'a', 'n', 'y', 'l', 'k', '\'', 'f', 'v'}, + {'u', 'n', 'r', 'f', 'm', 't', 'w', 'o', 's', 'l', 'v', 'd', 'p', 'k', 'i', 'c'}, + {'e', 'r', 'a', 'o', 'l', 'p', 'i', 't', 'u', 's', 'h', 'y', 'b', '-', '\'', 'm'}, + {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'e', 'i', 'o', 'a', 's', 'y', 't', 'd', 'r', 'n', 'c', 'm', 'l', 'u', 'g', 'f'}, + {'e', 't', 'h', 'i', 'o', 's', 'a', 'u', 'p', 'c', 'l', 'w', 'm', 'k', 'f', 'y'}, + {'h', 'o', 'e', 'i', 'a', 't', 'r', 'u', 'y', 'l', 's', 'w', 'c', 'f', '\'', '-'}, + {'r', 't', 'l', 's', 'n', 'g', 'c', 'p', 'e', 'i', 'a', 'd', 'm', 'b', 'f', 'o'}, + {'e', 'i', 'a', 'o', 'y', 'u', 'r', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, + {'a', 'i', 'h', 'e', 'o', 'n', 'r', 's', 'l', 'd', 'k', '-', 'f', '\'', 'c', 'b'}, + {'p', 't', 'c', 'a', 'i', 'e', 'h', 'q', 'u', 'f', '-', 'y', 'o', '\x00', '\x00', '\x00'}, + {'o', 'e', 's', 't', 'i', 'd', '\'', 'l', 'b', '-', 'm', 'a', 'r', 'n', 'p', 'w'}, + }, + + character_count = 32, + successor_count = 16, + + max_successor_n = 7, + packs = { + { 0x80000000, 1, 2, { 26, 24, 24, 24, 24, 24, 24, 24 }, { 15, 3, 0, 0, 0, 0, 0, 0 }, 0xc0, 0x80 }, + { 0xc0000000, 2, 4, { 25, 22, 19, 16, 16, 16, 16, 16 }, { 15, 7, 7, 7, 0, 0, 0, 0 }, 0xe0, 0xc0 }, + { 0xe0000000, 4, 8, { 23, 19, 15, 11, 8, 5, 2, 0 }, { 31, 15, 15, 15, 7, 7, 7, 3 }, 0xf0, 0xe0 }, + }, +} \ No newline at end of file diff --git a/core/compress/shoco/shoco.odin b/core/compress/shoco/shoco.odin new file mode 100644 index 000000000..9c5008f5d --- /dev/null +++ b/core/compress/shoco/shoco.odin @@ -0,0 +1,318 @@ +/* + Copyright 2022 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + List of contributors: + Jeroen van Rijn: Initial implementation. + + An implementation of [shoco](https://github.com/Ed-von-Schleck/shoco) by Christian Schramm. +*/ + +// package shoco is an implementation of the shoco short string compressor +package shoco + +import "core:intrinsics" +import "core:compress" + +Shoco_Pack :: struct { + word: u32, + bytes_packed: i8, + bytes_unpacked: i8, + offsets: [8]u16, + masks: [8]i16, + header_mask: u8, + header: u8, +} + +Shoco_Model :: struct { + min_char: u8, + max_char: u8, + characters_by_id: []u8, + ids_by_character: [256]i16, + successors_by_bigram: [][]i8, + successors_reversed: [][]u8, + + character_count: u8, + successor_count: u8, + max_successor_n: i8, + packs: []Shoco_Pack, +} + +compress_bound :: proc(uncompressed_size: int) -> (worst_case_compressed_size: int) { + // Worst case compression happens when input is non-ASCII (128-255) + // Encoded as 0x00 + the byte in question. + return uncompressed_size * 2 +} + +decompress_bound :: proc(compressed_size: int, model := DEFAULT_MODEL) -> (maximum_decompressed_size: int) { + // Best case compression is 2:1 + most: f64 + for pack in model.packs { + val := f64(compressed_size) / f64(pack.bytes_packed) * f64(pack.bytes_unpacked) + most = max(most, val) + } + return int(most) +} + +find_best_encoding :: proc(indices: []i16, n_consecutive: i8, model := DEFAULT_MODEL) -> (res: int) { + for p := len(model.packs); p > 0; p -= 1 { + pack := model.packs[p - 1] + if n_consecutive >= pack.bytes_unpacked { + have_index := true + for i := 0; i < int(pack.bytes_unpacked); i += 1 { + if indices[i] > pack.masks[i] { + have_index = false + break + } + } + if have_index { + return p - 1 + } + } + } + return -1 +} + +validate_model :: proc(model: Shoco_Model) -> (int, compress.Error) { + if len(model.successors_reversed) != int(model.max_char - model.min_char) { + return 0, .Unknown_Compression_Method + } + + if len(model.characters_by_id) != int(model.character_count) { + return 0, .Unknown_Compression_Method + } + + if len(model.successors_by_bigram) != int(model.character_count) || len(model.successors_by_bigram[0]) != int(model.character_count) { + return 0, .Unknown_Compression_Method + } + + if len(model.successors_reversed[0]) != int(model.successor_count) { + return 0, .Unknown_Compression_Method + } + + // Model seems legit. + return 0, nil +} + +// Decompresses into provided buffer. +decompress_slice_to_output_buffer :: proc(input: []u8, output: []u8, model := DEFAULT_MODEL) -> (size: int, err: compress.Error) { + inp, inp_end := 0, len(input) + out, out_end := 0, len(output) + + validate_model(model) or_return + + for inp < inp_end { + val := transmute(i8)input[inp] + mark := int(-1) + + for val < 0 { + val <<= 1 + mark += 1 + } + + if mark > len(model.packs) { + return out, .Unknown_Compression_Method + } + + if mark < 0 { + if out >= out_end { + return out, .Output_Too_Short + } + + // Ignore the sentinel value for non-ASCII chars + if input[inp] == 0x00 { + inp += 1 + if inp >= inp_end { + return out, .Stream_Too_Short + } + } + output[out] = input[inp] + inp, out = inp + 1, out + 1 + + } else { + pack := model.packs[mark] + + if out + int(pack.bytes_unpacked) > out_end { + return out, .Output_Too_Short + } else if inp + int(pack.bytes_packed) > inp_end { + return out, .Stream_Too_Short + } + + code := intrinsics.unaligned_load((^u32)(&input[inp])) + when ODIN_ENDIAN == .Little { + code = intrinsics.byte_swap(code) + } + + // Unpack the leading char + offset := pack.offsets[0] + mask := pack.masks[0] + + last_chr := model.characters_by_id[(code >> offset) & u32(mask)] + output[out] = last_chr + + // Unpack the successor chars + for i := 1; i < int(pack.bytes_unpacked); i += 1 { + offset = pack.offsets[i] + mask = pack.masks[i] + + last_chr = model.successors_reversed[last_chr - model.min_char][(code >> offset) & u32(mask)] + output[out + i] = last_chr + } + + out += int(pack.bytes_unpacked) + inp += int(pack.bytes_packed) + } + } + + return out, nil +} + +decompress_slice_to_string :: proc(input: []u8, model := DEFAULT_MODEL, allocator := context.allocator) -> (res: string, err: compress.Error) { + context.allocator = allocator + + if len(input) == 0 { + return "", .Stream_Too_Short + } + + max_output_size := decompress_bound(len(input), model) + + buf: [dynamic]u8 + if !resize(&buf, max_output_size) { + return "", .Out_Of_Memory + } + + length, result := decompress_slice_to_output_buffer(input, buf[:]) + resize(&buf, length) + return string(buf[:]), result +} +decompress :: proc{decompress_slice_to_output_buffer, decompress_slice_to_string} + +compress_string_to_buffer :: proc(input: string, output: []u8, model := DEFAULT_MODEL, allocator := context.allocator) -> (size: int, err: compress.Error) { + inp, inp_end := 0, len(input) + out, out_end := 0, len(output) + output := output + + validate_model(model) or_return + + indices := make([]i16, model.max_successor_n + 1) + defer delete(indices) + + last_resort := false + + encode: for inp < inp_end { + if last_resort { + last_resort = false + + if input[inp] & 0x80 == 0x80 { + // Non-ASCII case + if out + 2 > out_end { + return out, .Output_Too_Short + } + + // Put in a sentinel byte + output[out] = 0x00 + out += 1 + } else { + // An ASCII byte + if out + 1 > out_end { + return out, .Output_Too_Short + } + } + output[out] = input[inp] + out, inp = out + 1, inp + 1 + } else { + // Find the longest string of known successors + indices[0] = model.ids_by_character[input[inp]] + last_chr_index := indices[0] + + if last_chr_index < 0 { + last_resort = true + continue encode + } + + rest := inp_end - inp + n_consecutive: i8 = 1 + for ; n_consecutive <= model.max_successor_n; n_consecutive += 1 { + if inp_end > 0 && int(n_consecutive) == rest { + break + } + + current_index := model.ids_by_character[input[inp + int(n_consecutive)]] + if current_index < 0 { // '\0' is always -1 + break + } + + successor_index := model.successors_by_bigram[last_chr_index][current_index] + if successor_index < 0 { + break + } + + indices[n_consecutive] = i16(successor_index) + last_chr_index = current_index + } + + if n_consecutive < 2 { + last_resort = true + continue encode + } + + pack_n := find_best_encoding(indices, n_consecutive) + if pack_n >= 0 { + if out + int(model.packs[pack_n].bytes_packed) > out_end { + return out, .Output_Too_Short + } + + pack := model.packs[pack_n] + code := pack.word + + for i := 0; i < int(pack.bytes_unpacked); i += 1 { + code |= u32(indices[i]) << pack.offsets[i] + } + + // In the little-endian world, we need to swap what's in the register to match the memory representation. + when ODIN_ENDIAN == .Little { + code = intrinsics.byte_swap(code) + } + out_ptr := raw_data(output[out:]) + + switch pack.bytes_packed { + case 4: + intrinsics.unaligned_store(transmute(^u32)out_ptr, code) + case 2: + intrinsics.unaligned_store(transmute(^u16)out_ptr, u16(code)) + case 1: + intrinsics.unaligned_store(transmute(^u8)out_ptr, u8(code)) + case: + return out, .Unknown_Compression_Method + } + + out += int(pack.bytes_packed) + inp += int(pack.bytes_unpacked) + } else { + last_resort = true + continue encode + } + } + } + return out, nil +} + +compress_string :: proc(input: string, model := DEFAULT_MODEL, allocator := context.allocator) -> (output: []u8, err: compress.Error) { + context.allocator = allocator + + if len(input) == 0 { + return {}, .Stream_Too_Short + } + + max_output_size := compress_bound(len(input)) + + buf: [dynamic]u8 + if !resize(&buf, max_output_size) { + return {}, .Out_Of_Memory + } + + length, result := compress_string_to_buffer(input, buf[:]) + resize(&buf, length) + return buf[:length], result +} +compress :: proc{compress_string_to_buffer, compress_string} \ No newline at end of file diff --git a/examples/all/all_main.odin b/examples/all/all_main.odin index 4f5bfbdc1..27f199062 100644 --- a/examples/all/all_main.odin +++ b/examples/all/all_main.odin @@ -10,6 +10,7 @@ import c "core:c" import libc "core:c/libc" import compress "core:compress" +import shoco "core:compress/shoco" import gzip "core:compress/gzip" import zlib "core:compress/zlib" @@ -115,6 +116,7 @@ _ :: bytes _ :: c _ :: libc _ :: compress +_ :: shoco _ :: gzip _ :: zlib _ :: bit_array diff --git a/tests/core/assets/Shoco/LICENSE b/tests/core/assets/Shoco/LICENSE new file mode 100644 index 000000000..9ca94bcdf --- /dev/null +++ b/tests/core/assets/Shoco/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2016-2021 Ginger Bill. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/core/assets/Shoco/LICENSE.shoco b/tests/core/assets/Shoco/LICENSE.shoco new file mode 100644 index 0000000000000000000000000000000000000000..5d5e4d6236019bf208cc2ebb7a81ed06469b78a8 GIT binary patch literal 1269 zcmah{O=}cE6pS}Bg5XV079PalxF%5)^kUCU?{3TN^w{0A>*B?HY+}H~xDx`oi3E&d zqUIt(5=6*Jy{V{(hy;uW1AY+vKKOxN1%H6ACmW3@dTAc7Usu(uYX8{a$aL?KA#CYt zL3?Z4u5InD?QJN`HP_+w&E~*nVeeBctgjEB%sGyFaC+$Dxl6|}+KsCoBQqUx_>RQ>;av8k~9e?XTDtcz)f#vB>};yZd=z)$zV@`(U(t`ve~CtK;?l>5il6 z(#GTnO}!P?I8w7Chp0ApqHdc59XCii70X*)eBOkCiN=*R;Y@0A7uQU`e_T&-a|HMB z;vp{JfmgpwMh`ArnFQ_dJ;mEq!za-*)87#ObmPnQSq?}CYh8UkeO)x%iC1SP2W%X> z$4QDQ3Jopurot2l)v=Og5%2(B6{g7H%ebmUp=eN)VZfCJivsFKMhP!5k{X+srk9%> zd%>bA_`z6ltsxaeDGmjJ5xQcL5nQ_vksl_3hzc$U0Fg8ZMM)SMGt#xB&Y#a9^C)rU z7YT)VA{3^Y3CRl+S*|>hF~m$6;U^(eh!Yh{%|UB9fzW=)M2QDEVkbuMGLH;&k%i%3 zx3t=S+tK4Bl6fI_GA7zG5Q_W8uJfz%$vM&&y3jHAg)R6&P988-bsLzPA4q73E}ejt zSb=M7`Hv-(iJvH5vYN>Uop{;^lNgSIlz}zc(Oi{<&-Gq}(psNKqPdHJ8I} + Odin logo +
+ The Data-Oriented Language for Sane Software Development. +
+
+ + + + + + +
+ + + + + + +

+ +# The Odin Programming Language + + +Odin is a general-purpose programming language with distinct typing, built for high performance, modern systems, and built-in data-oriented data types. The Odin Programming Language, the C alternative for the joy of programming. + +Website: [https://odin-lang.org/](https://odin-lang.org/) + +```odin +package main + +import "core:fmt" + +main :: proc() { + program := "+ + * 😃 - /" + accumulator := 0 + + for token in program { + switch token { + case '+': accumulator += 1 + case '-': accumulator -= 1 + case '*': accumulator *= 2 + case '/': accumulator /= 2 + case '😃': accumulator *= accumulator + case: // Ignore everything else + } + } + + fmt.printf("The program \"%s\" calculates the value %d\n", + program, accumulator) +} + +``` + +## Documentation + +#### [Getting Started](https://odin-lang.org/docs/install) + +Instructions for downloading and installing the Odin compiler and libraries. + +#### [Nightly Builds](https://odin-lang.org/docs/nightly/) + +Get the latest nightly builds of Odin. + +### Learning Odin + +#### [Overview of Odin](https://odin-lang.org/docs/overview) + +An overview of the Odin programming language. + +#### [Frequently Asked Questions (FAQ)](https://odin-lang.org/docs/faq) + +Answers to common questions about Odin. + +#### [Packages](https://pkg.odin-lang.org/) + +Documentation for all the official packages part of the [core](https://pkg.odin-lang.org/core/) and [vendor](https://pkg.odin-lang.org/vendor/) library collections. + +#### [The Odin Wiki](https://github.com/odin-lang/Odin/wiki) + +A wiki maintained by the Odin community. + +#### [Odin Discord](https://discord.gg/sVBPHEv) + +Get live support and talk with other odiners on the Odin Discord. + +### Articles + +#### [The Odin Blog](https://odin-lang.org/news/) + +The official blog of the Odin programming language, featuring announcements, news, and in-depth articles by the Odin team and guests. + +## Warnings + +* The Odin compiler is still in development. diff --git a/tests/core/assets/Shoco/README.md.shoco b/tests/core/assets/Shoco/README.md.shoco new file mode 100644 index 0000000000000000000000000000000000000000..013f4f46928fb3d5c6ea62d76ddda91f29c6e420 GIT binary patch literal 2227 zcmb_eZ%i9y7)PfcxS7ZoGd~#fEaBQ-k6W+3qjxPf zYC>CJWrU6`YCtCgWoR-o!-q{UF$3*1i-IUxNkiNghgqT@O#E!(``#apDK*B}K(6m| z&+~hJ&-47=C#_L{n-M~8I{SPdeTV}5sTmeP4SU`6N2(V|vLG|+OpsN^Y2YNyO*b!u zfLLjmicP!eIp3(Osn1P%LqoV9o(QS`lz)fu^`-;E}x(e4@V52eP04=7Z z1P5FfC5E%64(RvkK1>QWzGCi--aLh$Bsj_Dq2Yj0 z9_sOJIUf!Uh*4el*m6pa!C8w8OHu8-XZ2ebbXYRQ^TVoU(*inK@^@P)MRj#`VINB2 zya95XatcLdehF(pXNz3^phif&IRI-G8mHZLUm5Nq8rH}y9)ct_qo7~DtCj^v@*7pv8`pc zwag6_rZ2Ls$hKO+A1j5D|17L!;BMuHbap<8<&>LNFM`BH>*7lgAMAxPHTj<%Qu5KxpDm zFE5d^h2I?Qi;8@&;gm0fY&hWzwIMJMIKm`;|4rIre&j+=B#x=qs(}QwMi#%s<(C|( zZ7t20#n1KV8PxR*oP*imwGi|eVcd^#y9V6#7G3N~8%PNVEWQQiO~EMEoJhmDON8m? zzj{SZ0^^<}K{>4%6?9~a^5JBQP$`*6M|80+oX?58!)|zLF_^uZYU)w9I$-Xjb}=B@ z(SO-@cob+ZAdtle(Tr5pBtYZ+5i9m literal 0 HcmV?d00001 diff --git a/tests/core/compress/test_core_compress.odin b/tests/core/compress/test_core_compress.odin index 51952a568..ee7233e52 100644 --- a/tests/core/compress/test_core_compress.odin +++ b/tests/core/compress/test_core_compress.odin @@ -7,13 +7,14 @@ package test_core_compress List of contributors: Jeroen van Rijn: Initial implementation. - A test suite for ZLIB, GZIP. + A test suite for ZLIB, GZIP and Shoco. */ import "core:testing" import "core:compress/zlib" import "core:compress/gzip" +import "core:compress/shoco" import "core:bytes" import "core:fmt" @@ -48,6 +49,7 @@ main :: proc() { t := testing.T{w=w} zlib_test(&t) gzip_test(&t) + shoco_test(&t) fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) if TEST_fail > 0 { @@ -134,3 +136,56 @@ gzip_test :: proc(t: ^testing.T) { expect(t, false, error) } } + +@test +shoco_test :: proc(t: ^testing.T) { + + Shoco_Tests :: []struct{ + compressed: []u8, + raw: []u8, + short_pack: int, + short_sentinel: int, + }{ + { #load("../assets/Shoco/README.md.shoco"), #load("../assets/Shoco/README.md"), 10, 1006 }, + { #load("../assets/Shoco/LICENSE.shoco"), #load("../assets/Shoco/LICENSE"), 25, 68 }, + } + + for v in Shoco_Tests { + expected_raw := len(v.raw) + expected_compressed := len(v.compressed) + + biggest_unpacked := shoco.decompress_bound(expected_compressed) + biggest_packed := shoco.compress_bound(expected_raw) + + buffer := make([]u8, max(biggest_packed, biggest_unpacked)) + defer delete(buffer) + + size, err := shoco.decompress(v.compressed, buffer[:]) + msg := fmt.tprintf("Expected `decompress` to return `nil`, got %v", err) + expect(t, err == nil, msg) + + msg = fmt.tprintf("Decompressed %v bytes into %v. Expected to decompress into %v bytes.", len(v.compressed), size, expected_raw) + expect(t, size == expected_raw, msg) + expect(t, string(buffer[:size]) == string(v.raw), "Decompressed contents don't match.") + + size, err = shoco.compress(string(v.raw), buffer[:]) + expect(t, err == nil, "Expected `compress` to return `nil`.") + + msg = fmt.tprintf("Compressed %v bytes into %v. Expected to compress into %v bytes.", expected_raw, size, expected_compressed) + expect(t, size == expected_compressed, msg) + + size, err = shoco.decompress(v.compressed, buffer[:expected_raw - 10]) + msg = fmt.tprintf("Decompressing into too small a buffer returned %v, expected `.Output_Too_Short`", err) + expect(t, err == .Output_Too_Short, msg) + + size, err = shoco.compress(string(v.raw), buffer[:expected_compressed - 10]) + msg = fmt.tprintf("Compressing into too small a buffer returned %v, expected `.Output_Too_Short`", err) + expect(t, err == .Output_Too_Short, msg) + + size, err = shoco.decompress(v.compressed[:v.short_pack], buffer[:]) + expect(t, err == .Stream_Too_Short, "Expected `decompress` to return `Stream_Too_Short` because there was no more data after selecting a pack.") + + size, err = shoco.decompress(v.compressed[:v.short_sentinel], buffer[:]) + expect(t, err == .Stream_Too_Short, "Expected `decompress` to return `Stream_Too_Short` because there was no more data after non-ASCII sentinel.") + } +} \ No newline at end of file From ac9a358c65820894968cc11b55b08de0646d98d4 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 22 Apr 2022 17:52:38 +0200 Subject: [PATCH 30/79] [shoco] Replace 2D slices in model with 1D slices. --- core/compress/shoco/model.odin | 230 ++++++++++++++++----------------- core/compress/shoco/shoco.odin | 22 ++-- 2 files changed, 127 insertions(+), 125 deletions(-) diff --git a/core/compress/shoco/model.odin b/core/compress/shoco/model.odin index 49e3dd97e..bbc38903d 100644 --- a/core/compress/shoco/model.odin +++ b/core/compress/shoco/model.odin @@ -17,123 +17,123 @@ DEFAULT_MODEL :: Shoco_Model { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 26, -1, -1, -1, -1, -1, 22, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 27, -1, -1, -1, -1, -1, 23, 29, -1, -1, 31, 24, -1, -1, -1, -1, -1, -1, 25, -1, -1, 30, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 15, 11, 14, 0, 17, 18, 5, 2, -1, 21, 9, 13, 6, 3, 16, -1, 7, 8, 4, 10, 19, 12, 28, 20, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, }, successors_by_bigram = { - {7, 4, 12, -1, 6, -1, 1, 0, 3, 5, -1, 9, -1, 8, 2, -1, 15, 14, -1, 10, 11, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1}, - {-1, -1, 6, -1, 1, -1, 0, 3, 2, 4, 15, 11, -1, 9, 5, 10, 13, -1, 12, 8, 7, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {9, 11, -1, 4, 2, -1, 0, 8, 1, 5, -1, 6, -1, 3, 7, 15, -1, 12, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {-1, -1, 14, 7, 5, -1, 1, 2, 8, 9, 0, 15, 6, 4, 11, -1, 12, 3, -1, 10, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {2, 4, 3, 1, 5, 0, -1, 6, 10, 9, 7, 12, 11, -1, -1, -1, -1, 13, -1, -1, 8, -1, 15, -1, -1, -1, 14, -1, -1, -1, -1, -1}, - {0, 1, 2, 3, 4, -1, -1, 5, 9, 10, 6, -1, -1, 8, 15, 11, -1, 14, -1, -1, 7, -1, 13, -1, -1, -1, 12, -1, -1, -1, -1, -1}, - {2, 8, 7, 4, 3, -1, 9, -1, 6, 11, -1, 5, -1, -1, 0, -1, -1, 14, 1, 15, 10, 12, -1, -1, -1, -1, 13, -1, -1, -1, -1, -1}, - {0, 3, 1, 2, 6, -1, 9, 8, 4, 12, 13, 10, -1, 11, 7, -1, -1, 15, 14, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {0, 6, 3, 4, 1, 2, -1, -1, 5, 10, 7, 9, 11, 12, -1, -1, 8, 14, -1, -1, 15, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {0, 6, 2, 5, 9, -1, -1, -1, 10, 1, 8, -1, 12, 14, 4, -1, 15, 7, -1, 13, 3, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {8, 10, 9, 15, 1, -1, 4, 0, 3, 2, -1, 6, -1, 12, 11, 13, 7, 14, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {1, 3, 6, 0, 4, 2, -1, 7, 13, 8, 9, 11, -1, -1, 15, -1, -1, -1, -1, -1, 10, 5, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {3, 0, 1, 4, -1, 2, 5, 6, 7, 8, -1, 14, -1, -1, 9, 15, -1, 12, -1, -1, -1, 10, 11, -1, -1, -1, 13, -1, -1, -1, -1, -1}, - {0, 1, 3, 2, 15, -1, 12, -1, 7, 14, 4, -1, -1, 9, -1, 8, 5, 10, -1, -1, 6, -1, 13, -1, -1, -1, 11, -1, -1, -1, -1, -1}, - {0, 3, 1, 2, -1, -1, 12, 6, 4, 9, 7, -1, -1, 14, 8, -1, -1, 15, 11, 13, 5, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {0, 5, 7, 2, 10, 13, -1, 6, 8, 1, 3, -1, -1, 14, 15, 11, -1, -1, -1, 12, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {0, 2, 6, 3, 7, 10, -1, 1, 9, 4, 8, -1, -1, 15, -1, 12, 5, -1, -1, -1, 11, -1, 13, -1, -1, -1, 14, -1, -1, -1, -1, -1}, - {1, 3, 4, 0, 7, -1, 12, 2, 11, 8, 6, 13, -1, -1, -1, -1, -1, 5, -1, -1, 10, 15, 9, -1, -1, -1, 14, -1, -1, -1, -1, -1}, - {1, 3, 5, 2, 13, 0, 9, 4, 7, 6, 8, -1, -1, 15, -1, 11, -1, -1, 10, -1, 14, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {0, 2, 1, 3, -1, -1, -1, 6, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {1, 11, 4, 0, 3, -1, 13, 12, 2, 7, -1, -1, 15, 10, 5, 8, 14, -1, -1, -1, -1, -1, 9, -1, -1, -1, 6, -1, -1, -1, -1, -1}, - {0, 9, 2, 14, 15, 4, 1, 13, 3, 5, -1, -1, 10, -1, -1, -1, -1, 6, 12, -1, 7, -1, 8, -1, -1, -1, 11, -1, -1, -1, -1, -1}, - {-1, 2, 14, -1, 1, 5, 8, 7, 4, 12, -1, 6, 9, 11, 13, 3, 10, 15, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {0, 1, 3, 2, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {4, 3, 1, 5, -1, -1, -1, 0, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {2, 8, 4, 1, -1, 0, -1, 6, -1, -1, 5, -1, 7, -1, -1, -1, -1, -1, -1, -1, 10, -1, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1}, - {12, 5, -1, -1, 1, -1, -1, 7, 0, 3, -1, 2, -1, 4, 6, -1, -1, -1, -1, 8, -1, -1, 15, -1, 13, 9, -1, -1, -1, -1, -1, 11}, - {1, 3, 2, 4, -1, -1, -1, 5, -1, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, -1}, - {5, 3, 4, 12, 1, 6, -1, -1, -1, -1, 8, 2, -1, -1, -1, -1, 0, 9, -1, -1, 11, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1}, - {-1, -1, -1, -1, 0, -1, 1, 12, 3, -1, -1, -1, -1, 5, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, 6, -1, 10}, - {2, 3, 1, 4, -1, 0, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 7, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1}, - {5, 1, 3, 0, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, 9, -1, -1, 6, -1, 7}, + 7, 4, 12, -1, 6, -1, 1, 0, 3, 5, -1, 9, -1, 8, 2, -1, 15, 14, -1, 10, 11, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1, + 1, -1, 6, -1, 1, -1, 0, 3, 2, 4, 15, 11, -1, 9, 5, 10, 13, -1, 12, 8, 7, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 9, 11, -1, 4, 2, -1, 0, 8, 1, 5, -1, 6, -1, 3, 7, 15, -1, 12, 10, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 14, 7, 5, -1, 1, 2, 8, 9, 0, 15, 6, 4, 11, -1, 12, 3, -1, 10, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 2, 4, 3, 1, 5, 0, -1, 6, 10, 9, 7, 12, 11, -1, -1, -1, -1, 13, -1, -1, 8, -1, 15, -1, -1, -1, 14, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, -1, -1, 5, 9, 10, 6, -1, -1, 8, 15, 11, -1, 14, -1, -1, 7, -1, 13, -1, -1, -1, 12, -1, -1, -1, -1, -1, + 2, 8, 7, 4, 3, -1, 9, -1, 6, 11, -1, 5, -1, -1, 0, -1, -1, 14, 1, 15, 10, 12, -1, -1, -1, -1, 13, -1, -1, -1, -1, -1, + 0, 3, 1, 2, 6, -1, 9, 8, 4, 12, 13, 10, -1, 11, 7, -1, -1, 15, 14, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 6, 3, 4, 1, 2, -1, -1, 5, 10, 7, 9, 11, 12, -1, -1, 8, 14, -1, -1, 15, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 6, 2, 5, 9, -1, -1, -1, 10, 1, 8, -1, 12, 14, 4, -1, 15, 7, -1, 13, 3, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 8, 10, 9, 15, 1, -1, 4, 0, 3, 2, -1, 6, -1, 12, 11, 13, 7, 14, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 3, 6, 0, 4, 2, -1, 7, 13, 8, 9, 11, -1, -1, 15, -1, -1, -1, -1, -1, 10, 5, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 3, 0, 1, 4, -1, 2, 5, 6, 7, 8, -1, 14, -1, -1, 9, 15, -1, 12, -1, -1, -1, 10, 11, -1, -1, -1, 13, -1, -1, -1, -1, -1, + 0, 1, 3, 2, 15, -1, 12, -1, 7, 14, 4, -1, -1, 9, -1, 8, 5, 10, -1, -1, 6, -1, 13, -1, -1, -1, 11, -1, -1, -1, -1, -1, + 0, 3, 1, 2, -1, -1, 12, 6, 4, 9, 7, -1, -1, 14, 8, -1, -1, 15, 11, 13, 5, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 5, 7, 2, 10, 13, -1, 6, 8, 1, 3, -1, -1, 14, 15, 11, -1, -1, -1, 12, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 2, 6, 3, 7, 10, -1, 1, 9, 4, 8, -1, -1, 15, -1, 12, 5, -1, -1, -1, 11, -1, 13, -1, -1, -1, 14, -1, -1, -1, -1, -1, + 1, 3, 4, 0, 7, -1, 12, 2, 11, 8, 6, 13, -1, -1, -1, -1, -1, 5, -1, -1, 10, 15, 9, -1, -1, -1, 14, -1, -1, -1, -1, -1, + 1, 3, 5, 2, 13, 0, 9, 4, 7, 6, 8, -1, -1, 15, -1, 11, -1, -1, 10, -1, 14, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 2, 1, 3, -1, -1, -1, 6, -1, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 1, 11, 4, 0, 3, -1, 13, 12, 2, 7, -1, -1, 15, 10, 5, 8, 14, -1, -1, -1, -1, -1, 9, -1, -1, -1, 6, -1, -1, -1, -1, -1, + 0, 9, 2, 14, 15, 4, 1, 13, 3, 5, -1, -1, 10, -1, -1, -1, -1, 6, 12, -1, 7, -1, 8, -1, -1, -1, 11, -1, -1, -1, -1, -1, + -1, 2, 14, -1, 1, 5, 8, 7, 4, 12, -1, 6, 9, 11, 13, 3, 10, 15, -1, -1, -1, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 3, 2, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 4, 3, 1, 5, -1, -1, -1, 0, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 2, 8, 4, 1, -1, 0, -1, 6, -1, -1, 5, -1, 7, -1, -1, -1, -1, -1, -1, -1, 10, -1, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1, + 12, 5, -1, -1, 1, -1, -1, 7, 0, 3, -1, 2, -1, 4, 6, -1, -1, -1, -1, 8, -1, -1, 15, -1, 13, 9, -1, -1, -1, -1, -1, 11, + 1, 3, 2, 4, -1, -1, -1, 5, -1, 7, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, 8, -1, -1, + 5, 3, 4, 12, 1, 6, -1, -1, -1, -1, 8, 2, -1, -1, -1, -1, 0, 9, -1, -1, 11, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 0, -1, 1, 12, 3, -1, -1, -1, -1, 5, -1, -1, -1, 2, -1, -1, -1, -1, -1, -1, -1, -1, 4, -1, -1, 6, -1, 10, + 2, 3, 1, 4, -1, 0, -1, 5, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 7, -1, -1, -1, -1, -1, -1, -1, -1, 6, -1, -1, + 5, 1, 3, 0, -1, -1, -1, -1, -1, -1, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, -1, -1, -1, -1, -1, 9, -1, -1, 6, -1, 7, }, successors_reversed = { - {'s', 't', 'c', 'l', 'm', 'a', 'd', 'r', 'v', 'T', 'A', 'L', 'e', 'M', 'Y', '-'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'-', 't', 'a', 'b', 's', 'h', 'c', 'r', 'n', 'w', 'p', 'm', 'l', 'd', 'i', 'f'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'u', 'e', 'i', 'a', 'o', 'r', 'y', 'l', 'I', 'E', 'R', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'e', 'a', 'o', 'i', 'u', 'A', 'y', 'E', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'t', 'n', 'f', 's', '\'', 'm', 'I', 'N', 'A', 'E', 'L', 'Z', 'r', 'V', 'R', 'C'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'o', 'a', 'y', 'i', 'u', 'e', 'I', 'L', 'D', '\'', 'E', 'Y', '\x00', '\x00', '\x00', '\x00'}, - {'r', 'i', 'y', 'a', 'e', 'o', 'u', 'Y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'h', 'o', 'e', 'E', 'i', 'u', 'r', 'w', 'a', 'H', 'y', 'R', 'Z', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'h', 'i', 'e', 'a', 'o', 'r', 'I', 'y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'n', 't', 's', 'r', 'l', 'd', 'i', 'y', 'v', 'm', 'b', 'c', 'g', 'p', 'k', 'u'}, - {'e', 'l', 'o', 'u', 'y', 'a', 'r', 'i', 's', 'j', 't', 'b', 'v', 'h', 'm', 'd'}, - {'o', 'e', 'h', 'a', 't', 'k', 'i', 'r', 'l', 'u', 'y', 'c', 'q', 's', '-', 'd'}, - {'e', 'i', 'o', 'a', 's', 'y', 'r', 'u', 'd', 'l', '-', 'g', 'n', 'v', 'm', 'f'}, - {'r', 'n', 'd', 's', 'a', 'l', 't', 'e', 'm', 'c', 'v', 'y', 'i', 'x', 'f', 'p'}, - {'o', 'e', 'r', 'a', 'i', 'f', 'u', 't', 'l', '-', 'y', 's', 'n', 'c', '\'', 'k'}, - {'h', 'e', 'o', 'a', 'r', 'i', 'l', 's', 'u', 'n', 'g', 'b', '-', 't', 'y', 'm'}, - {'e', 'a', 'i', 'o', 't', 'r', 'u', 'y', 'm', 's', 'l', 'b', '\'', '-', 'f', 'd'}, - {'n', 's', 't', 'm', 'o', 'l', 'c', 'd', 'r', 'e', 'g', 'a', 'f', 'v', 'z', 'b'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'e', 'n', 'i', 's', 'h', 'l', 'f', 'y', '-', 'a', 'w', '\'', 'g', 'r', 'o', 't'}, - {'e', 'l', 'i', 'y', 'd', 'o', 'a', 'f', 'u', 't', 's', 'k', 'w', 'v', 'm', 'p'}, - {'e', 'a', 'o', 'i', 'u', 'p', 'y', 's', 'b', 'm', 'f', '\'', 'n', '-', 'l', 't'}, - {'d', 'g', 'e', 't', 'o', 'c', 's', 'i', 'a', 'n', 'y', 'l', 'k', '\'', 'f', 'v'}, - {'u', 'n', 'r', 'f', 'm', 't', 'w', 'o', 's', 'l', 'v', 'd', 'p', 'k', 'i', 'c'}, - {'e', 'r', 'a', 'o', 'l', 'p', 'i', 't', 'u', 's', 'h', 'y', 'b', '-', '\'', 'm'}, - {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'e', 'i', 'o', 'a', 's', 'y', 't', 'd', 'r', 'n', 'c', 'm', 'l', 'u', 'g', 'f'}, - {'e', 't', 'h', 'i', 'o', 's', 'a', 'u', 'p', 'c', 'l', 'w', 'm', 'k', 'f', 'y'}, - {'h', 'o', 'e', 'i', 'a', 't', 'r', 'u', 'y', 'l', 's', 'w', 'c', 'f', '\'', '-'}, - {'r', 't', 'l', 's', 'n', 'g', 'c', 'p', 'e', 'i', 'a', 'd', 'm', 'b', 'f', 'o'}, - {'e', 'i', 'a', 'o', 'y', 'u', 'r', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}, - {'a', 'i', 'h', 'e', 'o', 'n', 'r', 's', 'l', 'd', 'k', '-', 'f', '\'', 'c', 'b'}, - {'p', 't', 'c', 'a', 'i', 'e', 'h', 'q', 'u', 'f', '-', 'y', 'o', '\x00', '\x00', '\x00'}, - {'o', 'e', 's', 't', 'i', 'd', '\'', 'l', 'b', '-', 'm', 'a', 'r', 'n', 'p', 'w'}, + 's', 't', 'c', 'l', 'm', 'a', 'd', 'r', 'v', 'T', 'A', 'L', 'e', 'M', 'Y', '-', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '-', 't', 'a', 'b', 's', 'h', 'c', 'r', 'n', 'w', 'p', 'm', 'l', 'd', 'i', 'f', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'u', 'e', 'i', 'a', 'o', 'r', 'y', 'l', 'I', 'E', 'R', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'e', 'a', 'o', 'i', 'u', 'A', 'y', 'E', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 't', 'n', 'f', 's', '\'', 'm', 'I', 'N', 'A', 'E', 'L', 'Z', 'r', 'V', 'R', 'C', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'o', 'a', 'y', 'i', 'u', 'e', 'I', 'L', 'D', '\'', 'E', 'Y', '\x00', '\x00', '\x00', '\x00', + 'r', 'i', 'y', 'a', 'e', 'o', 'u', 'Y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'h', 'o', 'e', 'E', 'i', 'u', 'r', 'w', 'a', 'H', 'y', 'R', 'Z', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'h', 'i', 'e', 'a', 'o', 'r', 'I', 'y', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'n', 't', 's', 'r', 'l', 'd', 'i', 'y', 'v', 'm', 'b', 'c', 'g', 'p', 'k', 'u', + 'e', 'l', 'o', 'u', 'y', 'a', 'r', 'i', 's', 'j', 't', 'b', 'v', 'h', 'm', 'd', + 'o', 'e', 'h', 'a', 't', 'k', 'i', 'r', 'l', 'u', 'y', 'c', 'q', 's', '-', 'd', + 'e', 'i', 'o', 'a', 's', 'y', 'r', 'u', 'd', 'l', '-', 'g', 'n', 'v', 'm', 'f', + 'r', 'n', 'd', 's', 'a', 'l', 't', 'e', 'm', 'c', 'v', 'y', 'i', 'x', 'f', 'p', + 'o', 'e', 'r', 'a', 'i', 'f', 'u', 't', 'l', '-', 'y', 's', 'n', 'c', '\'', 'k', + 'h', 'e', 'o', 'a', 'r', 'i', 'l', 's', 'u', 'n', 'g', 'b', '-', 't', 'y', 'm', + 'e', 'a', 'i', 'o', 't', 'r', 'u', 'y', 'm', 's', 'l', 'b', '\'', '-', 'f', 'd', + 'n', 's', 't', 'm', 'o', 'l', 'c', 'd', 'r', 'e', 'g', 'a', 'f', 'v', 'z', 'b', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'e', 'n', 'i', 's', 'h', 'l', 'f', 'y', '-', 'a', 'w', '\'', 'g', 'r', 'o', 't', + 'e', 'l', 'i', 'y', 'd', 'o', 'a', 'f', 'u', 't', 's', 'k', 'w', 'v', 'm', 'p', + 'e', 'a', 'o', 'i', 'u', 'p', 'y', 's', 'b', 'm', 'f', '\'', 'n', '-', 'l', 't', + 'd', 'g', 'e', 't', 'o', 'c', 's', 'i', 'a', 'n', 'y', 'l', 'k', '\'', 'f', 'v', + 'u', 'n', 'r', 'f', 'm', 't', 'w', 'o', 's', 'l', 'v', 'd', 'p', 'k', 'i', 'c', + 'e', 'r', 'a', 'o', 'l', 'p', 'i', 't', 'u', 's', 'h', 'y', 'b', '-', '\'', 'm', + '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'e', 'i', 'o', 'a', 's', 'y', 't', 'd', 'r', 'n', 'c', 'm', 'l', 'u', 'g', 'f', + 'e', 't', 'h', 'i', 'o', 's', 'a', 'u', 'p', 'c', 'l', 'w', 'm', 'k', 'f', 'y', + 'h', 'o', 'e', 'i', 'a', 't', 'r', 'u', 'y', 'l', 's', 'w', 'c', 'f', '\'', '-', + 'r', 't', 'l', 's', 'n', 'g', 'c', 'p', 'e', 'i', 'a', 'd', 'm', 'b', 'f', 'o', + 'e', 'i', 'a', 'o', 'y', 'u', 'r', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', + 'a', 'i', 'h', 'e', 'o', 'n', 'r', 's', 'l', 'd', 'k', '-', 'f', '\'', 'c', 'b', + 'p', 't', 'c', 'a', 'i', 'e', 'h', 'q', 'u', 'f', '-', 'y', 'o', '\x00', '\x00', '\x00', + 'o', 'e', 's', 't', 'i', 'd', '\'', 'l', 'b', '-', 'm', 'a', 'r', 'n', 'p', 'w', }, character_count = 32, diff --git a/core/compress/shoco/shoco.odin b/core/compress/shoco/shoco.odin index 9c5008f5d..3f5b696ea 100644 --- a/core/compress/shoco/shoco.odin +++ b/core/compress/shoco/shoco.odin @@ -14,6 +14,8 @@ package shoco import "core:intrinsics" import "core:compress" +import "core:fmt" + Shoco_Pack :: struct { word: u32, bytes_packed: i8, @@ -29,8 +31,8 @@ Shoco_Model :: struct { max_char: u8, characters_by_id: []u8, ids_by_character: [256]i16, - successors_by_bigram: [][]i8, - successors_reversed: [][]u8, + successors_by_bigram: []i8, + successors_reversed: []u8, character_count: u8, successor_count: u8, @@ -74,19 +76,15 @@ find_best_encoding :: proc(indices: []i16, n_consecutive: i8, model := DEFAULT_M } validate_model :: proc(model: Shoco_Model) -> (int, compress.Error) { - if len(model.successors_reversed) != int(model.max_char - model.min_char) { - return 0, .Unknown_Compression_Method - } - if len(model.characters_by_id) != int(model.character_count) { return 0, .Unknown_Compression_Method } - if len(model.successors_by_bigram) != int(model.character_count) || len(model.successors_by_bigram[0]) != int(model.character_count) { + if len(model.successors_by_bigram) != int(model.character_count) * int(model.character_count) { return 0, .Unknown_Compression_Method } - if len(model.successors_reversed[0]) != int(model.successor_count) { + if len(model.successors_reversed) != int(model.successor_count) * int(model.max_char - model.min_char) { return 0, .Unknown_Compression_Method } @@ -155,7 +153,11 @@ decompress_slice_to_output_buffer :: proc(input: []u8, output: []u8, model := DE offset = pack.offsets[i] mask = pack.masks[i] - last_chr = model.successors_reversed[last_chr - model.min_char][(code >> offset) & u32(mask)] + index_major := u32(last_chr - model.min_char) * u32(model.successor_count) + index_minor := (code >> offset) & u32(mask) + + last_chr = model.successors_reversed[index_major + index_minor] + output[out + i] = last_chr } @@ -242,7 +244,7 @@ compress_string_to_buffer :: proc(input: string, output: []u8, model := DEFAULT_ break } - successor_index := model.successors_by_bigram[last_chr_index][current_index] + successor_index := model.successors_by_bigram[last_chr_index * i16(model.character_count) + current_index] if successor_index < 0 { break } From b022167df12b3d62ddacd51e64f7ab3b4e170852 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 22 Apr 2022 17:56:34 +0200 Subject: [PATCH 31/79] Remove unused fmt. --- core/compress/shoco/shoco.odin | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/compress/shoco/shoco.odin b/core/compress/shoco/shoco.odin index 3f5b696ea..f94ce70b7 100644 --- a/core/compress/shoco/shoco.odin +++ b/core/compress/shoco/shoco.odin @@ -14,8 +14,6 @@ package shoco import "core:intrinsics" import "core:compress" -import "core:fmt" - Shoco_Pack :: struct { word: u32, bytes_packed: i8, From b44b6e7e5099cab83f4a6d0feb5af9f245dea738 Mon Sep 17 00:00:00 2001 From: Tetralux Date: Sat, 23 Apr 2022 03:33:35 +0000 Subject: [PATCH 32/79] [path/filepath] Add file stem and long-extension procedures Adds stem(), short_stem(), and long_ext(); also adds doc-comments to base() and ext(). The 'stem' is usually 'the name' of the file; the basename without the file extension. To this end, this adds stem(), which is such that: stem(path) + ext(path) = base(path) However, 'file extension' has two different meanings to what constitutes it! > What is the extension of: 'name.tar.gz' ? Colloquially, you would likely think of it as 'a tarball' - which you might think is '.tar.gz'. But, if you're writing code to process a file of this type, you would first treat it as a Gzip file, and then treat the result as a TAR file - i.e: '.gz' ... _followed by_ '.tar'. ext() returns '.gz' here, since that is the most-immediate format that you would need to use to decode it; it would be a Gzip stream. Sometimes though, you do actually want to consider these longer file extensions. Perhaps you're extracting a tarball, and what to know what to call the intermediate tar file; perhaps you want to check to see if this file is a tarball, or just a Gzip file; or maybe you just want 'the name' of the file, and not this "strange 'name-and-part-of-the-extension' thing". So, this also adds short_stem() and long_ext(), such that: short_stem(path) + long_ext(path) = base(path) Thus, we can use either, but the most immediately-useful one is the easiest to reach for: stem('name.tar.gz') -> 'name.tar' ext('name.tar.gz') -> '.gz' short_stem('name.tar.gz') -> 'name' long_ext('name.tar.gz') -> '.tar.gz' These procedures are identical to their counterparts when the path only has a simple extension: stem('name.txt') -> 'name' ext('name.txt') -> '.txt' short_stem('name.txt') -> 'name' long_ext('name.txt') -> '.txt' --- core/path/filepath/path.odin | 133 ++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 9 deletions(-) diff --git a/core/path/filepath/path.odin b/core/path/filepath/path.odin index 42714d736..32e4a8a37 100644 --- a/core/path/filepath/path.odin +++ b/core/path/filepath/path.odin @@ -4,6 +4,8 @@ package filepath import "core:strings" +SEPARATOR_CHARS :: `/\` + // is_separator checks whether the byte is a valid separator character is_separator :: proc(c: byte) -> bool { switch c { @@ -69,6 +71,16 @@ volume_name_len :: proc(path: string) -> int { return 0 } +/* + Gets the file name and extension from a path. + + i.e: + 'path/to/name.tar.gz' -> 'name.tar.gz' + 'path/to/name.txt' -> 'name.txt' + 'path/to/name' -> 'name' + + Returns "." if the path is an empty string. +*/ base :: proc(path: string) -> string { if path == "" { return "." @@ -94,6 +106,118 @@ base :: proc(path: string) -> string { return path } +/* + Gets the name of a file from a path. + + The stem of a file is such that stem(path) + ext(path) = base(path). + + Only the last dot is considered when splitting the file extension. + See `short_stem`. + + i.e: + 'name.tar.gz' -> 'name.tar' + 'name.txt' -> 'name' + + Returns an empty string if there is no stem. e.g: '.gitignore'. + Returns an empty string if there's a trailing path separator. +*/ +stem :: proc(path: string) -> string { + if len(path) > 0 && is_separator(path[len(path) - 1]) { + // NOTE(tetra): Trailing separator + return "" + } + + // NOTE(tetra): Get the basename + path := path + if i := strings.last_index_any(path, SEPARATOR_CHARS); i != -1 { + path = path[i+1:] + } + + if i := strings.last_index_byte(path, '.'); i != -1 { + return path[:i] + } + + return path +} + +/* + Gets the name of a file from a path. + + The short stem is such that short_stem(path) + long_ext(path) = base(path). + + The first dot is used to split off the file extension, unlike `stem` which uses the last dot. + + i.e: + 'name.tar.gz' -> 'name' + 'name.txt' -> 'name' + + Returns an empty string if there is no stem. e.g: '.gitignore'. + Returns an empty string if there's a trailing path separator. +*/ +short_stem :: proc(path: string) -> string { + s := stem(path) + if i := strings.index_byte(s, '.'); i != -1 { + return s[:i] + } + return s +} + +/* + Gets the file extension from a path, including the dot. + + The file extension is such that stem(path) + ext(path) = base(path). + + Only the last dot is considered when splitting the file extension. + See `long_ext`. + + i.e: + 'name.tar.gz' -> '.gz' + 'name.txt' -> '.txt' + + Returns an empty string if there is no dot. + Returns an empty string if there is a trailing path separator. +*/ +ext :: proc(path: string) -> string { + for i := len(path)-1; i >= 0 && !is_separator(path[i]); i -= 1 { + if path[i] == '.' { + return path[i:] + } + } + return "" +} + +/* + Gets the file extension from a path, including the dot. + + The long file extension is such that short_stem(path) + long_ext(path) = base(path). + + The first dot is used to split off the file extension, unlike `ext` which uses the last dot. + + i.e: + 'name.tar.gz' -> '.tar.gz' + 'name.txt' -> '.txt' + + Returns an empty string if there is no dot. + Returns an empty string if there is a trailing path separator. +*/ +long_ext :: proc(path: string) -> string { + if len(path) > 0 && is_separator(path[len(path) - 1]) { + // NOTE(tetra): Trailing separator + return "" + } + + // NOTE(tetra): Get the basename + path := path + if i := strings.last_index_any(path, SEPARATOR_CHARS); i != -1 { + path = path[i+1:] + } + + if i := strings.index_byte(path, '.'); i != -1 { + return path[i:] + } + + return "" +} clean :: proc(path: string, allocator := context.allocator) -> string { context.allocator = allocator @@ -189,15 +313,6 @@ to_slash :: proc(path: string, allocator := context.allocator) -> (new_path: str return strings.replace_all(path, SEPARATOR_STRING, "/", allocator) } -ext :: proc(path: string) -> string { - for i := len(path)-1; i >= 0 && !is_separator(path[i]); i -= 1 { - if path[i] == '.' { - return path[i:] - } - } - return "" -} - Relative_Error :: enum { None, From 3cab2592c3e5a06882ffd711871a08c893b043f1 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 6 Apr 2022 18:26:23 +0200 Subject: [PATCH 33/79] Compiler: Add early error for output path being a directory. - Introduce new `Path` type and an array of build paths on the build context. - Resolve input and output paths/files early (before parsing). - Error early if inputs are missing or outputs are directories. - Plumb new file path generation into linker stage instead of its adhoc method. TODO: - Remove more adhoc file path generation in parser and linker stage. - Make intermediate object file generation use new path system. - Round out and robustify Path helper functions. --- .gitignore | 1 + Makefile | 4 +- build_odin.sh | 4 +- src/build_settings.cpp | 220 +++++++++++++++++++--- src/common.cpp | 257 +------------------------- src/gb/gb.h | 50 +++-- src/llvm_backend.cpp | 14 +- src/llvm_backend_general.cpp | 1 - src/main.cpp | 152 ++++++++-------- src/parser.cpp | 2 +- src/path.cpp | 333 ++++++++++++++++++++++++++++++++++ src/string.cpp | 10 +- tests/core/build.bat | 28 +-- tests/core/math/big/build.bat | 2 +- tests/issues/run.sh | 4 +- 15 files changed, 676 insertions(+), 406 deletions(-) create mode 100644 src/path.cpp diff --git a/.gitignore b/.gitignore index e8b3d3050..d03a86fd7 100644 --- a/.gitignore +++ b/.gitignore @@ -269,6 +269,7 @@ bin/ # - Linux/MacOS odin odin.dSYM +*.bin # shared collection shared/ diff --git a/Makefile b/Makefile index 82150c6a2..1a1c93180 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -all: debug demo +all: debug demo: - ./odin run examples/demo/demo.odin + ./odin run examples/demo report: ./odin report diff --git a/build_odin.sh b/build_odin.sh index aef3f2836..4810cafd2 100755 --- a/build_odin.sh +++ b/build_odin.sh @@ -102,7 +102,7 @@ build_odin() { } run_demo() { - ./odin run examples/demo/demo.odin -file + ./odin run examples/demo } case $OS in @@ -147,4 +147,4 @@ if [[ $# -eq 1 ]]; then exit 0 else panic "Too many arguments!" -fi +fi \ No newline at end of file diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 2f3eb03a5..0b582eac8 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -3,7 +3,6 @@ #include #endif - // #if defined(GB_SYSTEM_WINDOWS) // #define DEFAULT_TO_THREADED_CHECKER // #endif @@ -198,6 +197,22 @@ enum RelocMode : u8 { RelocMode_DynamicNoPIC, }; +enum BuildPath : u8 { + BuildPath_Main_Package, // Input Path to the package directory (or file) we're building. + BuildPath_RC, // Input Path for .rc file, can be set with `-resource:`. + BuildPath_RES, // Output Path for .res file, generated from previous. + BuildPath_Win_SDK_Root, // windows_sdk_root + BuildPath_Win_SDK_UM_Lib, // windows_sdk_um_library_path + BuildPath_Win_SDK_UCRT_Lib, // windows_sdk_ucrt_library_path + BuildPath_VS_EXE, // vs_exe_path + BuildPath_VS_LIB, // vs_library_path + + BuildPath_Output, // Output Path for .exe, .dll, .so, etc. Can be overridden with `-out:`. + BuildPath_PDB, // Output Path for .pdb file, can be overridden with `-pdb-name:`. + + BuildPathCOUNT, +}; + // This stores the information for the specify architecture of this build struct BuildContext { // Constants @@ -226,9 +241,13 @@ struct BuildContext { bool show_help; + Array build_paths; // Contains `Path` objects to output filename, pdb, resource and intermediate files. + // BuildPath enum contains the indices of paths we know *before* the work starts. + String out_filepath; String resource_filepath; String pdb_filepath; + bool has_resource; String link_flags; String extra_linker_flags; @@ -300,8 +319,6 @@ struct BuildContext { }; - - gb_global BuildContext build_context = {0}; bool global_warnings_as_errors(void) { @@ -605,28 +622,6 @@ bool allow_check_foreign_filepath(void) { // is_abs_path // has_subdir -enum TargetFileValidity : u8 { - TargetFileValidity_Invalid, - - TargetFileValidity_Writable_File, - TargetFileValidity_No_Write_Permission, - TargetFileValidity_Directory, - - TargetTargetFileValidity_COUNT, -}; - -TargetFileValidity set_output_filename(void) { - // Assembles the output filename from build_context information. - // Returns `true` if it doesn't exist or is a file. - // Returns `false` if a directory or write-protected file. - - - - - return TargetFileValidity_Writable_File; -} - - String const WIN32_SEPARATOR_STRING = {cast(u8 *)"\\", 1}; String const NIX_SEPARATOR_STRING = {cast(u8 *)"/", 1}; @@ -973,7 +968,6 @@ char *token_pos_to_string(TokenPos const &pos) { return s; } - void init_build_context(TargetMetrics *cross_target) { BuildContext *bc = &build_context; @@ -1152,8 +1146,178 @@ void init_build_context(TargetMetrics *cross_target) { bc->optimization_level = gb_clamp(bc->optimization_level, 0, 3); - - #undef LINK_FLAG_X64 #undef LINK_FLAG_386 } + +#if defined(GB_SYSTEM_WINDOWS) +// NOTE(IC): In order to find Visual C++ paths without relying on environment variables. +// NOTE(Jeroen): No longer needed in `main.cpp -> linker_stage`. We now resolve those paths in `init_build_paths`. +#include "microsoft_craziness.h" +#endif + +// NOTE(Jeroen): Set/create the output and other paths and report an error as appropriate. +// We've previously called `parse_build_flags`, so `out_filepath` should be set. +bool init_build_paths(String init_filename) { + gbAllocator ha = heap_allocator(); + BuildContext *bc = &build_context; + + // NOTE(Jeroen): We're pre-allocating BuildPathCOUNT slots so that certain paths are always at the same enumerated index. + array_init(&bc->build_paths, permanent_allocator(), BuildPathCOUNT); + + // [BuildPathMainPackage] Turn given init path into a `Path`, which includes normalizing it into a full path. + bc->build_paths[BuildPath_Main_Package] = path_from_string(ha, init_filename); + + bool produces_output_file = false; + if (bc->command_kind == Command_doc && bc->cmd_doc_flags & CmdDocFlag_DocFormat) { + produces_output_file = true; + } else if (bc->command_kind & Command__does_build) { + produces_output_file = true; + } + + if (!produces_output_file) { + // Command doesn't produce output files. We're done. + return true; + } + + #if defined(GB_SYSTEM_WINDOWS) + if (bc->resource_filepath.len > 0) { + bc->build_paths[BuildPath_RC] = path_from_string(ha, bc->resource_filepath); + bc->build_paths[BuildPath_RES] = path_from_string(ha, bc->resource_filepath); + bc->build_paths[BuildPath_RC].ext = copy_string(ha, STR_LIT("rc")); + bc->build_paths[BuildPath_RES].ext = copy_string(ha, STR_LIT("res")); + } + + if (bc->pdb_filepath.len > 0) { + bc->build_paths[BuildPath_PDB] = path_from_string(ha, bc->pdb_filepath); + } + + if ((bc->command_kind & Command__does_build) && (!bc->ignore_microsoft_magic)) { + // NOTE(ic): It would be nice to extend this so that we could specify the Visual Studio version that we want instead of defaulting to the latest. + Find_Result_Utf8 find_result = find_visual_studio_and_windows_sdk_utf8(); + + if (find_result.windows_sdk_version == 0) { + gb_printf_err("Windows SDK not found.\n"); + return false; + } + + GB_ASSERT(find_result.windows_sdk_um_library_path.len > 0); + GB_ASSERT(find_result.windows_sdk_ucrt_library_path.len > 0); + + if (find_result.windows_sdk_root.len > 0) { + bc->build_paths[BuildPath_Win_SDK_Root] = path_from_string(ha, find_result.windows_sdk_root); + } + + if (find_result.windows_sdk_um_library_path.len > 0) { + bc->build_paths[BuildPath_Win_SDK_UM_Lib] = path_from_string(ha, find_result.windows_sdk_um_library_path); + } + + if (find_result.windows_sdk_ucrt_library_path.len > 0) { + bc->build_paths[BuildPath_Win_SDK_UCRT_Lib] = path_from_string(ha, find_result.windows_sdk_ucrt_library_path); + } + + if (find_result.vs_exe_path.len > 0) { + bc->build_paths[BuildPath_VS_EXE] = path_from_string(ha, find_result.vs_exe_path); + } + + if (find_result.vs_library_path.len > 0) { + bc->build_paths[BuildPath_VS_LIB] = path_from_string(ha, find_result.vs_library_path); + } + + gb_free(ha, find_result.windows_sdk_root.text); + gb_free(ha, find_result.windows_sdk_um_library_path.text); + gb_free(ha, find_result.windows_sdk_ucrt_library_path.text); + gb_free(ha, find_result.vs_exe_path.text); + gb_free(ha, find_result.vs_library_path.text); + + } + #endif + + // All the build targets and OSes. + String output_extension; + + if (bc->command_kind == Command_doc && bc->cmd_doc_flags & CmdDocFlag_DocFormat) { + output_extension = STR_LIT("odin-doc"); + } else if (is_arch_wasm()) { + output_extension = STR_LIT("wasm"); + } else if (build_context.build_mode == BuildMode_Executable) { + // By default use a .bin executable extension. + output_extension = STR_LIT("bin"); + + if (build_context.metrics.os == TargetOs_windows) { + output_extension = STR_LIT("exe"); + } else if (build_context.cross_compiling && selected_target_metrics->metrics == &target_essence_amd64) { + output_extension = make_string(nullptr, 0); + } + } else if (build_context.build_mode == BuildMode_DynamicLibrary) { + // By default use a .so shared library extension. + output_extension = STR_LIT("so"); + + if (build_context.metrics.os == TargetOs_windows) { + output_extension = STR_LIT("dll"); + } else if (build_context.metrics.os == TargetOs_darwin) { + output_extension = STR_LIT("dylib"); + } + } else if (build_context.build_mode == BuildMode_Object) { + // By default use a .o object extension. + output_extension = STR_LIT("o"); + + if (build_context.metrics.os == TargetOs_windows) { + output_extension = STR_LIT("obj"); + } + } else if (build_context.build_mode == BuildMode_Assembly) { + // By default use a .S asm extension. + output_extension = STR_LIT("S"); + } else if (build_context.build_mode == BuildMode_LLVM_IR) { + output_extension = STR_LIT("ll"); + } else { + GB_PANIC("Unhandled build mode/target combination.\n"); + } + + if (bc->out_filepath.len > 0) { + bc->build_paths[BuildPath_Output] = path_from_string(ha, bc->out_filepath); + } else { + String output_name = remove_directory_from_path(init_filename); + output_name = remove_extension_from_path(output_name); + output_name = copy_string(ha, string_trim_whitespace(output_name)); + + /* + NOTE(Jeroen): This fallback substitution can't be made at this stage. + if (gen->output_name.len == 0) { + gen->output_name = c->info.init_scope->pkg->name; + } + */ + Path output_path = path_from_string(ha, output_name); + + #ifndef GB_SYSTEM_WINDOWS + char cwd[4096]; + getcwd(&cwd[0], 4096); + + const u8 * cwd_str = (const u8 *)&cwd[0]; + output_path.basename = copy_string(ha, make_string(cwd_str, strlen(cwd))); + #endif + + // Replace extension. + if (output_path.ext.len > 0) { + gb_free(ha, output_path.ext.text); + } + output_path.ext = copy_string(ha, output_extension); + + bc->build_paths[BuildPath_Output] = output_path; + } + + // Do we have an extension? We might not if the output filename was supplied. + if (bc->build_paths[BuildPath_Output].ext.len == 0) { + bc->build_paths[BuildPath_Output].ext = copy_string(ha, output_extension); + } + + // Check if output path is a directory. + if (path_is_directory(bc->build_paths[BuildPath_Output])) { + String output_file = path_to_string(ha, bc->build_paths[BuildPath_Output]); + defer (gb_free(ha, output_file.text)); + gb_printf_err("Output path %.*s is a directory.\n", LIT(output_file)); + return false; + } + + return true; +} \ No newline at end of file diff --git a/src/common.cpp b/src/common.cpp index aaacda04b..94248fb62 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -675,262 +675,7 @@ wchar_t **command_line_to_wargv(wchar_t *cmd_line, int *_argc) { #endif - -#if defined(GB_SYSTEM_WINDOWS) - bool path_is_directory(String path) { - gbAllocator a = heap_allocator(); - String16 wstr = string_to_string16(a, path); - defer (gb_free(a, wstr.text)); - - i32 attribs = GetFileAttributesW(wstr.text); - if (attribs < 0) return false; - - return (attribs & FILE_ATTRIBUTE_DIRECTORY) != 0; - } - -#else - bool path_is_directory(String path) { - gbAllocator a = heap_allocator(); - char *copy = cast(char *)copy_string(a, path).text; - defer (gb_free(a, copy)); - - struct stat s; - if (stat(copy, &s) == 0) { - return (s.st_mode & S_IFDIR) != 0; - } - return false; - } -#endif - - -String path_to_full_path(gbAllocator a, String path) { - gbAllocator ha = heap_allocator(); - char *path_c = gb_alloc_str_len(ha, cast(char *)path.text, path.len); - defer (gb_free(ha, path_c)); - - char *fullpath = gb_path_get_full_name(a, path_c); - String res = string_trim_whitespace(make_string_c(fullpath)); -#if defined(GB_SYSTEM_WINDOWS) - for (isize i = 0; i < res.len; i++) { - if (res.text[i] == '\\') { - res.text[i] = '/'; - } - } -#endif - return res; -} - - - -struct FileInfo { - String name; - String fullpath; - i64 size; - bool is_dir; -}; - -enum ReadDirectoryError { - ReadDirectory_None, - - ReadDirectory_InvalidPath, - ReadDirectory_NotExists, - ReadDirectory_Permission, - ReadDirectory_NotDir, - ReadDirectory_Empty, - ReadDirectory_Unknown, - - ReadDirectory_COUNT, -}; - -i64 get_file_size(String path) { - char *c_str = alloc_cstring(heap_allocator(), path); - defer (gb_free(heap_allocator(), c_str)); - - gbFile f = {}; - gbFileError err = gb_file_open(&f, c_str); - defer (gb_file_close(&f)); - if (err != gbFileError_None) { - return -1; - } - return gb_file_size(&f); -} - - -#if defined(GB_SYSTEM_WINDOWS) -ReadDirectoryError read_directory(String path, Array *fi) { - GB_ASSERT(fi != nullptr); - - gbAllocator a = heap_allocator(); - - while (path.len > 0) { - Rune end = path[path.len-1]; - if (end == '/') { - path.len -= 1; - } else if (end == '\\') { - path.len -= 1; - } else { - break; - } - } - - if (path.len == 0) { - return ReadDirectory_InvalidPath; - } - { - char *c_str = alloc_cstring(a, path); - defer (gb_free(a, c_str)); - - gbFile f = {}; - gbFileError file_err = gb_file_open(&f, c_str); - defer (gb_file_close(&f)); - - switch (file_err) { - case gbFileError_Invalid: return ReadDirectory_InvalidPath; - case gbFileError_NotExists: return ReadDirectory_NotExists; - // case gbFileError_Permission: return ReadDirectory_Permission; - } - } - - if (!path_is_directory(path)) { - return ReadDirectory_NotDir; - } - - - char *new_path = gb_alloc_array(a, char, path.len+3); - defer (gb_free(a, new_path)); - - gb_memmove(new_path, path.text, path.len); - gb_memmove(new_path+path.len, "/*", 2); - new_path[path.len+2] = 0; - - String np = make_string(cast(u8 *)new_path, path.len+2); - String16 wstr = string_to_string16(a, np); - defer (gb_free(a, wstr.text)); - - WIN32_FIND_DATAW file_data = {}; - HANDLE find_file = FindFirstFileW(wstr.text, &file_data); - if (find_file == INVALID_HANDLE_VALUE) { - return ReadDirectory_Unknown; - } - defer (FindClose(find_file)); - - array_init(fi, a, 0, 100); - - do { - wchar_t *filename_w = file_data.cFileName; - i64 size = cast(i64)file_data.nFileSizeLow; - size |= (cast(i64)file_data.nFileSizeHigh) << 32; - String name = string16_to_string(a, make_string16_c(filename_w)); - if (name == "." || name == "..") { - gb_free(a, name.text); - continue; - } - - String filepath = {}; - filepath.len = path.len+1+name.len; - filepath.text = gb_alloc_array(a, u8, filepath.len+1); - defer (gb_free(a, filepath.text)); - gb_memmove(filepath.text, path.text, path.len); - gb_memmove(filepath.text+path.len, "/", 1); - gb_memmove(filepath.text+path.len+1, name.text, name.len); - - FileInfo info = {}; - info.name = name; - info.fullpath = path_to_full_path(a, filepath); - info.size = size; - info.is_dir = (file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; - array_add(fi, info); - } while (FindNextFileW(find_file, &file_data)); - - if (fi->count == 0) { - return ReadDirectory_Empty; - } - - return ReadDirectory_None; -} -#elif defined(GB_SYSTEM_LINUX) || defined(GB_SYSTEM_OSX) || defined(GB_SYSTEM_FREEBSD) || defined(GB_SYSTEM_OPENBSD) - -#include - -ReadDirectoryError read_directory(String path, Array *fi) { - GB_ASSERT(fi != nullptr); - - gbAllocator a = heap_allocator(); - - char *c_path = alloc_cstring(a, path); - defer (gb_free(a, c_path)); - - DIR *dir = opendir(c_path); - if (!dir) { - switch (errno) { - case ENOENT: - return ReadDirectory_NotExists; - case EACCES: - return ReadDirectory_Permission; - case ENOTDIR: - return ReadDirectory_NotDir; - default: - // ENOMEM: out of memory - // EMFILE: per-process limit on open fds reached - // ENFILE: system-wide limit on total open files reached - return ReadDirectory_Unknown; - } - GB_PANIC("unreachable"); - } - - array_init(fi, a, 0, 100); - - for (;;) { - struct dirent *entry = readdir(dir); - if (entry == nullptr) { - break; - } - - String name = make_string_c(entry->d_name); - if (name == "." || name == "..") { - continue; - } - - String filepath = {}; - filepath.len = path.len+1+name.len; - filepath.text = gb_alloc_array(a, u8, filepath.len+1); - defer (gb_free(a, filepath.text)); - gb_memmove(filepath.text, path.text, path.len); - gb_memmove(filepath.text+path.len, "/", 1); - gb_memmove(filepath.text+path.len+1, name.text, name.len); - filepath.text[filepath.len] = 0; - - - struct stat dir_stat = {}; - - if (stat((char *)filepath.text, &dir_stat)) { - continue; - } - - if (S_ISDIR(dir_stat.st_mode)) { - continue; - } - - i64 size = dir_stat.st_size; - - FileInfo info = {}; - info.name = name; - info.fullpath = path_to_full_path(a, filepath); - info.size = size; - array_add(fi, info); - } - - if (fi->count == 0) { - return ReadDirectory_Empty; - } - - return ReadDirectory_None; -} -#else -#error Implement read_directory -#endif - - +#include "path.cpp" struct LoadedFile { void *handle; diff --git a/src/gb/gb.h b/src/gb/gb.h index b72a893f7..3b2d6434c 100644 --- a/src/gb/gb.h +++ b/src/gb/gb.h @@ -6273,20 +6273,44 @@ char *gb_path_get_full_name(gbAllocator a, char const *path) { #else char *p, *result, *fullpath = NULL; isize len; - p = realpath(path, NULL); - fullpath = p; - if (p == NULL) { - // NOTE(bill): File does not exist - fullpath = cast(char *)path; + fullpath = realpath(path, NULL); + + if (fullpath == NULL) { + // NOTE(Jeroen): Path doesn't exist. + if (gb_strlen(path) > 0 && path[0] == '/') { + // But it is an absolute path, so return as-is. + + fullpath = (char *)path; + len = gb_strlen(fullpath) + 1; + result = gb_alloc_array(a, char, len + 1); + + gb_memmove(result, fullpath, len); + result[len] = 0; + + } else { + // Appears to be a relative path, so construct an absolute one relative to . + char cwd[4096]; + getcwd(&cwd[0], 4096); + + isize path_len = gb_strlen(path); + isize cwd_len = gb_strlen(cwd); + len = cwd_len + 1 + path_len + 1; + result = gb_alloc_array(a, char, len); + + gb_memmove(result, (void *)cwd, cwd_len); + result[cwd_len] = '/'; + + gb_memmove(result + cwd_len + 1, (void *)path, gb_strlen(path)); + result[len] = 0; + + } + } else { + len = gb_strlen(fullpath) + 1; + result = gb_alloc_array(a, char, len + 1); + gb_memmove(result, fullpath, len); + result[len] = 0; + free(fullpath); } - - len = gb_strlen(fullpath); - - result = gb_alloc_array(a, char, len + 1); - gb_memmove(result, fullpath, len); - result[len] = 0; - free(p); - return result; #endif } diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index f5cb84785..7781997f7 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -967,7 +967,12 @@ lbProcedure *lb_create_main_procedure(lbModule *m, lbProcedure *startup_runtime) } String lb_filepath_ll_for_module(lbModule *m) { - String path = m->gen->output_base; + String path = concatenate3_strings(permanent_allocator(), + build_context.build_paths[BuildPath_Output].basename, + STR_LIT("/"), + build_context.build_paths[BuildPath_Output].name + ); + if (m->pkg) { path = concatenate3_strings(permanent_allocator(), path, STR_LIT("-"), m->pkg->name); } else if (USE_SEPARATE_MODULES) { @@ -978,7 +983,12 @@ String lb_filepath_ll_for_module(lbModule *m) { return path; } String lb_filepath_obj_for_module(lbModule *m) { - String path = m->gen->output_base; + String path = concatenate3_strings(permanent_allocator(), + build_context.build_paths[BuildPath_Output].basename, + STR_LIT("/"), + build_context.build_paths[BuildPath_Output].name + ); + if (m->pkg) { path = concatenate3_strings(permanent_allocator(), path, STR_LIT("-"), m->pkg->name); } diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 330059622..1a431a4ac 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -87,7 +87,6 @@ bool lb_init_generator(lbGenerator *gen, Checker *c) { return false; } - String init_fullpath = c->parser->init_fullpath; if (build_context.out_filepath.len == 0) { diff --git a/src/main.cpp b/src/main.cpp index fc8792ceb..7b0364149 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -46,7 +46,6 @@ gb_global Timings global_timings = {0}; #include "checker.cpp" #include "docs.cpp" - #include "llvm_backend.cpp" #if defined(GB_SYSTEM_OSX) @@ -57,16 +56,8 @@ gb_global Timings global_timings = {0}; #endif #include "query_data.cpp" - - -#if defined(GB_SYSTEM_WINDOWS) -// NOTE(IC): In order to find Visual C++ paths without relying on environment variables. -#include "microsoft_craziness.h" -#endif - #include "bug_report.cpp" - // NOTE(bill): 'name' is used in debugging and profiling modes i32 system_exec_command_line_app(char const *name, char const *fmt, ...) { isize const cmd_cap = 64<<20; // 64 MiB should be more than enough @@ -130,34 +121,35 @@ i32 system_exec_command_line_app(char const *name, char const *fmt, ...) { } - - i32 linker_stage(lbGenerator *gen) { i32 result = 0; Timings *timings = &global_timings; - String output_base = gen->output_base; + String output_filename = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_Output]); + debugf("Linking %.*s\n", LIT(output_filename)); + + // TOOD(Jeroen): Make a `build_paths[BuildPath_Object] to avoid `%.*s.o`. if (is_arch_wasm()) { timings_start_section(timings, str_lit("wasm-ld")); #if defined(GB_SYSTEM_WINDOWS) result = system_exec_command_line_app("wasm-ld", - "\"%.*s\\bin\\wasm-ld\" \"%.*s.wasm.o\" -o \"%.*s.wasm\" %.*s %.*s", + "\"%.*s\\bin\\wasm-ld\" \"%.*s.o\" -o \"%.*s\" %.*s %.*s", LIT(build_context.ODIN_ROOT), - LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); + LIT(output_filename), LIT(output_filename), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); #else result = system_exec_command_line_app("wasm-ld", - "wasm-ld \"%.*s.wasm.o\" -o \"%.*s.wasm\" %.*s %.*s", - LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); + "wasm-ld \"%.*s.o\" -o \"%.*s\" %.*s %.*s", + LIT(output_filename), LIT(output_filename), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); #endif return result; } if (build_context.cross_compiling && selected_target_metrics->metrics == &target_essence_amd64) { -#ifdef GB_SYSTEM_UNIX +#if defined(GB_SYSTEM_UNIX) result = system_exec_command_line_app("linker", "x86_64-essence-gcc \"%.*s.o\" -o \"%.*s\" %.*s %.*s", - LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); + LIT(output_filename), LIT(output_filename), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); #else gb_printf_err("Linking for cross compilation for this platform is not yet supported (%.*s %.*s)\n", LIT(target_os_names[build_context.metrics.os]), @@ -181,28 +173,11 @@ i32 linker_stage(lbGenerator *gen) { gbString lib_str = gb_string_make(heap_allocator(), ""); defer (gb_string_free(lib_str)); - char const *output_ext = "exe"; gbString link_settings = gb_string_make_reserve(heap_allocator(), 256); defer (gb_string_free(link_settings)); - - // NOTE(ic): It would be nice to extend this so that we could specify the Visual Studio version that we want instead of defaulting to the latest. - Find_Result_Utf8 find_result = find_visual_studio_and_windows_sdk_utf8(); - - if (find_result.windows_sdk_version == 0) { - gb_printf_err("Windows SDK not found.\n"); - exit(1); - } - - if (build_context.ignore_microsoft_magic) { - find_result = {}; - } - // Add library search paths. - if (find_result.vs_library_path.len > 0) { - GB_ASSERT(find_result.windows_sdk_um_library_path.len > 0); - GB_ASSERT(find_result.windows_sdk_ucrt_library_path.len > 0); - + if (build_context.build_paths[BuildPath_VS_LIB].basename.len > 0) { String path = {}; auto add_path = [&](String path) { if (path[path.len-1] == '\\') { @@ -210,9 +185,9 @@ i32 linker_stage(lbGenerator *gen) { } link_settings = gb_string_append_fmt(link_settings, " /LIBPATH:\"%.*s\"", LIT(path)); }; - add_path(find_result.windows_sdk_um_library_path); - add_path(find_result.windows_sdk_ucrt_library_path); - add_path(find_result.vs_library_path); + add_path(build_context.build_paths[BuildPath_Win_SDK_UM_Lib].basename); + add_path(build_context.build_paths[BuildPath_Win_SDK_UCRT_Lib].basename); + add_path(build_context.build_paths[BuildPath_VS_LIB].basename); } @@ -252,14 +227,14 @@ i32 linker_stage(lbGenerator *gen) { if (build_context.build_mode == BuildMode_DynamicLibrary) { - output_ext = "dll"; link_settings = gb_string_append_fmt(link_settings, " /DLL"); } else { link_settings = gb_string_append_fmt(link_settings, " /ENTRY:mainCRTStartup"); } if (build_context.pdb_filepath != "") { - link_settings = gb_string_append_fmt(link_settings, " /PDB:%.*s", LIT(build_context.pdb_filepath)); + String pdb_path = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_PDB]); + link_settings = gb_string_append_fmt(link_settings, " /PDB:%.*s", LIT(pdb_path)); } if (build_context.no_crt) { @@ -300,13 +275,21 @@ i32 linker_stage(lbGenerator *gen) { object_files = gb_string_append_fmt(object_files, "\"%.*s\" ", LIT(object_path)); } + String vs_exe_path = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_VS_EXE]); + defer (gb_free(heap_allocator(), vs_exe_path.text)); + char const *subsystem_str = build_context.use_subsystem_windows ? "WINDOWS" : "CONSOLE"; if (!build_context.use_lld) { // msvc if (build_context.has_resource) { + String rc_path = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_RC]); + String res_path = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_RES]); + defer (gb_free(heap_allocator(), rc_path.text)); + defer (gb_free(heap_allocator(), res_path.text)); + result = system_exec_command_line_app("msvc-link", - "\"rc.exe\" /nologo /fo \"%.*s.res\" \"%.*s.rc\"", - LIT(output_base), - LIT(build_context.resource_filepath) + "\"rc.exe\" /nologo /fo \"%.*s\" \"%.*s\"", + LIT(res_path), + LIT(rc_path) ); if (result) { @@ -314,13 +297,13 @@ i32 linker_stage(lbGenerator *gen) { } result = system_exec_command_line_app("msvc-link", - "\"%.*slink.exe\" %s \"%.*s.res\" -OUT:\"%.*s.%s\" %s " + "\"%.*slink.exe\" %s \"%.*s\" -OUT:\"%.*s\" %s " "/nologo /incremental:no /opt:ref /subsystem:%s " " %.*s " " %.*s " " %s " "", - LIT(find_result.vs_exe_path), object_files, LIT(output_base), LIT(output_base), output_ext, + LIT(vs_exe_path), object_files, LIT(res_path), LIT(output_filename), link_settings, subsystem_str, LIT(build_context.link_flags), @@ -329,13 +312,13 @@ i32 linker_stage(lbGenerator *gen) { ); } else { result = system_exec_command_line_app("msvc-link", - "\"%.*slink.exe\" %s -OUT:\"%.*s.%s\" %s " + "\"%.*slink.exe\" %s -OUT:\"%.*s\" %s " "/nologo /incremental:no /opt:ref /subsystem:%s " " %.*s " " %.*s " " %s " "", - LIT(find_result.vs_exe_path), object_files, LIT(output_base), output_ext, + LIT(vs_exe_path), object_files, LIT(output_filename), link_settings, subsystem_str, LIT(build_context.link_flags), @@ -350,13 +333,13 @@ i32 linker_stage(lbGenerator *gen) { } else { // lld result = system_exec_command_line_app("msvc-lld-link", - "\"%.*s\\bin\\lld-link\" %s -OUT:\"%.*s.%s\" %s " + "\"%.*s\\bin\\lld-link\" %s -OUT:\"%.*s\" %s " "/nologo /incremental:no /opt:ref /subsystem:%s " " %.*s " " %.*s " " %s " "", - LIT(build_context.ODIN_ROOT), object_files, LIT(output_base),output_ext, + LIT(build_context.ODIN_ROOT), object_files, LIT(output_filename), link_settings, subsystem_str, LIT(build_context.link_flags), @@ -415,7 +398,7 @@ i32 linker_stage(lbGenerator *gen) { } else if (string_ends_with(lib, str_lit(".so"))) { // dynamic lib, relative path to executable // NOTE(vassvik): it is the user's responsibility to make sure the shared library files are visible - // at runtimeto the executable + // at runtime to the executable lib_str = gb_string_append_fmt(lib_str, " -l:\"%s/%.*s\" ", cwd, LIT(lib)); } else { // dynamic or static system lib, just link regularly searching system library paths @@ -431,9 +414,6 @@ i32 linker_stage(lbGenerator *gen) { object_files = gb_string_append_fmt(object_files, "\"%.*s\" ", LIT(object_path)); } - // Unlike the Win32 linker code, the output_ext includes the dot, because - // typically executable files on *NIX systems don't have extensions. - String output_ext = {}; gbString link_settings = gb_string_make_reserve(heap_allocator(), 32); if (build_context.no_crt) { @@ -461,26 +441,12 @@ i32 linker_stage(lbGenerator *gen) { // correctly this way since all the other dependencies provided implicitly // by the compiler frontend are still needed and most of the command // line arguments prepared previously are incompatible with ld. - // - // Shared libraries are .dylib on MacOS and .so on Linux. - if (build_context.metrics.os == TargetOs_darwin) { - output_ext = STR_LIT(".dylib"); - } else { - output_ext = STR_LIT(".so"); - } link_settings = gb_string_appendc(link_settings, "-Wl,-init,'_odin_entry_point' "); link_settings = gb_string_appendc(link_settings, "-Wl,-fini,'_odin_exit_point' "); } else if (build_context.metrics.os != TargetOs_openbsd) { // OpenBSD defaults to PIE executable. do not pass -no-pie for it. link_settings = gb_string_appendc(link_settings, "-no-pie "); } - if (build_context.out_filepath.len > 0) { - //NOTE(thebirk): We have a custom -out arguments, so we should use the extension from that - isize pos = string_extension_position(build_context.out_filepath); - if (pos > 0) { - output_ext = substring(build_context.out_filepath, pos, build_context.out_filepath.len); - } - } gbString platform_lib_str = gb_string_make(heap_allocator(), ""); defer (gb_string_free(platform_lib_str)); @@ -507,7 +473,7 @@ i32 linker_stage(lbGenerator *gen) { defer (gb_string_free(link_command_line)); link_command_line = gb_string_appendc(link_command_line, object_files); - link_command_line = gb_string_append_fmt(link_command_line, " -o \"%.*s%.*s\" ", LIT(output_base), LIT(output_ext)); + link_command_line = gb_string_append_fmt(link_command_line, " -o \"%.*s\" ", LIT(output_filename)); link_command_line = gb_string_append_fmt(link_command_line, " %s ", platform_lib_str); link_command_line = gb_string_append_fmt(link_command_line, " %s ", lib_str); link_command_line = gb_string_append_fmt(link_command_line, " %.*s ", LIT(build_context.link_flags)); @@ -524,9 +490,7 @@ i32 linker_stage(lbGenerator *gen) { if (build_context.ODIN_DEBUG) { // NOTE: macOS links DWARF symbols dynamically. Dsymutil will map the stubs in the exe // to the symbols in the object file - result = system_exec_command_line_app("dsymutil", - "dsymutil %.*s%.*s", LIT(output_base), LIT(output_ext) - ); + result = system_exec_command_line_app("dsymutil", "dsymutil %.*s", LIT(output_filename)); if (result) { return result; @@ -1526,6 +1490,10 @@ bool parse_build_flags(Array args) { gb_printf_err("Invalid -resource path %.*s, missing .rc\n", LIT(path)); bad_flags = true; break; + } else if (!gb_file_exists((const char *)path.text)) { + gb_printf_err("Invalid -resource path %.*s, file does not exist.\n", LIT(path)); + bad_flags = true; + break; } build_context.resource_filepath = substring(path, 0, string_extension_position(path)); build_context.has_resource = true; @@ -1540,6 +1508,11 @@ bool parse_build_flags(Array args) { String path = value.value_string; path = string_trim_whitespace(path); if (is_build_flag_path_valid(path)) { + if (path_is_directory(path)) { + gb_printf_err("Invalid -pdb-name path. %.*s, is a directory.\n", LIT(path)); + bad_flags = true; + break; + } // #if defined(GB_SYSTEM_WINDOWS) // String ext = path_extension(path); // if (ext != ".pdb") { @@ -2666,6 +2639,8 @@ int main(int arg_count, char const **arg_ptr) { return 1; } + init_filename = copy_string(permanent_allocator(), init_filename); + if (init_filename == "-help" || init_filename == "--help") { build_context.show_help = true; @@ -2688,6 +2663,12 @@ int main(int arg_count, char const **arg_ptr) { gb_printf_err("Did you mean `%.*s %.*s %.*s -file`?\n", LIT(args[0]), LIT(command), LIT(init_filename)); gb_printf_err("The `-file` flag tells it to treat a file as a self-contained package.\n"); return 1; + } else { + String const ext = str_lit(".odin"); + if (!string_ends_with(init_filename, ext)) { + gb_printf_err("Expected either a directory or a .odin file, got '%.*s'\n", LIT(init_filename)); + return 1; + } } } } @@ -2709,13 +2690,24 @@ int main(int arg_count, char const **arg_ptr) { get_fullpath_relative(heap_allocator(), odin_root_dir(), str_lit("shared"))); } - init_build_context(selected_target_metrics ? selected_target_metrics->metrics : nullptr); // if (build_context.word_size == 4 && build_context.metrics.os != TargetOs_js) { // print_usage_line(0, "%.*s 32-bit is not yet supported for this platform", LIT(args[0])); // return 1; // } + // Set and check build paths... + if (!init_build_paths(init_filename)) { + return 1; + } + + if (build_context.show_debug_messages) { + for_array(i, build_context.build_paths) { + String build_path = path_to_string(heap_allocator(), build_context.build_paths[i]); + debugf("build_paths[%ld]: %.*s\n", i, LIT(build_path)); + } + } + init_global_thread_pool(); defer (thread_pool_destroy(&global_thread_pool)); @@ -2732,6 +2724,8 @@ int main(int arg_count, char const **arg_ptr) { } defer (destroy_parser(parser)); + // TODO(jeroen): Remove the `init_filename` param. + // Let's put that on `build_context.build_paths[0]` instead. if (parse_packages(parser, init_filename) != ParseFile_None) { return 1; } @@ -2810,16 +2804,14 @@ int main(int arg_count, char const **arg_ptr) { } if (run_output) { + String exe_name = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_Output]); + defer (gb_free(heap_allocator(), exe_name.text)); + #if defined(GB_SYSTEM_WINDOWS) - return system_exec_command_line_app("odin run", "%.*s.exe %.*s", LIT(gen->output_base), LIT(run_args_string)); + return system_exec_command_line_app("odin run", "%.*s %.*s", LIT(exe_name), LIT(run_args_string)); #else - //NOTE(thebirk): This whole thing is a little leaky - String output_ext = {}; - String complete_path = concatenate_strings(permanent_allocator(), gen->output_base, output_ext); - complete_path = path_to_full_path(permanent_allocator(), complete_path); - return system_exec_command_line_app("odin run", "\"%.*s\" %.*s", LIT(complete_path), LIT(run_args_string)); + return system_exec_command_line_app("odin run", "\"%.*s\" %.*s", LIT(exe_name), LIT(run_args_string)); #endif } - return 0; } diff --git a/src/parser.cpp b/src/parser.cpp index 767119aa8..df7f908a6 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -5751,7 +5751,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } } } - + { // Add these packages serially and then process them parallel mutex_lock(&p->wait_mutex); diff --git a/src/path.cpp b/src/path.cpp new file mode 100644 index 000000000..8d8e532b8 --- /dev/null +++ b/src/path.cpp @@ -0,0 +1,333 @@ +/* + Path handling utilities. +*/ + +#if defined(GB_SYSTEM_WINDOWS) + bool path_is_directory(String path) { + gbAllocator a = heap_allocator(); + String16 wstr = string_to_string16(a, path); + defer (gb_free(a, wstr.text)); + + i32 attribs = GetFileAttributesW(wstr.text); + if (attribs < 0) return false; + + return (attribs & FILE_ATTRIBUTE_DIRECTORY) != 0; + } + +#else + bool path_is_directory(String path) { + gbAllocator a = heap_allocator(); + char *copy = cast(char *)copy_string(a, path).text; + defer (gb_free(a, copy)); + + struct stat s; + if (stat(copy, &s) == 0) { + return (s.st_mode & S_IFDIR) != 0; + } + return false; + } +#endif + + +String path_to_full_path(gbAllocator a, String path) { + gbAllocator ha = heap_allocator(); + char *path_c = gb_alloc_str_len(ha, cast(char *)path.text, path.len); + defer (gb_free(ha, path_c)); + + char *fullpath = gb_path_get_full_name(a, path_c); + String res = string_trim_whitespace(make_string_c(fullpath)); +#if defined(GB_SYSTEM_WINDOWS) + for (isize i = 0; i < res.len; i++) { + if (res.text[i] == '\\') { + res.text[i] = '/'; + } + } +#endif + return copy_string(a, res); +} + +struct Path { + String basename; + String name; + String ext; +}; + +// NOTE(Jeroen): Naively turns a Path into a string. +String path_to_string(gbAllocator a, Path path) { + if (path.basename.len + path.name.len + path.ext.len == 0) { + return make_string(nullptr, 0); + } + + isize len = path.basename.len + 1 + path.name.len + 1; + if (path.ext.len > 0) { + len += path.ext.len + 1; + } + + u8 *str = gb_alloc_array(a, u8, len); + + isize i = 0; + gb_memmove(str+i, path.basename.text, path.basename.len); i += path.basename.len; + gb_memmove(str+i, "/", 1); i += 1; + gb_memmove(str+i, path.name.text, path.name.len); i += path.name.len; + if (path.ext.len > 0) { + gb_memmove(str+i, ".", 1); i += 1; + gb_memmove(str+i, path.ext.text, path.ext.len); i += path.ext.len; + } + str[i] = 0; + + String res = make_string(str, i); + res = string_trim_whitespace(res); + return res; +} + +// NOTE(Jeroen): Naively turns a Path into a string, then normalizes it using `path_to_full_path`. +String path_to_full_path(gbAllocator a, Path path) { + String temp = path_to_string(heap_allocator(), path); + defer (gb_free(heap_allocator(), temp.text)); + + return path_to_full_path(a, temp); +} + +// NOTE(Jeroen): Takes a path like "odin" or "W:\Odin", turns it into a full path, +// and then breaks it into its components to make a Path. +Path path_from_string(gbAllocator a, String const &path) { + Path res = {}; + + if (path.len == 0) return res; + + String fullpath = path_to_full_path(a, path); + defer (gb_free(heap_allocator(), fullpath.text)); + + res.basename = directory_from_path(fullpath); + res.basename = copy_string(a, res.basename); + + if (string_ends_with(fullpath, '/')) { + // It's a directory. We don't need to tinker with the name and extension. + return res; + } + + isize name_start = (res.basename.len > 0) ? res.basename.len + 1 : res.basename.len; + res.name = substring(fullpath, name_start, fullpath.len); + res.name = remove_extension_from_path(res.name); + res.name = copy_string(a, res.name); + + res.ext = path_extension(fullpath, false); // false says not to include the dot. + res.ext = copy_string(a, res.ext); + return res; +} + +bool path_is_directory(Path path) { + String path_string = path_to_full_path(heap_allocator(), path); + defer (gb_free(heap_allocator(), path_string.text)); + + return path_is_directory(path_string); +} + +struct FileInfo { + String name; + String fullpath; + i64 size; + bool is_dir; +}; + +enum ReadDirectoryError { + ReadDirectory_None, + + ReadDirectory_InvalidPath, + ReadDirectory_NotExists, + ReadDirectory_Permission, + ReadDirectory_NotDir, + ReadDirectory_Empty, + ReadDirectory_Unknown, + + ReadDirectory_COUNT, +}; + +i64 get_file_size(String path) { + char *c_str = alloc_cstring(heap_allocator(), path); + defer (gb_free(heap_allocator(), c_str)); + + gbFile f = {}; + gbFileError err = gb_file_open(&f, c_str); + defer (gb_file_close(&f)); + if (err != gbFileError_None) { + return -1; + } + return gb_file_size(&f); +} + + +#if defined(GB_SYSTEM_WINDOWS) +ReadDirectoryError read_directory(String path, Array *fi) { + GB_ASSERT(fi != nullptr); + + gbAllocator a = heap_allocator(); + + while (path.len > 0) { + Rune end = path[path.len-1]; + if (end == '/') { + path.len -= 1; + } else if (end == '\\') { + path.len -= 1; + } else { + break; + } + } + + if (path.len == 0) { + return ReadDirectory_InvalidPath; + } + { + char *c_str = alloc_cstring(a, path); + defer (gb_free(a, c_str)); + + gbFile f = {}; + gbFileError file_err = gb_file_open(&f, c_str); + defer (gb_file_close(&f)); + + switch (file_err) { + case gbFileError_Invalid: return ReadDirectory_InvalidPath; + case gbFileError_NotExists: return ReadDirectory_NotExists; + // case gbFileError_Permission: return ReadDirectory_Permission; + } + } + + if (!path_is_directory(path)) { + return ReadDirectory_NotDir; + } + + + char *new_path = gb_alloc_array(a, char, path.len+3); + defer (gb_free(a, new_path)); + + gb_memmove(new_path, path.text, path.len); + gb_memmove(new_path+path.len, "/*", 2); + new_path[path.len+2] = 0; + + String np = make_string(cast(u8 *)new_path, path.len+2); + String16 wstr = string_to_string16(a, np); + defer (gb_free(a, wstr.text)); + + WIN32_FIND_DATAW file_data = {}; + HANDLE find_file = FindFirstFileW(wstr.text, &file_data); + if (find_file == INVALID_HANDLE_VALUE) { + return ReadDirectory_Unknown; + } + defer (FindClose(find_file)); + + array_init(fi, a, 0, 100); + + do { + wchar_t *filename_w = file_data.cFileName; + i64 size = cast(i64)file_data.nFileSizeLow; + size |= (cast(i64)file_data.nFileSizeHigh) << 32; + String name = string16_to_string(a, make_string16_c(filename_w)); + if (name == "." || name == "..") { + gb_free(a, name.text); + continue; + } + + String filepath = {}; + filepath.len = path.len+1+name.len; + filepath.text = gb_alloc_array(a, u8, filepath.len+1); + defer (gb_free(a, filepath.text)); + gb_memmove(filepath.text, path.text, path.len); + gb_memmove(filepath.text+path.len, "/", 1); + gb_memmove(filepath.text+path.len+1, name.text, name.len); + + FileInfo info = {}; + info.name = name; + info.fullpath = path_to_full_path(a, filepath); + info.size = size; + info.is_dir = (file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + array_add(fi, info); + } while (FindNextFileW(find_file, &file_data)); + + if (fi->count == 0) { + return ReadDirectory_Empty; + } + + return ReadDirectory_None; +} +#elif defined(GB_SYSTEM_LINUX) || defined(GB_SYSTEM_OSX) || defined(GB_SYSTEM_FREEBSD) || defined(GB_SYSTEM_OPENBSD) + +#include + +ReadDirectoryError read_directory(String path, Array *fi) { + GB_ASSERT(fi != nullptr); + + gbAllocator a = heap_allocator(); + + char *c_path = alloc_cstring(a, path); + defer (gb_free(a, c_path)); + + DIR *dir = opendir(c_path); + if (!dir) { + switch (errno) { + case ENOENT: + return ReadDirectory_NotExists; + case EACCES: + return ReadDirectory_Permission; + case ENOTDIR: + return ReadDirectory_NotDir; + default: + // ENOMEM: out of memory + // EMFILE: per-process limit on open fds reached + // ENFILE: system-wide limit on total open files reached + return ReadDirectory_Unknown; + } + GB_PANIC("unreachable"); + } + + array_init(fi, a, 0, 100); + + for (;;) { + struct dirent *entry = readdir(dir); + if (entry == nullptr) { + break; + } + + String name = make_string_c(entry->d_name); + if (name == "." || name == "..") { + continue; + } + + String filepath = {}; + filepath.len = path.len+1+name.len; + filepath.text = gb_alloc_array(a, u8, filepath.len+1); + defer (gb_free(a, filepath.text)); + gb_memmove(filepath.text, path.text, path.len); + gb_memmove(filepath.text+path.len, "/", 1); + gb_memmove(filepath.text+path.len+1, name.text, name.len); + filepath.text[filepath.len] = 0; + + + struct stat dir_stat = {}; + + if (stat((char *)filepath.text, &dir_stat)) { + continue; + } + + if (S_ISDIR(dir_stat.st_mode)) { + continue; + } + + i64 size = dir_stat.st_size; + + FileInfo info = {}; + info.name = name; + info.fullpath = path_to_full_path(a, filepath); + info.size = size; + array_add(fi, info); + } + + if (fi->count == 0) { + return ReadDirectory_Empty; + } + + return ReadDirectory_None; +} +#else +#error Implement read_directory +#endif + diff --git a/src/string.cpp b/src/string.cpp index d3dbc6904..3515df48e 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -245,15 +245,14 @@ gb_inline isize string_extension_position(String const &str) { return dot_pos; } -String path_extension(String const &str) { +String path_extension(String const &str, bool include_dot = true) { isize pos = string_extension_position(str); if (pos < 0) { return make_string(nullptr, 0); } - return substring(str, pos, str.len); + return substring(str, include_dot ? pos : pos + 1, str.len); } - String string_trim_whitespace(String str) { while (str.len > 0 && rune_is_whitespace(str[str.len-1])) { str.len--; @@ -328,7 +327,10 @@ String directory_from_path(String const &s) { break; } } - return substring(s, 0, i); + if (i >= 0) { + return substring(s, 0, i); + } + return substring(s, 0, 0); } String concatenate_strings(gbAllocator a, String const &x, String const &y) { diff --git a/tests/core/build.bat b/tests/core/build.bat index 2f9ba672e..1973c22aa 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -5,61 +5,61 @@ python3 download_assets.py echo --- echo Running core:image tests echo --- -%PATH_TO_ODIN% run image %COMMON% +%PATH_TO_ODIN% run image %COMMON% -out:test_image echo --- echo Running core:compress tests echo --- -%PATH_TO_ODIN% run compress %COMMON% +%PATH_TO_ODIN% run compress %COMMON% -out:test_compress echo --- echo Running core:strings tests echo --- -%PATH_TO_ODIN% run strings %COMMON% +%PATH_TO_ODIN% run strings %COMMON% -out:test_strings echo --- echo Running core:hash tests echo --- -%PATH_TO_ODIN% run hash %COMMON% -o:size +%PATH_TO_ODIN% run hash %COMMON% -o:size -out:test_hash echo --- echo Running core:odin tests echo --- -%PATH_TO_ODIN% run odin %COMMON% -o:size +%PATH_TO_ODIN% run odin %COMMON% -o:size -out:test_odin echo --- echo Running core:crypto hash tests echo --- -%PATH_TO_ODIN% run crypto %COMMON% +%PATH_TO_ODIN% run crypto %COMMON% -out:test_crypto echo --- echo Running core:encoding tests echo --- -%PATH_TO_ODIN% run encoding/hxa %COMMON% -%PATH_TO_ODIN% run encoding/json %COMMON% -%PATH_TO_ODIN% run encoding/varint %COMMON% +%PATH_TO_ODIN% run encoding/hxa %COMMON% -out:test_hxa +%PATH_TO_ODIN% run encoding/json %COMMON% -out:test_json +%PATH_TO_ODIN% run encoding/varint %COMMON% -out:test_varint echo --- echo Running core:math/noise tests echo --- -%PATH_TO_ODIN% run math/noise %COMMON% +%PATH_TO_ODIN% run math/noise %COMMON% -out:test_noise echo --- echo Running core:math tests echo --- -%PATH_TO_ODIN% run math %COMMON% +%PATH_TO_ODIN% run math %COMMON% -out:test_math echo --- echo Running core:math/linalg/glsl tests echo --- -%PATH_TO_ODIN% run math/linalg/glsl %COMMON% +%PATH_TO_ODIN% run math/linalg/glsl %COMMON% -out:test_glsl echo --- echo Running core:path/filepath tests echo --- -%PATH_TO_ODIN% run path/filepath %COMMON% +%PATH_TO_ODIN% run path/filepath %COMMON% -out:test_filepath echo --- echo Running core:reflect tests echo --- -%PATH_TO_ODIN% run reflect %COMMON% +%PATH_TO_ODIN% run reflect %COMMON% -out:test_reflect diff --git a/tests/core/math/big/build.bat b/tests/core/math/big/build.bat index 16bdbc8ca..ad199d775 100644 --- a/tests/core/math/big/build.bat +++ b/tests/core/math/big/build.bat @@ -4,7 +4,7 @@ set PATH_TO_ODIN==..\..\..\..\odin set TEST_ARGS=-fast-tests set TEST_ARGS=-no-random set TEST_ARGS= -set OUT_NAME=math_big_test_library +set OUT_NAME=math_big_test_library.dll set COMMON=-build-mode:shared -show-timings -no-bounds-check -define:MATH_BIG_EXE=false -vet -strict-style echo --- echo Running core:math/big tests diff --git a/tests/issues/run.sh b/tests/issues/run.sh index 117a9a5f1..91ec99e05 100755 --- a/tests/issues/run.sh +++ b/tests/issues/run.sh @@ -8,10 +8,10 @@ COMMON="-collection:tests=tests -out:tests/issues/build/test_issue" set -x ./odin build tests/issues/test_issue_829.odin $COMMON -file -tests/issues/build/test_issue +tests/issues/build/test_issue.bin ./odin build tests/issues/test_issue_1592.odin $COMMON -file -tests/issues/build/test_issue +tests/issues/build/test_issue.bin set +x From 76d48b38d394b953ea4bbe1420ecd11e6e7dd028 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 8 Apr 2022 19:02:14 +0200 Subject: [PATCH 34/79] Compiler: Allow -out: to not have an extension on *nix for executables (only). --- src/build_settings.cpp | 4 +++- tests/issues/run.sh | 4 ++-- tests/vendor/Makefile | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 0b582eac8..55d129124 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -1308,7 +1308,9 @@ bool init_build_paths(String init_filename) { // Do we have an extension? We might not if the output filename was supplied. if (bc->build_paths[BuildPath_Output].ext.len == 0) { - bc->build_paths[BuildPath_Output].ext = copy_string(ha, output_extension); + if (build_context.metrics.os == TargetOs_windows || build_context.build_mode != BuildMode_Executable) { + bc->build_paths[BuildPath_Output].ext = copy_string(ha, output_extension); + } } // Check if output path is a directory. diff --git a/tests/issues/run.sh b/tests/issues/run.sh index 91ec99e05..117a9a5f1 100755 --- a/tests/issues/run.sh +++ b/tests/issues/run.sh @@ -8,10 +8,10 @@ COMMON="-collection:tests=tests -out:tests/issues/build/test_issue" set -x ./odin build tests/issues/test_issue_829.odin $COMMON -file -tests/issues/build/test_issue.bin +tests/issues/build/test_issue ./odin build tests/issues/test_issue_1592.odin $COMMON -file -tests/issues/build/test_issue.bin +tests/issues/build/test_issue set +x diff --git a/tests/vendor/Makefile b/tests/vendor/Makefile index 341067c6e..c508f6c50 100644 --- a/tests/vendor/Makefile +++ b/tests/vendor/Makefile @@ -10,4 +10,4 @@ endif all: botan_test botan_test: - $(ODIN) run botan -out=botan_hash -o:speed -no-bounds-check $(ODINFLAGS) + $(ODIN) run botan -out=test_botan_hash -o:speed -no-bounds-check $(ODINFLAGS) From f4723aea4cb610a8ccc7d3614f8787d638d284d6 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 8 Apr 2022 19:14:59 +0200 Subject: [PATCH 35/79] Remove redundant bit for non-Windows. --- src/build_settings.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 55d129124..212ded5c8 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -1281,22 +1281,8 @@ bool init_build_paths(String init_filename) { output_name = remove_extension_from_path(output_name); output_name = copy_string(ha, string_trim_whitespace(output_name)); - /* - NOTE(Jeroen): This fallback substitution can't be made at this stage. - if (gen->output_name.len == 0) { - gen->output_name = c->info.init_scope->pkg->name; - } - */ Path output_path = path_from_string(ha, output_name); - #ifndef GB_SYSTEM_WINDOWS - char cwd[4096]; - getcwd(&cwd[0], 4096); - - const u8 * cwd_str = (const u8 *)&cwd[0]; - output_path.basename = copy_string(ha, make_string(cwd_str, strlen(cwd))); - #endif - // Replace extension. if (output_path.ext.len > 0) { gb_free(ha, output_path.ext.text); From 3d2856db31456e9a117209eccf8e1167b4401205 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 24 Apr 2022 14:19:25 +0200 Subject: [PATCH 36/79] Update tests to use new filename generation code. --- .github/workflows/ci.yml | 11 ++++++++--- core/crypto/util/util.odin | 1 + tests/core/Makefile | 29 +++++++++++++++-------------- tests/core/build.bat | 32 +++++++++++++++++--------------- tests/issues/run.bat | 21 ++++++++------------- tests/issues/run.sh | 19 +++++++++---------- tests/vendor/Makefile | 3 ++- tests/vendor/build.bat | 3 ++- 8 files changed, 62 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cc4283b0..989f56712 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,9 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: tests/issues/run.sh + run: | + cd tests/issues + run.sh timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 @@ -91,7 +93,9 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: tests/issues/run.sh + run: | + cd tests/issues + run.sh timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 @@ -163,7 +167,8 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat - call tests\issues\run.bat + cd tests\issues + call build.bat timeout-minutes: 10 - name: Odin check examples/all for Windows 32bits shell: cmd diff --git a/core/crypto/util/util.odin b/core/crypto/util/util.odin index 6273a232e..83b07e546 100644 --- a/core/crypto/util/util.odin +++ b/core/crypto/util/util.odin @@ -11,6 +11,7 @@ package util */ import "core:mem" +_ :: mem // @note(bp): this can replace the other two cast_slice :: #force_inline proc "contextless" ($D: typeid/[]$DE, src: $S/[]$SE) -> D { diff --git a/tests/core/Makefile b/tests/core/Makefile index 6a92b4efb..9bb622633 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -1,5 +1,6 @@ ODIN=../../odin PYTHON=$(shell which python3) +OUT_FILE=test_binary.bin all: download_test_assets image_test compress_test strings_test hash_test crypto_test noise_test encoding_test \ math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test @@ -8,39 +9,39 @@ download_test_assets: $(PYTHON) download_assets.py image_test: - $(ODIN) run image/test_core_image.odin -file + $(ODIN) run image/test_core_image.odin -out=$(OUT_FILE) -file compress_test: - $(ODIN) run compress/test_core_compress.odin -file + $(ODIN) run compress/test_core_compress.odin -out=$(OUT_FILE) -file strings_test: - $(ODIN) run strings/test_core_strings.odin -file + $(ODIN) run strings/test_core_strings.odin -out=$(OUT_FILE) -file hash_test: - $(ODIN) run hash -out=test_hash -o:speed -no-bounds-check + $(ODIN) run hash -out=$(OUT_FILE) -o:speed -no-bounds-check crypto_test: - $(ODIN) run crypto -out=test_crypto_hash -o:speed -no-bounds-check + $(ODIN) run crypto -out=$(OUT_FILE) -o:speed -no-bounds-check noise_test: - $(ODIN) run math/noise -out=test_noise + $(ODIN) run math/noise -out=$(OUT_FILE) encoding_test: - $(ODIN) run encoding/hxa -out=test_hxa -collection:tests=.. - $(ODIN) run encoding/json -out=test_json - $(ODIN) run encoding/varint -out=test_varint + $(ODIN) run encoding/hxa -out=$(OUT_FILE) -collection:tests=.. + $(ODIN) run encoding/json -out=$(OUT_FILE) + $(ODIN) run encoding/varint -out=$(OUT_FILE) math_test: - $(ODIN) run math/test_core_math.odin -out=test_core_math -file -collection:tests=.. + $(ODIN) run math/test_core_math.odin -out=$(OUT_FILE) -file -collection:tests=.. linalg_glsl_math_test: - $(ODIN) run math/linalg/glsl/test_linalg_glsl_math.odin -file -out=test_linalg_glsl_math -collection:tests=.. + $(ODIN) run math/linalg/glsl/test_linalg_glsl_math.odin -file -out=$(OUT_FILE) -collection:tests=.. filepath_test: - $(ODIN) run path/filepath/test_core_filepath.odin -file -out=test_core_filepath -collection:tests=.. + $(ODIN) run path/filepath/test_core_filepath.odin -file -out=$(OUT_FILE) -collection:tests=.. reflect_test: - $(ODIN) run reflect/test_core_reflect.odin -file -out=test_core_reflect -collection:tests=.. + $(ODIN) run reflect/test_core_reflect.odin -file -out=$(OUT_FILE) -collection:tests=.. os_exit_test: - $(ODIN) run os/test_core_os_exit.odin -file -out=test_core_os_exit && exit 1 || exit 0 + $(ODIN) run os/test_core_os_exit.odin -file -out=$(OUT_FILE) && exit 1 || exit 0 diff --git a/tests/core/build.bat b/tests/core/build.bat index 1973c22aa..331a473aa 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -1,65 +1,67 @@ @echo off -set COMMON=-show-timings -no-bounds-check -vet -strict-style -collection:tests=.. +set OUT_FILE=test_binary.exe +set COMMON=-show-timings -no-bounds-check -vet -strict-style -collection:tests=.. -out:%OUT_FILE% set PATH_TO_ODIN==..\..\odin + python3 download_assets.py echo --- echo Running core:image tests echo --- -%PATH_TO_ODIN% run image %COMMON% -out:test_image +%PATH_TO_ODIN% run image %COMMON% echo --- echo Running core:compress tests echo --- -%PATH_TO_ODIN% run compress %COMMON% -out:test_compress +%PATH_TO_ODIN% run compress %COMMON% echo --- echo Running core:strings tests echo --- -%PATH_TO_ODIN% run strings %COMMON% -out:test_strings +%PATH_TO_ODIN% run strings %COMMON% echo --- echo Running core:hash tests echo --- -%PATH_TO_ODIN% run hash %COMMON% -o:size -out:test_hash +%PATH_TO_ODIN% run hash %COMMON% -o:size echo --- echo Running core:odin tests echo --- -%PATH_TO_ODIN% run odin %COMMON% -o:size -out:test_odin +%PATH_TO_ODIN% run odin %COMMON% -o:size echo --- echo Running core:crypto hash tests echo --- -%PATH_TO_ODIN% run crypto %COMMON% -out:test_crypto +%PATH_TO_ODIN% run crypto %COMMON% echo --- echo Running core:encoding tests echo --- -%PATH_TO_ODIN% run encoding/hxa %COMMON% -out:test_hxa -%PATH_TO_ODIN% run encoding/json %COMMON% -out:test_json -%PATH_TO_ODIN% run encoding/varint %COMMON% -out:test_varint +%PATH_TO_ODIN% run encoding/hxa %COMMON% +%PATH_TO_ODIN% run encoding/json %COMMON% +%PATH_TO_ODIN% run encoding/varint %COMMON% echo --- echo Running core:math/noise tests echo --- -%PATH_TO_ODIN% run math/noise %COMMON% -out:test_noise +%PATH_TO_ODIN% run math/noise %COMMON% echo --- echo Running core:math tests echo --- -%PATH_TO_ODIN% run math %COMMON% -out:test_math +%PATH_TO_ODIN% run math %COMMON% echo --- echo Running core:math/linalg/glsl tests echo --- -%PATH_TO_ODIN% run math/linalg/glsl %COMMON% -out:test_glsl +%PATH_TO_ODIN% run math/linalg/glsl %COMMON% echo --- echo Running core:path/filepath tests echo --- -%PATH_TO_ODIN% run path/filepath %COMMON% -out:test_filepath +%PATH_TO_ODIN% run path/filepath %COMMON% echo --- echo Running core:reflect tests echo --- -%PATH_TO_ODIN% run reflect %COMMON% -out:test_reflect +%PATH_TO_ODIN% run reflect %COMMON% diff --git a/tests/issues/run.bat b/tests/issues/run.bat index a652d9694..a936bd896 100644 --- a/tests/issues/run.bat +++ b/tests/issues/run.bat @@ -1,17 +1,12 @@ @echo off +set PATH_TO_ODIN==..\..\odin +set COMMON=-collection:tests=.. -out:build\test_issue +if not exist "build" mkdir build -if not exist "tests\issues\build\" mkdir tests\issues\build +%PATH_TO_ODIN% build test_issue_829.odin %COMMON% -file +build\test_issue -set COMMON=-collection:tests=tests -out:tests\issues\build\test_issue +%PATH_TO_ODIN% build test_issue_1592.odin %COMMON% -file +build\test_issue -@echo on - -.\odin build tests\issues\test_issue_829.odin %COMMON% -file -tests\issues\build\test_issue - -.\odin build tests\issues\test_issue_1592.odin %COMMON% -file -tests\issues\build\test_issue - -@echo off - -rmdir /S /Q tests\issues\build +rmdir /S /Q build diff --git a/tests/issues/run.sh b/tests/issues/run.sh index 117a9a5f1..ec6d7130d 100755 --- a/tests/issues/run.sh +++ b/tests/issues/run.sh @@ -1,18 +1,17 @@ #!/bin/bash +ODIN=../../odin +COMMON="-collection:tests=.. -out:build/test_issue.bin" + set -eu - -mkdir -p tests/issues/build - -COMMON="-collection:tests=tests -out:tests/issues/build/test_issue" - +mkdir -p build set -x -./odin build tests/issues/test_issue_829.odin $COMMON -file -tests/issues/build/test_issue +$ODIN build test_issue_829.odin $COMMON -file +build/test_issue.bin -./odin build tests/issues/test_issue_1592.odin $COMMON -file -tests/issues/build/test_issue +$ODIN build test_issue_1592.odin $COMMON -file +build/test_issue.bin set +x -rm -rf tests/issues/build +rm -rf build diff --git a/tests/vendor/Makefile b/tests/vendor/Makefile index c508f6c50..380e64e09 100644 --- a/tests/vendor/Makefile +++ b/tests/vendor/Makefile @@ -1,5 +1,6 @@ ODIN=../../odin ODINFLAGS= +OUT_FILE=test_binary.bin OS=$(shell uname) @@ -10,4 +11,4 @@ endif all: botan_test botan_test: - $(ODIN) run botan -out=test_botan_hash -o:speed -no-bounds-check $(ODINFLAGS) + $(ODIN) run botan -out=$(OUT_FILE) -o:speed -no-bounds-check $(ODINFLAGS) diff --git a/tests/vendor/build.bat b/tests/vendor/build.bat index e70d9f1d5..4bd9a6496 100644 --- a/tests/vendor/build.bat +++ b/tests/vendor/build.bat @@ -1,5 +1,6 @@ @echo off -set COMMON=-show-timings -no-bounds-check -vet -strict-style +set OUT_FILE=test_binary.exe +set COMMON=-show-timings -no-bounds-check -vet -strict-style -out:%OUT_FILE% set PATH_TO_ODIN==..\..\odin echo --- From 9f8d90f466454f4d14e755d44e4ba47ccbf0c92e Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 24 Apr 2022 14:28:00 +0200 Subject: [PATCH 37/79] Update CI paths for issue tests. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 989f56712..d72775636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,7 +41,7 @@ jobs: - name: Odin issues tests run: | cd tests/issues - run.sh + ./run.sh timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 @@ -95,7 +95,7 @@ jobs: - name: Odin issues tests run: | cd tests/issues - run.sh + ./run.sh timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 @@ -168,7 +168,7 @@ jobs: run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat cd tests\issues - call build.bat + call run.bat timeout-minutes: 10 - name: Odin check examples/all for Windows 32bits shell: cmd From 63331ef731209ec8db65d7f26bdbebdf9459107d Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Sun, 24 Apr 2022 19:53:36 +0200 Subject: [PATCH 38/79] Revert "Merge pull request #1702 from Kelimion/filename_generation" This reverts commit a40a53b10447c9223c24cccf565a95f1773d3922, reversing changes made to 5422a3b17eae821df4adf869960995e922eb0e76. --- .github/workflows/ci.yml | 11 +- .gitignore | 1 - Makefile | 4 +- build_odin.sh | 4 +- core/crypto/util/util.odin | 1 - src/build_settings.cpp | 208 +++------------------ src/common.cpp | 257 +++++++++++++++++++++++++- src/gb/gb.h | 50 ++--- src/llvm_backend.cpp | 14 +- src/llvm_backend_general.cpp | 1 + src/main.cpp | 152 ++++++++-------- src/parser.cpp | 2 +- src/path.cpp | 333 ---------------------------------- src/string.cpp | 10 +- tests/core/Makefile | 29 ++- tests/core/build.bat | 10 +- tests/core/math/big/build.bat | 2 +- tests/issues/run.bat | 21 ++- tests/issues/run.sh | 19 +- tests/vendor/Makefile | 3 +- tests/vendor/build.bat | 3 +- 21 files changed, 436 insertions(+), 699 deletions(-) delete mode 100644 src/path.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d72775636..3cc4283b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,7 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: | - cd tests/issues - ./run.sh + run: tests/issues/run.sh timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 @@ -93,9 +91,7 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: | - cd tests/issues - ./run.sh + run: tests/issues/run.sh timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 @@ -167,8 +163,7 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat - cd tests\issues - call run.bat + call tests\issues\run.bat timeout-minutes: 10 - name: Odin check examples/all for Windows 32bits shell: cmd diff --git a/.gitignore b/.gitignore index d03a86fd7..e8b3d3050 100644 --- a/.gitignore +++ b/.gitignore @@ -269,7 +269,6 @@ bin/ # - Linux/MacOS odin odin.dSYM -*.bin # shared collection shared/ diff --git a/Makefile b/Makefile index 1a1c93180..82150c6a2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -all: debug +all: debug demo demo: - ./odin run examples/demo + ./odin run examples/demo/demo.odin report: ./odin report diff --git a/build_odin.sh b/build_odin.sh index 4810cafd2..aef3f2836 100755 --- a/build_odin.sh +++ b/build_odin.sh @@ -102,7 +102,7 @@ build_odin() { } run_demo() { - ./odin run examples/demo + ./odin run examples/demo/demo.odin -file } case $OS in @@ -147,4 +147,4 @@ if [[ $# -eq 1 ]]; then exit 0 else panic "Too many arguments!" -fi \ No newline at end of file +fi diff --git a/core/crypto/util/util.odin b/core/crypto/util/util.odin index 83b07e546..6273a232e 100644 --- a/core/crypto/util/util.odin +++ b/core/crypto/util/util.odin @@ -11,7 +11,6 @@ package util */ import "core:mem" -_ :: mem // @note(bp): this can replace the other two cast_slice :: #force_inline proc "contextless" ($D: typeid/[]$DE, src: $S/[]$SE) -> D { diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 212ded5c8..2f3eb03a5 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -3,6 +3,7 @@ #include #endif + // #if defined(GB_SYSTEM_WINDOWS) // #define DEFAULT_TO_THREADED_CHECKER // #endif @@ -197,22 +198,6 @@ enum RelocMode : u8 { RelocMode_DynamicNoPIC, }; -enum BuildPath : u8 { - BuildPath_Main_Package, // Input Path to the package directory (or file) we're building. - BuildPath_RC, // Input Path for .rc file, can be set with `-resource:`. - BuildPath_RES, // Output Path for .res file, generated from previous. - BuildPath_Win_SDK_Root, // windows_sdk_root - BuildPath_Win_SDK_UM_Lib, // windows_sdk_um_library_path - BuildPath_Win_SDK_UCRT_Lib, // windows_sdk_ucrt_library_path - BuildPath_VS_EXE, // vs_exe_path - BuildPath_VS_LIB, // vs_library_path - - BuildPath_Output, // Output Path for .exe, .dll, .so, etc. Can be overridden with `-out:`. - BuildPath_PDB, // Output Path for .pdb file, can be overridden with `-pdb-name:`. - - BuildPathCOUNT, -}; - // This stores the information for the specify architecture of this build struct BuildContext { // Constants @@ -241,13 +226,9 @@ struct BuildContext { bool show_help; - Array build_paths; // Contains `Path` objects to output filename, pdb, resource and intermediate files. - // BuildPath enum contains the indices of paths we know *before* the work starts. - String out_filepath; String resource_filepath; String pdb_filepath; - bool has_resource; String link_flags; String extra_linker_flags; @@ -319,6 +300,8 @@ struct BuildContext { }; + + gb_global BuildContext build_context = {0}; bool global_warnings_as_errors(void) { @@ -622,6 +605,28 @@ bool allow_check_foreign_filepath(void) { // is_abs_path // has_subdir +enum TargetFileValidity : u8 { + TargetFileValidity_Invalid, + + TargetFileValidity_Writable_File, + TargetFileValidity_No_Write_Permission, + TargetFileValidity_Directory, + + TargetTargetFileValidity_COUNT, +}; + +TargetFileValidity set_output_filename(void) { + // Assembles the output filename from build_context information. + // Returns `true` if it doesn't exist or is a file. + // Returns `false` if a directory or write-protected file. + + + + + return TargetFileValidity_Writable_File; +} + + String const WIN32_SEPARATOR_STRING = {cast(u8 *)"\\", 1}; String const NIX_SEPARATOR_STRING = {cast(u8 *)"/", 1}; @@ -968,6 +973,7 @@ char *token_pos_to_string(TokenPos const &pos) { return s; } + void init_build_context(TargetMetrics *cross_target) { BuildContext *bc = &build_context; @@ -1146,166 +1152,8 @@ void init_build_context(TargetMetrics *cross_target) { bc->optimization_level = gb_clamp(bc->optimization_level, 0, 3); + + #undef LINK_FLAG_X64 #undef LINK_FLAG_386 } - -#if defined(GB_SYSTEM_WINDOWS) -// NOTE(IC): In order to find Visual C++ paths without relying on environment variables. -// NOTE(Jeroen): No longer needed in `main.cpp -> linker_stage`. We now resolve those paths in `init_build_paths`. -#include "microsoft_craziness.h" -#endif - -// NOTE(Jeroen): Set/create the output and other paths and report an error as appropriate. -// We've previously called `parse_build_flags`, so `out_filepath` should be set. -bool init_build_paths(String init_filename) { - gbAllocator ha = heap_allocator(); - BuildContext *bc = &build_context; - - // NOTE(Jeroen): We're pre-allocating BuildPathCOUNT slots so that certain paths are always at the same enumerated index. - array_init(&bc->build_paths, permanent_allocator(), BuildPathCOUNT); - - // [BuildPathMainPackage] Turn given init path into a `Path`, which includes normalizing it into a full path. - bc->build_paths[BuildPath_Main_Package] = path_from_string(ha, init_filename); - - bool produces_output_file = false; - if (bc->command_kind == Command_doc && bc->cmd_doc_flags & CmdDocFlag_DocFormat) { - produces_output_file = true; - } else if (bc->command_kind & Command__does_build) { - produces_output_file = true; - } - - if (!produces_output_file) { - // Command doesn't produce output files. We're done. - return true; - } - - #if defined(GB_SYSTEM_WINDOWS) - if (bc->resource_filepath.len > 0) { - bc->build_paths[BuildPath_RC] = path_from_string(ha, bc->resource_filepath); - bc->build_paths[BuildPath_RES] = path_from_string(ha, bc->resource_filepath); - bc->build_paths[BuildPath_RC].ext = copy_string(ha, STR_LIT("rc")); - bc->build_paths[BuildPath_RES].ext = copy_string(ha, STR_LIT("res")); - } - - if (bc->pdb_filepath.len > 0) { - bc->build_paths[BuildPath_PDB] = path_from_string(ha, bc->pdb_filepath); - } - - if ((bc->command_kind & Command__does_build) && (!bc->ignore_microsoft_magic)) { - // NOTE(ic): It would be nice to extend this so that we could specify the Visual Studio version that we want instead of defaulting to the latest. - Find_Result_Utf8 find_result = find_visual_studio_and_windows_sdk_utf8(); - - if (find_result.windows_sdk_version == 0) { - gb_printf_err("Windows SDK not found.\n"); - return false; - } - - GB_ASSERT(find_result.windows_sdk_um_library_path.len > 0); - GB_ASSERT(find_result.windows_sdk_ucrt_library_path.len > 0); - - if (find_result.windows_sdk_root.len > 0) { - bc->build_paths[BuildPath_Win_SDK_Root] = path_from_string(ha, find_result.windows_sdk_root); - } - - if (find_result.windows_sdk_um_library_path.len > 0) { - bc->build_paths[BuildPath_Win_SDK_UM_Lib] = path_from_string(ha, find_result.windows_sdk_um_library_path); - } - - if (find_result.windows_sdk_ucrt_library_path.len > 0) { - bc->build_paths[BuildPath_Win_SDK_UCRT_Lib] = path_from_string(ha, find_result.windows_sdk_ucrt_library_path); - } - - if (find_result.vs_exe_path.len > 0) { - bc->build_paths[BuildPath_VS_EXE] = path_from_string(ha, find_result.vs_exe_path); - } - - if (find_result.vs_library_path.len > 0) { - bc->build_paths[BuildPath_VS_LIB] = path_from_string(ha, find_result.vs_library_path); - } - - gb_free(ha, find_result.windows_sdk_root.text); - gb_free(ha, find_result.windows_sdk_um_library_path.text); - gb_free(ha, find_result.windows_sdk_ucrt_library_path.text); - gb_free(ha, find_result.vs_exe_path.text); - gb_free(ha, find_result.vs_library_path.text); - - } - #endif - - // All the build targets and OSes. - String output_extension; - - if (bc->command_kind == Command_doc && bc->cmd_doc_flags & CmdDocFlag_DocFormat) { - output_extension = STR_LIT("odin-doc"); - } else if (is_arch_wasm()) { - output_extension = STR_LIT("wasm"); - } else if (build_context.build_mode == BuildMode_Executable) { - // By default use a .bin executable extension. - output_extension = STR_LIT("bin"); - - if (build_context.metrics.os == TargetOs_windows) { - output_extension = STR_LIT("exe"); - } else if (build_context.cross_compiling && selected_target_metrics->metrics == &target_essence_amd64) { - output_extension = make_string(nullptr, 0); - } - } else if (build_context.build_mode == BuildMode_DynamicLibrary) { - // By default use a .so shared library extension. - output_extension = STR_LIT("so"); - - if (build_context.metrics.os == TargetOs_windows) { - output_extension = STR_LIT("dll"); - } else if (build_context.metrics.os == TargetOs_darwin) { - output_extension = STR_LIT("dylib"); - } - } else if (build_context.build_mode == BuildMode_Object) { - // By default use a .o object extension. - output_extension = STR_LIT("o"); - - if (build_context.metrics.os == TargetOs_windows) { - output_extension = STR_LIT("obj"); - } - } else if (build_context.build_mode == BuildMode_Assembly) { - // By default use a .S asm extension. - output_extension = STR_LIT("S"); - } else if (build_context.build_mode == BuildMode_LLVM_IR) { - output_extension = STR_LIT("ll"); - } else { - GB_PANIC("Unhandled build mode/target combination.\n"); - } - - if (bc->out_filepath.len > 0) { - bc->build_paths[BuildPath_Output] = path_from_string(ha, bc->out_filepath); - } else { - String output_name = remove_directory_from_path(init_filename); - output_name = remove_extension_from_path(output_name); - output_name = copy_string(ha, string_trim_whitespace(output_name)); - - Path output_path = path_from_string(ha, output_name); - - // Replace extension. - if (output_path.ext.len > 0) { - gb_free(ha, output_path.ext.text); - } - output_path.ext = copy_string(ha, output_extension); - - bc->build_paths[BuildPath_Output] = output_path; - } - - // Do we have an extension? We might not if the output filename was supplied. - if (bc->build_paths[BuildPath_Output].ext.len == 0) { - if (build_context.metrics.os == TargetOs_windows || build_context.build_mode != BuildMode_Executable) { - bc->build_paths[BuildPath_Output].ext = copy_string(ha, output_extension); - } - } - - // Check if output path is a directory. - if (path_is_directory(bc->build_paths[BuildPath_Output])) { - String output_file = path_to_string(ha, bc->build_paths[BuildPath_Output]); - defer (gb_free(ha, output_file.text)); - gb_printf_err("Output path %.*s is a directory.\n", LIT(output_file)); - return false; - } - - return true; -} \ No newline at end of file diff --git a/src/common.cpp b/src/common.cpp index 94248fb62..aaacda04b 100644 --- a/src/common.cpp +++ b/src/common.cpp @@ -675,7 +675,262 @@ wchar_t **command_line_to_wargv(wchar_t *cmd_line, int *_argc) { #endif -#include "path.cpp" + +#if defined(GB_SYSTEM_WINDOWS) + bool path_is_directory(String path) { + gbAllocator a = heap_allocator(); + String16 wstr = string_to_string16(a, path); + defer (gb_free(a, wstr.text)); + + i32 attribs = GetFileAttributesW(wstr.text); + if (attribs < 0) return false; + + return (attribs & FILE_ATTRIBUTE_DIRECTORY) != 0; + } + +#else + bool path_is_directory(String path) { + gbAllocator a = heap_allocator(); + char *copy = cast(char *)copy_string(a, path).text; + defer (gb_free(a, copy)); + + struct stat s; + if (stat(copy, &s) == 0) { + return (s.st_mode & S_IFDIR) != 0; + } + return false; + } +#endif + + +String path_to_full_path(gbAllocator a, String path) { + gbAllocator ha = heap_allocator(); + char *path_c = gb_alloc_str_len(ha, cast(char *)path.text, path.len); + defer (gb_free(ha, path_c)); + + char *fullpath = gb_path_get_full_name(a, path_c); + String res = string_trim_whitespace(make_string_c(fullpath)); +#if defined(GB_SYSTEM_WINDOWS) + for (isize i = 0; i < res.len; i++) { + if (res.text[i] == '\\') { + res.text[i] = '/'; + } + } +#endif + return res; +} + + + +struct FileInfo { + String name; + String fullpath; + i64 size; + bool is_dir; +}; + +enum ReadDirectoryError { + ReadDirectory_None, + + ReadDirectory_InvalidPath, + ReadDirectory_NotExists, + ReadDirectory_Permission, + ReadDirectory_NotDir, + ReadDirectory_Empty, + ReadDirectory_Unknown, + + ReadDirectory_COUNT, +}; + +i64 get_file_size(String path) { + char *c_str = alloc_cstring(heap_allocator(), path); + defer (gb_free(heap_allocator(), c_str)); + + gbFile f = {}; + gbFileError err = gb_file_open(&f, c_str); + defer (gb_file_close(&f)); + if (err != gbFileError_None) { + return -1; + } + return gb_file_size(&f); +} + + +#if defined(GB_SYSTEM_WINDOWS) +ReadDirectoryError read_directory(String path, Array *fi) { + GB_ASSERT(fi != nullptr); + + gbAllocator a = heap_allocator(); + + while (path.len > 0) { + Rune end = path[path.len-1]; + if (end == '/') { + path.len -= 1; + } else if (end == '\\') { + path.len -= 1; + } else { + break; + } + } + + if (path.len == 0) { + return ReadDirectory_InvalidPath; + } + { + char *c_str = alloc_cstring(a, path); + defer (gb_free(a, c_str)); + + gbFile f = {}; + gbFileError file_err = gb_file_open(&f, c_str); + defer (gb_file_close(&f)); + + switch (file_err) { + case gbFileError_Invalid: return ReadDirectory_InvalidPath; + case gbFileError_NotExists: return ReadDirectory_NotExists; + // case gbFileError_Permission: return ReadDirectory_Permission; + } + } + + if (!path_is_directory(path)) { + return ReadDirectory_NotDir; + } + + + char *new_path = gb_alloc_array(a, char, path.len+3); + defer (gb_free(a, new_path)); + + gb_memmove(new_path, path.text, path.len); + gb_memmove(new_path+path.len, "/*", 2); + new_path[path.len+2] = 0; + + String np = make_string(cast(u8 *)new_path, path.len+2); + String16 wstr = string_to_string16(a, np); + defer (gb_free(a, wstr.text)); + + WIN32_FIND_DATAW file_data = {}; + HANDLE find_file = FindFirstFileW(wstr.text, &file_data); + if (find_file == INVALID_HANDLE_VALUE) { + return ReadDirectory_Unknown; + } + defer (FindClose(find_file)); + + array_init(fi, a, 0, 100); + + do { + wchar_t *filename_w = file_data.cFileName; + i64 size = cast(i64)file_data.nFileSizeLow; + size |= (cast(i64)file_data.nFileSizeHigh) << 32; + String name = string16_to_string(a, make_string16_c(filename_w)); + if (name == "." || name == "..") { + gb_free(a, name.text); + continue; + } + + String filepath = {}; + filepath.len = path.len+1+name.len; + filepath.text = gb_alloc_array(a, u8, filepath.len+1); + defer (gb_free(a, filepath.text)); + gb_memmove(filepath.text, path.text, path.len); + gb_memmove(filepath.text+path.len, "/", 1); + gb_memmove(filepath.text+path.len+1, name.text, name.len); + + FileInfo info = {}; + info.name = name; + info.fullpath = path_to_full_path(a, filepath); + info.size = size; + info.is_dir = (file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; + array_add(fi, info); + } while (FindNextFileW(find_file, &file_data)); + + if (fi->count == 0) { + return ReadDirectory_Empty; + } + + return ReadDirectory_None; +} +#elif defined(GB_SYSTEM_LINUX) || defined(GB_SYSTEM_OSX) || defined(GB_SYSTEM_FREEBSD) || defined(GB_SYSTEM_OPENBSD) + +#include + +ReadDirectoryError read_directory(String path, Array *fi) { + GB_ASSERT(fi != nullptr); + + gbAllocator a = heap_allocator(); + + char *c_path = alloc_cstring(a, path); + defer (gb_free(a, c_path)); + + DIR *dir = opendir(c_path); + if (!dir) { + switch (errno) { + case ENOENT: + return ReadDirectory_NotExists; + case EACCES: + return ReadDirectory_Permission; + case ENOTDIR: + return ReadDirectory_NotDir; + default: + // ENOMEM: out of memory + // EMFILE: per-process limit on open fds reached + // ENFILE: system-wide limit on total open files reached + return ReadDirectory_Unknown; + } + GB_PANIC("unreachable"); + } + + array_init(fi, a, 0, 100); + + for (;;) { + struct dirent *entry = readdir(dir); + if (entry == nullptr) { + break; + } + + String name = make_string_c(entry->d_name); + if (name == "." || name == "..") { + continue; + } + + String filepath = {}; + filepath.len = path.len+1+name.len; + filepath.text = gb_alloc_array(a, u8, filepath.len+1); + defer (gb_free(a, filepath.text)); + gb_memmove(filepath.text, path.text, path.len); + gb_memmove(filepath.text+path.len, "/", 1); + gb_memmove(filepath.text+path.len+1, name.text, name.len); + filepath.text[filepath.len] = 0; + + + struct stat dir_stat = {}; + + if (stat((char *)filepath.text, &dir_stat)) { + continue; + } + + if (S_ISDIR(dir_stat.st_mode)) { + continue; + } + + i64 size = dir_stat.st_size; + + FileInfo info = {}; + info.name = name; + info.fullpath = path_to_full_path(a, filepath); + info.size = size; + array_add(fi, info); + } + + if (fi->count == 0) { + return ReadDirectory_Empty; + } + + return ReadDirectory_None; +} +#else +#error Implement read_directory +#endif + + struct LoadedFile { void *handle; diff --git a/src/gb/gb.h b/src/gb/gb.h index 3b2d6434c..b72a893f7 100644 --- a/src/gb/gb.h +++ b/src/gb/gb.h @@ -6273,44 +6273,20 @@ char *gb_path_get_full_name(gbAllocator a, char const *path) { #else char *p, *result, *fullpath = NULL; isize len; - fullpath = realpath(path, NULL); - - if (fullpath == NULL) { - // NOTE(Jeroen): Path doesn't exist. - if (gb_strlen(path) > 0 && path[0] == '/') { - // But it is an absolute path, so return as-is. - - fullpath = (char *)path; - len = gb_strlen(fullpath) + 1; - result = gb_alloc_array(a, char, len + 1); - - gb_memmove(result, fullpath, len); - result[len] = 0; - - } else { - // Appears to be a relative path, so construct an absolute one relative to . - char cwd[4096]; - getcwd(&cwd[0], 4096); - - isize path_len = gb_strlen(path); - isize cwd_len = gb_strlen(cwd); - len = cwd_len + 1 + path_len + 1; - result = gb_alloc_array(a, char, len); - - gb_memmove(result, (void *)cwd, cwd_len); - result[cwd_len] = '/'; - - gb_memmove(result + cwd_len + 1, (void *)path, gb_strlen(path)); - result[len] = 0; - - } - } else { - len = gb_strlen(fullpath) + 1; - result = gb_alloc_array(a, char, len + 1); - gb_memmove(result, fullpath, len); - result[len] = 0; - free(fullpath); + p = realpath(path, NULL); + fullpath = p; + if (p == NULL) { + // NOTE(bill): File does not exist + fullpath = cast(char *)path; } + + len = gb_strlen(fullpath); + + result = gb_alloc_array(a, char, len + 1); + gb_memmove(result, fullpath, len); + result[len] = 0; + free(p); + return result; #endif } diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 7781997f7..f5cb84785 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -967,12 +967,7 @@ lbProcedure *lb_create_main_procedure(lbModule *m, lbProcedure *startup_runtime) } String lb_filepath_ll_for_module(lbModule *m) { - String path = concatenate3_strings(permanent_allocator(), - build_context.build_paths[BuildPath_Output].basename, - STR_LIT("/"), - build_context.build_paths[BuildPath_Output].name - ); - + String path = m->gen->output_base; if (m->pkg) { path = concatenate3_strings(permanent_allocator(), path, STR_LIT("-"), m->pkg->name); } else if (USE_SEPARATE_MODULES) { @@ -983,12 +978,7 @@ String lb_filepath_ll_for_module(lbModule *m) { return path; } String lb_filepath_obj_for_module(lbModule *m) { - String path = concatenate3_strings(permanent_allocator(), - build_context.build_paths[BuildPath_Output].basename, - STR_LIT("/"), - build_context.build_paths[BuildPath_Output].name - ); - + String path = m->gen->output_base; if (m->pkg) { path = concatenate3_strings(permanent_allocator(), path, STR_LIT("-"), m->pkg->name); } diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 1a431a4ac..330059622 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -87,6 +87,7 @@ bool lb_init_generator(lbGenerator *gen, Checker *c) { return false; } + String init_fullpath = c->parser->init_fullpath; if (build_context.out_filepath.len == 0) { diff --git a/src/main.cpp b/src/main.cpp index 7b0364149..fc8792ceb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -46,6 +46,7 @@ gb_global Timings global_timings = {0}; #include "checker.cpp" #include "docs.cpp" + #include "llvm_backend.cpp" #if defined(GB_SYSTEM_OSX) @@ -56,8 +57,16 @@ gb_global Timings global_timings = {0}; #endif #include "query_data.cpp" + + +#if defined(GB_SYSTEM_WINDOWS) +// NOTE(IC): In order to find Visual C++ paths without relying on environment variables. +#include "microsoft_craziness.h" +#endif + #include "bug_report.cpp" + // NOTE(bill): 'name' is used in debugging and profiling modes i32 system_exec_command_line_app(char const *name, char const *fmt, ...) { isize const cmd_cap = 64<<20; // 64 MiB should be more than enough @@ -121,35 +130,34 @@ i32 system_exec_command_line_app(char const *name, char const *fmt, ...) { } + + i32 linker_stage(lbGenerator *gen) { i32 result = 0; Timings *timings = &global_timings; - String output_filename = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_Output]); - debugf("Linking %.*s\n", LIT(output_filename)); - - // TOOD(Jeroen): Make a `build_paths[BuildPath_Object] to avoid `%.*s.o`. + String output_base = gen->output_base; if (is_arch_wasm()) { timings_start_section(timings, str_lit("wasm-ld")); #if defined(GB_SYSTEM_WINDOWS) result = system_exec_command_line_app("wasm-ld", - "\"%.*s\\bin\\wasm-ld\" \"%.*s.o\" -o \"%.*s\" %.*s %.*s", + "\"%.*s\\bin\\wasm-ld\" \"%.*s.wasm.o\" -o \"%.*s.wasm\" %.*s %.*s", LIT(build_context.ODIN_ROOT), - LIT(output_filename), LIT(output_filename), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); + LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); #else result = system_exec_command_line_app("wasm-ld", - "wasm-ld \"%.*s.o\" -o \"%.*s\" %.*s %.*s", - LIT(output_filename), LIT(output_filename), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); + "wasm-ld \"%.*s.wasm.o\" -o \"%.*s.wasm\" %.*s %.*s", + LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); #endif return result; } if (build_context.cross_compiling && selected_target_metrics->metrics == &target_essence_amd64) { -#if defined(GB_SYSTEM_UNIX) +#ifdef GB_SYSTEM_UNIX result = system_exec_command_line_app("linker", "x86_64-essence-gcc \"%.*s.o\" -o \"%.*s\" %.*s %.*s", - LIT(output_filename), LIT(output_filename), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); + LIT(output_base), LIT(output_base), LIT(build_context.link_flags), LIT(build_context.extra_linker_flags)); #else gb_printf_err("Linking for cross compilation for this platform is not yet supported (%.*s %.*s)\n", LIT(target_os_names[build_context.metrics.os]), @@ -173,11 +181,28 @@ i32 linker_stage(lbGenerator *gen) { gbString lib_str = gb_string_make(heap_allocator(), ""); defer (gb_string_free(lib_str)); + char const *output_ext = "exe"; gbString link_settings = gb_string_make_reserve(heap_allocator(), 256); defer (gb_string_free(link_settings)); + + // NOTE(ic): It would be nice to extend this so that we could specify the Visual Studio version that we want instead of defaulting to the latest. + Find_Result_Utf8 find_result = find_visual_studio_and_windows_sdk_utf8(); + + if (find_result.windows_sdk_version == 0) { + gb_printf_err("Windows SDK not found.\n"); + exit(1); + } + + if (build_context.ignore_microsoft_magic) { + find_result = {}; + } + // Add library search paths. - if (build_context.build_paths[BuildPath_VS_LIB].basename.len > 0) { + if (find_result.vs_library_path.len > 0) { + GB_ASSERT(find_result.windows_sdk_um_library_path.len > 0); + GB_ASSERT(find_result.windows_sdk_ucrt_library_path.len > 0); + String path = {}; auto add_path = [&](String path) { if (path[path.len-1] == '\\') { @@ -185,9 +210,9 @@ i32 linker_stage(lbGenerator *gen) { } link_settings = gb_string_append_fmt(link_settings, " /LIBPATH:\"%.*s\"", LIT(path)); }; - add_path(build_context.build_paths[BuildPath_Win_SDK_UM_Lib].basename); - add_path(build_context.build_paths[BuildPath_Win_SDK_UCRT_Lib].basename); - add_path(build_context.build_paths[BuildPath_VS_LIB].basename); + add_path(find_result.windows_sdk_um_library_path); + add_path(find_result.windows_sdk_ucrt_library_path); + add_path(find_result.vs_library_path); } @@ -227,14 +252,14 @@ i32 linker_stage(lbGenerator *gen) { if (build_context.build_mode == BuildMode_DynamicLibrary) { + output_ext = "dll"; link_settings = gb_string_append_fmt(link_settings, " /DLL"); } else { link_settings = gb_string_append_fmt(link_settings, " /ENTRY:mainCRTStartup"); } if (build_context.pdb_filepath != "") { - String pdb_path = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_PDB]); - link_settings = gb_string_append_fmt(link_settings, " /PDB:%.*s", LIT(pdb_path)); + link_settings = gb_string_append_fmt(link_settings, " /PDB:%.*s", LIT(build_context.pdb_filepath)); } if (build_context.no_crt) { @@ -275,21 +300,13 @@ i32 linker_stage(lbGenerator *gen) { object_files = gb_string_append_fmt(object_files, "\"%.*s\" ", LIT(object_path)); } - String vs_exe_path = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_VS_EXE]); - defer (gb_free(heap_allocator(), vs_exe_path.text)); - char const *subsystem_str = build_context.use_subsystem_windows ? "WINDOWS" : "CONSOLE"; if (!build_context.use_lld) { // msvc if (build_context.has_resource) { - String rc_path = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_RC]); - String res_path = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_RES]); - defer (gb_free(heap_allocator(), rc_path.text)); - defer (gb_free(heap_allocator(), res_path.text)); - result = system_exec_command_line_app("msvc-link", - "\"rc.exe\" /nologo /fo \"%.*s\" \"%.*s\"", - LIT(res_path), - LIT(rc_path) + "\"rc.exe\" /nologo /fo \"%.*s.res\" \"%.*s.rc\"", + LIT(output_base), + LIT(build_context.resource_filepath) ); if (result) { @@ -297,13 +314,13 @@ i32 linker_stage(lbGenerator *gen) { } result = system_exec_command_line_app("msvc-link", - "\"%.*slink.exe\" %s \"%.*s\" -OUT:\"%.*s\" %s " + "\"%.*slink.exe\" %s \"%.*s.res\" -OUT:\"%.*s.%s\" %s " "/nologo /incremental:no /opt:ref /subsystem:%s " " %.*s " " %.*s " " %s " "", - LIT(vs_exe_path), object_files, LIT(res_path), LIT(output_filename), + LIT(find_result.vs_exe_path), object_files, LIT(output_base), LIT(output_base), output_ext, link_settings, subsystem_str, LIT(build_context.link_flags), @@ -312,13 +329,13 @@ i32 linker_stage(lbGenerator *gen) { ); } else { result = system_exec_command_line_app("msvc-link", - "\"%.*slink.exe\" %s -OUT:\"%.*s\" %s " + "\"%.*slink.exe\" %s -OUT:\"%.*s.%s\" %s " "/nologo /incremental:no /opt:ref /subsystem:%s " " %.*s " " %.*s " " %s " "", - LIT(vs_exe_path), object_files, LIT(output_filename), + LIT(find_result.vs_exe_path), object_files, LIT(output_base), output_ext, link_settings, subsystem_str, LIT(build_context.link_flags), @@ -333,13 +350,13 @@ i32 linker_stage(lbGenerator *gen) { } else { // lld result = system_exec_command_line_app("msvc-lld-link", - "\"%.*s\\bin\\lld-link\" %s -OUT:\"%.*s\" %s " + "\"%.*s\\bin\\lld-link\" %s -OUT:\"%.*s.%s\" %s " "/nologo /incremental:no /opt:ref /subsystem:%s " " %.*s " " %.*s " " %s " "", - LIT(build_context.ODIN_ROOT), object_files, LIT(output_filename), + LIT(build_context.ODIN_ROOT), object_files, LIT(output_base),output_ext, link_settings, subsystem_str, LIT(build_context.link_flags), @@ -398,7 +415,7 @@ i32 linker_stage(lbGenerator *gen) { } else if (string_ends_with(lib, str_lit(".so"))) { // dynamic lib, relative path to executable // NOTE(vassvik): it is the user's responsibility to make sure the shared library files are visible - // at runtime to the executable + // at runtimeto the executable lib_str = gb_string_append_fmt(lib_str, " -l:\"%s/%.*s\" ", cwd, LIT(lib)); } else { // dynamic or static system lib, just link regularly searching system library paths @@ -414,6 +431,9 @@ i32 linker_stage(lbGenerator *gen) { object_files = gb_string_append_fmt(object_files, "\"%.*s\" ", LIT(object_path)); } + // Unlike the Win32 linker code, the output_ext includes the dot, because + // typically executable files on *NIX systems don't have extensions. + String output_ext = {}; gbString link_settings = gb_string_make_reserve(heap_allocator(), 32); if (build_context.no_crt) { @@ -441,12 +461,26 @@ i32 linker_stage(lbGenerator *gen) { // correctly this way since all the other dependencies provided implicitly // by the compiler frontend are still needed and most of the command // line arguments prepared previously are incompatible with ld. + // + // Shared libraries are .dylib on MacOS and .so on Linux. + if (build_context.metrics.os == TargetOs_darwin) { + output_ext = STR_LIT(".dylib"); + } else { + output_ext = STR_LIT(".so"); + } link_settings = gb_string_appendc(link_settings, "-Wl,-init,'_odin_entry_point' "); link_settings = gb_string_appendc(link_settings, "-Wl,-fini,'_odin_exit_point' "); } else if (build_context.metrics.os != TargetOs_openbsd) { // OpenBSD defaults to PIE executable. do not pass -no-pie for it. link_settings = gb_string_appendc(link_settings, "-no-pie "); } + if (build_context.out_filepath.len > 0) { + //NOTE(thebirk): We have a custom -out arguments, so we should use the extension from that + isize pos = string_extension_position(build_context.out_filepath); + if (pos > 0) { + output_ext = substring(build_context.out_filepath, pos, build_context.out_filepath.len); + } + } gbString platform_lib_str = gb_string_make(heap_allocator(), ""); defer (gb_string_free(platform_lib_str)); @@ -473,7 +507,7 @@ i32 linker_stage(lbGenerator *gen) { defer (gb_string_free(link_command_line)); link_command_line = gb_string_appendc(link_command_line, object_files); - link_command_line = gb_string_append_fmt(link_command_line, " -o \"%.*s\" ", LIT(output_filename)); + link_command_line = gb_string_append_fmt(link_command_line, " -o \"%.*s%.*s\" ", LIT(output_base), LIT(output_ext)); link_command_line = gb_string_append_fmt(link_command_line, " %s ", platform_lib_str); link_command_line = gb_string_append_fmt(link_command_line, " %s ", lib_str); link_command_line = gb_string_append_fmt(link_command_line, " %.*s ", LIT(build_context.link_flags)); @@ -490,7 +524,9 @@ i32 linker_stage(lbGenerator *gen) { if (build_context.ODIN_DEBUG) { // NOTE: macOS links DWARF symbols dynamically. Dsymutil will map the stubs in the exe // to the symbols in the object file - result = system_exec_command_line_app("dsymutil", "dsymutil %.*s", LIT(output_filename)); + result = system_exec_command_line_app("dsymutil", + "dsymutil %.*s%.*s", LIT(output_base), LIT(output_ext) + ); if (result) { return result; @@ -1490,10 +1526,6 @@ bool parse_build_flags(Array args) { gb_printf_err("Invalid -resource path %.*s, missing .rc\n", LIT(path)); bad_flags = true; break; - } else if (!gb_file_exists((const char *)path.text)) { - gb_printf_err("Invalid -resource path %.*s, file does not exist.\n", LIT(path)); - bad_flags = true; - break; } build_context.resource_filepath = substring(path, 0, string_extension_position(path)); build_context.has_resource = true; @@ -1508,11 +1540,6 @@ bool parse_build_flags(Array args) { String path = value.value_string; path = string_trim_whitespace(path); if (is_build_flag_path_valid(path)) { - if (path_is_directory(path)) { - gb_printf_err("Invalid -pdb-name path. %.*s, is a directory.\n", LIT(path)); - bad_flags = true; - break; - } // #if defined(GB_SYSTEM_WINDOWS) // String ext = path_extension(path); // if (ext != ".pdb") { @@ -2639,8 +2666,6 @@ int main(int arg_count, char const **arg_ptr) { return 1; } - init_filename = copy_string(permanent_allocator(), init_filename); - if (init_filename == "-help" || init_filename == "--help") { build_context.show_help = true; @@ -2663,12 +2688,6 @@ int main(int arg_count, char const **arg_ptr) { gb_printf_err("Did you mean `%.*s %.*s %.*s -file`?\n", LIT(args[0]), LIT(command), LIT(init_filename)); gb_printf_err("The `-file` flag tells it to treat a file as a self-contained package.\n"); return 1; - } else { - String const ext = str_lit(".odin"); - if (!string_ends_with(init_filename, ext)) { - gb_printf_err("Expected either a directory or a .odin file, got '%.*s'\n", LIT(init_filename)); - return 1; - } } } } @@ -2690,24 +2709,13 @@ int main(int arg_count, char const **arg_ptr) { get_fullpath_relative(heap_allocator(), odin_root_dir(), str_lit("shared"))); } + init_build_context(selected_target_metrics ? selected_target_metrics->metrics : nullptr); // if (build_context.word_size == 4 && build_context.metrics.os != TargetOs_js) { // print_usage_line(0, "%.*s 32-bit is not yet supported for this platform", LIT(args[0])); // return 1; // } - // Set and check build paths... - if (!init_build_paths(init_filename)) { - return 1; - } - - if (build_context.show_debug_messages) { - for_array(i, build_context.build_paths) { - String build_path = path_to_string(heap_allocator(), build_context.build_paths[i]); - debugf("build_paths[%ld]: %.*s\n", i, LIT(build_path)); - } - } - init_global_thread_pool(); defer (thread_pool_destroy(&global_thread_pool)); @@ -2724,8 +2732,6 @@ int main(int arg_count, char const **arg_ptr) { } defer (destroy_parser(parser)); - // TODO(jeroen): Remove the `init_filename` param. - // Let's put that on `build_context.build_paths[0]` instead. if (parse_packages(parser, init_filename) != ParseFile_None) { return 1; } @@ -2804,14 +2810,16 @@ int main(int arg_count, char const **arg_ptr) { } if (run_output) { - String exe_name = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_Output]); - defer (gb_free(heap_allocator(), exe_name.text)); - #if defined(GB_SYSTEM_WINDOWS) - return system_exec_command_line_app("odin run", "%.*s %.*s", LIT(exe_name), LIT(run_args_string)); + return system_exec_command_line_app("odin run", "%.*s.exe %.*s", LIT(gen->output_base), LIT(run_args_string)); #else - return system_exec_command_line_app("odin run", "\"%.*s\" %.*s", LIT(exe_name), LIT(run_args_string)); + //NOTE(thebirk): This whole thing is a little leaky + String output_ext = {}; + String complete_path = concatenate_strings(permanent_allocator(), gen->output_base, output_ext); + complete_path = path_to_full_path(permanent_allocator(), complete_path); + return system_exec_command_line_app("odin run", "\"%.*s\" %.*s", LIT(complete_path), LIT(run_args_string)); #endif } + return 0; } diff --git a/src/parser.cpp b/src/parser.cpp index df7f908a6..767119aa8 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -5751,7 +5751,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } } } - + { // Add these packages serially and then process them parallel mutex_lock(&p->wait_mutex); diff --git a/src/path.cpp b/src/path.cpp deleted file mode 100644 index 8d8e532b8..000000000 --- a/src/path.cpp +++ /dev/null @@ -1,333 +0,0 @@ -/* - Path handling utilities. -*/ - -#if defined(GB_SYSTEM_WINDOWS) - bool path_is_directory(String path) { - gbAllocator a = heap_allocator(); - String16 wstr = string_to_string16(a, path); - defer (gb_free(a, wstr.text)); - - i32 attribs = GetFileAttributesW(wstr.text); - if (attribs < 0) return false; - - return (attribs & FILE_ATTRIBUTE_DIRECTORY) != 0; - } - -#else - bool path_is_directory(String path) { - gbAllocator a = heap_allocator(); - char *copy = cast(char *)copy_string(a, path).text; - defer (gb_free(a, copy)); - - struct stat s; - if (stat(copy, &s) == 0) { - return (s.st_mode & S_IFDIR) != 0; - } - return false; - } -#endif - - -String path_to_full_path(gbAllocator a, String path) { - gbAllocator ha = heap_allocator(); - char *path_c = gb_alloc_str_len(ha, cast(char *)path.text, path.len); - defer (gb_free(ha, path_c)); - - char *fullpath = gb_path_get_full_name(a, path_c); - String res = string_trim_whitespace(make_string_c(fullpath)); -#if defined(GB_SYSTEM_WINDOWS) - for (isize i = 0; i < res.len; i++) { - if (res.text[i] == '\\') { - res.text[i] = '/'; - } - } -#endif - return copy_string(a, res); -} - -struct Path { - String basename; - String name; - String ext; -}; - -// NOTE(Jeroen): Naively turns a Path into a string. -String path_to_string(gbAllocator a, Path path) { - if (path.basename.len + path.name.len + path.ext.len == 0) { - return make_string(nullptr, 0); - } - - isize len = path.basename.len + 1 + path.name.len + 1; - if (path.ext.len > 0) { - len += path.ext.len + 1; - } - - u8 *str = gb_alloc_array(a, u8, len); - - isize i = 0; - gb_memmove(str+i, path.basename.text, path.basename.len); i += path.basename.len; - gb_memmove(str+i, "/", 1); i += 1; - gb_memmove(str+i, path.name.text, path.name.len); i += path.name.len; - if (path.ext.len > 0) { - gb_memmove(str+i, ".", 1); i += 1; - gb_memmove(str+i, path.ext.text, path.ext.len); i += path.ext.len; - } - str[i] = 0; - - String res = make_string(str, i); - res = string_trim_whitespace(res); - return res; -} - -// NOTE(Jeroen): Naively turns a Path into a string, then normalizes it using `path_to_full_path`. -String path_to_full_path(gbAllocator a, Path path) { - String temp = path_to_string(heap_allocator(), path); - defer (gb_free(heap_allocator(), temp.text)); - - return path_to_full_path(a, temp); -} - -// NOTE(Jeroen): Takes a path like "odin" or "W:\Odin", turns it into a full path, -// and then breaks it into its components to make a Path. -Path path_from_string(gbAllocator a, String const &path) { - Path res = {}; - - if (path.len == 0) return res; - - String fullpath = path_to_full_path(a, path); - defer (gb_free(heap_allocator(), fullpath.text)); - - res.basename = directory_from_path(fullpath); - res.basename = copy_string(a, res.basename); - - if (string_ends_with(fullpath, '/')) { - // It's a directory. We don't need to tinker with the name and extension. - return res; - } - - isize name_start = (res.basename.len > 0) ? res.basename.len + 1 : res.basename.len; - res.name = substring(fullpath, name_start, fullpath.len); - res.name = remove_extension_from_path(res.name); - res.name = copy_string(a, res.name); - - res.ext = path_extension(fullpath, false); // false says not to include the dot. - res.ext = copy_string(a, res.ext); - return res; -} - -bool path_is_directory(Path path) { - String path_string = path_to_full_path(heap_allocator(), path); - defer (gb_free(heap_allocator(), path_string.text)); - - return path_is_directory(path_string); -} - -struct FileInfo { - String name; - String fullpath; - i64 size; - bool is_dir; -}; - -enum ReadDirectoryError { - ReadDirectory_None, - - ReadDirectory_InvalidPath, - ReadDirectory_NotExists, - ReadDirectory_Permission, - ReadDirectory_NotDir, - ReadDirectory_Empty, - ReadDirectory_Unknown, - - ReadDirectory_COUNT, -}; - -i64 get_file_size(String path) { - char *c_str = alloc_cstring(heap_allocator(), path); - defer (gb_free(heap_allocator(), c_str)); - - gbFile f = {}; - gbFileError err = gb_file_open(&f, c_str); - defer (gb_file_close(&f)); - if (err != gbFileError_None) { - return -1; - } - return gb_file_size(&f); -} - - -#if defined(GB_SYSTEM_WINDOWS) -ReadDirectoryError read_directory(String path, Array *fi) { - GB_ASSERT(fi != nullptr); - - gbAllocator a = heap_allocator(); - - while (path.len > 0) { - Rune end = path[path.len-1]; - if (end == '/') { - path.len -= 1; - } else if (end == '\\') { - path.len -= 1; - } else { - break; - } - } - - if (path.len == 0) { - return ReadDirectory_InvalidPath; - } - { - char *c_str = alloc_cstring(a, path); - defer (gb_free(a, c_str)); - - gbFile f = {}; - gbFileError file_err = gb_file_open(&f, c_str); - defer (gb_file_close(&f)); - - switch (file_err) { - case gbFileError_Invalid: return ReadDirectory_InvalidPath; - case gbFileError_NotExists: return ReadDirectory_NotExists; - // case gbFileError_Permission: return ReadDirectory_Permission; - } - } - - if (!path_is_directory(path)) { - return ReadDirectory_NotDir; - } - - - char *new_path = gb_alloc_array(a, char, path.len+3); - defer (gb_free(a, new_path)); - - gb_memmove(new_path, path.text, path.len); - gb_memmove(new_path+path.len, "/*", 2); - new_path[path.len+2] = 0; - - String np = make_string(cast(u8 *)new_path, path.len+2); - String16 wstr = string_to_string16(a, np); - defer (gb_free(a, wstr.text)); - - WIN32_FIND_DATAW file_data = {}; - HANDLE find_file = FindFirstFileW(wstr.text, &file_data); - if (find_file == INVALID_HANDLE_VALUE) { - return ReadDirectory_Unknown; - } - defer (FindClose(find_file)); - - array_init(fi, a, 0, 100); - - do { - wchar_t *filename_w = file_data.cFileName; - i64 size = cast(i64)file_data.nFileSizeLow; - size |= (cast(i64)file_data.nFileSizeHigh) << 32; - String name = string16_to_string(a, make_string16_c(filename_w)); - if (name == "." || name == "..") { - gb_free(a, name.text); - continue; - } - - String filepath = {}; - filepath.len = path.len+1+name.len; - filepath.text = gb_alloc_array(a, u8, filepath.len+1); - defer (gb_free(a, filepath.text)); - gb_memmove(filepath.text, path.text, path.len); - gb_memmove(filepath.text+path.len, "/", 1); - gb_memmove(filepath.text+path.len+1, name.text, name.len); - - FileInfo info = {}; - info.name = name; - info.fullpath = path_to_full_path(a, filepath); - info.size = size; - info.is_dir = (file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; - array_add(fi, info); - } while (FindNextFileW(find_file, &file_data)); - - if (fi->count == 0) { - return ReadDirectory_Empty; - } - - return ReadDirectory_None; -} -#elif defined(GB_SYSTEM_LINUX) || defined(GB_SYSTEM_OSX) || defined(GB_SYSTEM_FREEBSD) || defined(GB_SYSTEM_OPENBSD) - -#include - -ReadDirectoryError read_directory(String path, Array *fi) { - GB_ASSERT(fi != nullptr); - - gbAllocator a = heap_allocator(); - - char *c_path = alloc_cstring(a, path); - defer (gb_free(a, c_path)); - - DIR *dir = opendir(c_path); - if (!dir) { - switch (errno) { - case ENOENT: - return ReadDirectory_NotExists; - case EACCES: - return ReadDirectory_Permission; - case ENOTDIR: - return ReadDirectory_NotDir; - default: - // ENOMEM: out of memory - // EMFILE: per-process limit on open fds reached - // ENFILE: system-wide limit on total open files reached - return ReadDirectory_Unknown; - } - GB_PANIC("unreachable"); - } - - array_init(fi, a, 0, 100); - - for (;;) { - struct dirent *entry = readdir(dir); - if (entry == nullptr) { - break; - } - - String name = make_string_c(entry->d_name); - if (name == "." || name == "..") { - continue; - } - - String filepath = {}; - filepath.len = path.len+1+name.len; - filepath.text = gb_alloc_array(a, u8, filepath.len+1); - defer (gb_free(a, filepath.text)); - gb_memmove(filepath.text, path.text, path.len); - gb_memmove(filepath.text+path.len, "/", 1); - gb_memmove(filepath.text+path.len+1, name.text, name.len); - filepath.text[filepath.len] = 0; - - - struct stat dir_stat = {}; - - if (stat((char *)filepath.text, &dir_stat)) { - continue; - } - - if (S_ISDIR(dir_stat.st_mode)) { - continue; - } - - i64 size = dir_stat.st_size; - - FileInfo info = {}; - info.name = name; - info.fullpath = path_to_full_path(a, filepath); - info.size = size; - array_add(fi, info); - } - - if (fi->count == 0) { - return ReadDirectory_Empty; - } - - return ReadDirectory_None; -} -#else -#error Implement read_directory -#endif - diff --git a/src/string.cpp b/src/string.cpp index 3515df48e..d3dbc6904 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -245,14 +245,15 @@ gb_inline isize string_extension_position(String const &str) { return dot_pos; } -String path_extension(String const &str, bool include_dot = true) { +String path_extension(String const &str) { isize pos = string_extension_position(str); if (pos < 0) { return make_string(nullptr, 0); } - return substring(str, include_dot ? pos : pos + 1, str.len); + return substring(str, pos, str.len); } + String string_trim_whitespace(String str) { while (str.len > 0 && rune_is_whitespace(str[str.len-1])) { str.len--; @@ -327,10 +328,7 @@ String directory_from_path(String const &s) { break; } } - if (i >= 0) { - return substring(s, 0, i); - } - return substring(s, 0, 0); + return substring(s, 0, i); } String concatenate_strings(gbAllocator a, String const &x, String const &y) { diff --git a/tests/core/Makefile b/tests/core/Makefile index 9bb622633..6a92b4efb 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -1,6 +1,5 @@ ODIN=../../odin PYTHON=$(shell which python3) -OUT_FILE=test_binary.bin all: download_test_assets image_test compress_test strings_test hash_test crypto_test noise_test encoding_test \ math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test @@ -9,39 +8,39 @@ download_test_assets: $(PYTHON) download_assets.py image_test: - $(ODIN) run image/test_core_image.odin -out=$(OUT_FILE) -file + $(ODIN) run image/test_core_image.odin -file compress_test: - $(ODIN) run compress/test_core_compress.odin -out=$(OUT_FILE) -file + $(ODIN) run compress/test_core_compress.odin -file strings_test: - $(ODIN) run strings/test_core_strings.odin -out=$(OUT_FILE) -file + $(ODIN) run strings/test_core_strings.odin -file hash_test: - $(ODIN) run hash -out=$(OUT_FILE) -o:speed -no-bounds-check + $(ODIN) run hash -out=test_hash -o:speed -no-bounds-check crypto_test: - $(ODIN) run crypto -out=$(OUT_FILE) -o:speed -no-bounds-check + $(ODIN) run crypto -out=test_crypto_hash -o:speed -no-bounds-check noise_test: - $(ODIN) run math/noise -out=$(OUT_FILE) + $(ODIN) run math/noise -out=test_noise encoding_test: - $(ODIN) run encoding/hxa -out=$(OUT_FILE) -collection:tests=.. - $(ODIN) run encoding/json -out=$(OUT_FILE) - $(ODIN) run encoding/varint -out=$(OUT_FILE) + $(ODIN) run encoding/hxa -out=test_hxa -collection:tests=.. + $(ODIN) run encoding/json -out=test_json + $(ODIN) run encoding/varint -out=test_varint math_test: - $(ODIN) run math/test_core_math.odin -out=$(OUT_FILE) -file -collection:tests=.. + $(ODIN) run math/test_core_math.odin -out=test_core_math -file -collection:tests=.. linalg_glsl_math_test: - $(ODIN) run math/linalg/glsl/test_linalg_glsl_math.odin -file -out=$(OUT_FILE) -collection:tests=.. + $(ODIN) run math/linalg/glsl/test_linalg_glsl_math.odin -file -out=test_linalg_glsl_math -collection:tests=.. filepath_test: - $(ODIN) run path/filepath/test_core_filepath.odin -file -out=$(OUT_FILE) -collection:tests=.. + $(ODIN) run path/filepath/test_core_filepath.odin -file -out=test_core_filepath -collection:tests=.. reflect_test: - $(ODIN) run reflect/test_core_reflect.odin -file -out=$(OUT_FILE) -collection:tests=.. + $(ODIN) run reflect/test_core_reflect.odin -file -out=test_core_reflect -collection:tests=.. os_exit_test: - $(ODIN) run os/test_core_os_exit.odin -file -out=$(OUT_FILE) && exit 1 || exit 0 + $(ODIN) run os/test_core_os_exit.odin -file -out=test_core_os_exit && exit 1 || exit 0 diff --git a/tests/core/build.bat b/tests/core/build.bat index 331a473aa..2f9ba672e 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -1,8 +1,6 @@ @echo off -set OUT_FILE=test_binary.exe -set COMMON=-show-timings -no-bounds-check -vet -strict-style -collection:tests=.. -out:%OUT_FILE% +set COMMON=-show-timings -no-bounds-check -vet -strict-style -collection:tests=.. set PATH_TO_ODIN==..\..\odin - python3 download_assets.py echo --- echo Running core:image tests @@ -37,14 +35,14 @@ echo --- echo --- echo Running core:encoding tests echo --- -%PATH_TO_ODIN% run encoding/hxa %COMMON% -%PATH_TO_ODIN% run encoding/json %COMMON% +%PATH_TO_ODIN% run encoding/hxa %COMMON% +%PATH_TO_ODIN% run encoding/json %COMMON% %PATH_TO_ODIN% run encoding/varint %COMMON% echo --- echo Running core:math/noise tests echo --- -%PATH_TO_ODIN% run math/noise %COMMON% +%PATH_TO_ODIN% run math/noise %COMMON% echo --- echo Running core:math tests diff --git a/tests/core/math/big/build.bat b/tests/core/math/big/build.bat index ad199d775..16bdbc8ca 100644 --- a/tests/core/math/big/build.bat +++ b/tests/core/math/big/build.bat @@ -4,7 +4,7 @@ set PATH_TO_ODIN==..\..\..\..\odin set TEST_ARGS=-fast-tests set TEST_ARGS=-no-random set TEST_ARGS= -set OUT_NAME=math_big_test_library.dll +set OUT_NAME=math_big_test_library set COMMON=-build-mode:shared -show-timings -no-bounds-check -define:MATH_BIG_EXE=false -vet -strict-style echo --- echo Running core:math/big tests diff --git a/tests/issues/run.bat b/tests/issues/run.bat index a936bd896..a652d9694 100644 --- a/tests/issues/run.bat +++ b/tests/issues/run.bat @@ -1,12 +1,17 @@ @echo off -set PATH_TO_ODIN==..\..\odin -set COMMON=-collection:tests=.. -out:build\test_issue -if not exist "build" mkdir build -%PATH_TO_ODIN% build test_issue_829.odin %COMMON% -file -build\test_issue +if not exist "tests\issues\build\" mkdir tests\issues\build -%PATH_TO_ODIN% build test_issue_1592.odin %COMMON% -file -build\test_issue +set COMMON=-collection:tests=tests -out:tests\issues\build\test_issue -rmdir /S /Q build +@echo on + +.\odin build tests\issues\test_issue_829.odin %COMMON% -file +tests\issues\build\test_issue + +.\odin build tests\issues\test_issue_1592.odin %COMMON% -file +tests\issues\build\test_issue + +@echo off + +rmdir /S /Q tests\issues\build diff --git a/tests/issues/run.sh b/tests/issues/run.sh index ec6d7130d..117a9a5f1 100755 --- a/tests/issues/run.sh +++ b/tests/issues/run.sh @@ -1,17 +1,18 @@ #!/bin/bash -ODIN=../../odin -COMMON="-collection:tests=.. -out:build/test_issue.bin" - set -eu -mkdir -p build + +mkdir -p tests/issues/build + +COMMON="-collection:tests=tests -out:tests/issues/build/test_issue" + set -x -$ODIN build test_issue_829.odin $COMMON -file -build/test_issue.bin +./odin build tests/issues/test_issue_829.odin $COMMON -file +tests/issues/build/test_issue -$ODIN build test_issue_1592.odin $COMMON -file -build/test_issue.bin +./odin build tests/issues/test_issue_1592.odin $COMMON -file +tests/issues/build/test_issue set +x -rm -rf build +rm -rf tests/issues/build diff --git a/tests/vendor/Makefile b/tests/vendor/Makefile index 380e64e09..341067c6e 100644 --- a/tests/vendor/Makefile +++ b/tests/vendor/Makefile @@ -1,6 +1,5 @@ ODIN=../../odin ODINFLAGS= -OUT_FILE=test_binary.bin OS=$(shell uname) @@ -11,4 +10,4 @@ endif all: botan_test botan_test: - $(ODIN) run botan -out=$(OUT_FILE) -o:speed -no-bounds-check $(ODINFLAGS) + $(ODIN) run botan -out=botan_hash -o:speed -no-bounds-check $(ODINFLAGS) diff --git a/tests/vendor/build.bat b/tests/vendor/build.bat index 4bd9a6496..e70d9f1d5 100644 --- a/tests/vendor/build.bat +++ b/tests/vendor/build.bat @@ -1,6 +1,5 @@ @echo off -set OUT_FILE=test_binary.exe -set COMMON=-show-timings -no-bounds-check -vet -strict-style -out:%OUT_FILE% +set COMMON=-show-timings -no-bounds-check -vet -strict-style set PATH_TO_ODIN==..\..\odin echo --- From e01662c13959469c9c0bc4974a3b7c03d577d7a4 Mon Sep 17 00:00:00 2001 From: Florian Behr Date: Mon, 25 Apr 2022 13:23:05 +0200 Subject: [PATCH 39/79] Make allocator in pool_add_task() explicit --- core/thread/thread_pool.odin | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/thread/thread_pool.odin b/core/thread/thread_pool.odin index 3f782cf73..fe289d8aa 100644 --- a/core/thread/thread_pool.odin +++ b/core/thread/thread_pool.odin @@ -113,9 +113,8 @@ pool_join :: proc(pool: ^Pool) { // the thread pool. You can even add tasks from inside other tasks. // // Each task also needs an allocator which it either owns, or which is thread -// safe. By default, allocations in the task are disabled by use of the -// nil_allocator. -pool_add_task :: proc(pool: ^Pool, procedure: Task_Proc, data: rawptr, user_index: int = 0, allocator := context.allocator) { +// safe. +pool_add_task :: proc(pool: ^Pool, procedure: Task_Proc, data: rawptr, user_index: int = 0, allocator: mem.Allocator) { sync.guard(&pool.mutex) append(&pool.tasks, Task{ From 1fb76ad7687bc9efe538b79ac05dc0fafad06950 Mon Sep 17 00:00:00 2001 From: Florian Behr Date: Mon, 25 Apr 2022 13:41:19 +0200 Subject: [PATCH 40/79] change usage in demo.odin --- examples/demo/demo.odin | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/demo/demo.odin b/examples/demo/demo.odin index b03345849..a36acdf18 100644 --- a/examples/demo/demo.odin +++ b/examples/demo/demo.odin @@ -1166,7 +1166,8 @@ threading_example :: proc() { for i in 0..<30 { - thread.pool_add_task(pool=&pool, procedure=task_proc, data=nil, user_index=i) + // be mindful of the allocator used for tasks. The allocator needs to be thread safe, or be owned by the task for exclusive use + thread.pool_add_task(pool=&pool, procedure=task_proc, data=nil, user_index=i, allocator=context.allocator) } thread.pool_start(&pool) From ca6a1db7576dd72f5a5b4b73fec8cbad34e5bced Mon Sep 17 00:00:00 2001 From: Florian Behr Date: Mon, 25 Apr 2022 13:41:39 +0200 Subject: [PATCH 41/79] fix doc comment for pool_init --- core/thread/thread_pool.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/thread/thread_pool.odin b/core/thread/thread_pool.odin index fe289d8aa..4fc02309e 100644 --- a/core/thread/thread_pool.odin +++ b/core/thread/thread_pool.odin @@ -44,7 +44,7 @@ Pool :: struct { } // Once initialized, the pool's memory address is not allowed to change until -// it is destroyed. If thread_count < 1, thread count 1 will be used. +// it is destroyed. // // The thread pool requires an allocator which it either owns, or which is thread safe. pool_init :: proc(pool: ^Pool, thread_count: int, allocator: mem.Allocator) { From ee67a0b9a1d83acad3e8ade9314bdad3bb0d2197 Mon Sep 17 00:00:00 2001 From: Florian Behr Date: Mon, 25 Apr 2022 14:08:09 +0200 Subject: [PATCH 42/79] reorder procedure parameters to make sure the optional argument in pool_add_task() is last, and the argument order is consistent with pool_init() --- core/thread/thread_pool.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/thread/thread_pool.odin b/core/thread/thread_pool.odin index 4fc02309e..af80da9aa 100644 --- a/core/thread/thread_pool.odin +++ b/core/thread/thread_pool.odin @@ -47,7 +47,7 @@ Pool :: struct { // it is destroyed. // // The thread pool requires an allocator which it either owns, or which is thread safe. -pool_init :: proc(pool: ^Pool, thread_count: int, allocator: mem.Allocator) { +pool_init :: proc(pool: ^Pool, allocator: mem.Allocator, thread_count: int) { context.allocator = allocator pool.allocator = allocator pool.tasks = make([dynamic]Task) @@ -114,7 +114,7 @@ pool_join :: proc(pool: ^Pool) { // // Each task also needs an allocator which it either owns, or which is thread // safe. -pool_add_task :: proc(pool: ^Pool, procedure: Task_Proc, data: rawptr, user_index: int = 0, allocator: mem.Allocator) { +pool_add_task :: proc(pool: ^Pool, allocator: mem.Allocator, procedure: Task_Proc, data: rawptr, user_index: int = 0) { sync.guard(&pool.mutex) append(&pool.tasks, Task{ From a412d34574649cceacea6a171ccd4af6119c7a0f Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 26 Apr 2022 11:35:34 +0100 Subject: [PATCH 43/79] Fix #1740 --- core/mem/virtual/virtual.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/mem/virtual/virtual.odin b/core/mem/virtual/virtual.odin index 2035171a7..21ab5ef21 100644 --- a/core/mem/virtual/virtual.odin +++ b/core/mem/virtual/virtual.odin @@ -120,7 +120,7 @@ alloc_from_memory_block :: proc(block: ^Memory_Block, min_size, alignment: int) do_commit_if_necessary :: proc(block: ^Memory_Block, size: uint) -> (err: Allocator_Error) { if block.committed - block.used < size { pmblock := (^Platform_Memory_Block)(block) - base_offset := uint(uintptr(block) - uintptr(pmblock)) + base_offset := uint(uintptr(pmblock.block.base) - uintptr(pmblock)) platform_total_commit := base_offset + block.used + size assert(pmblock.committed <= pmblock.reserved) From 530401e5ee84bd3346b6a3213814a904696019df Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 26 Apr 2022 11:38:32 +0100 Subject: [PATCH 44/79] Fix #1729 --- src/check_type.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/check_type.cpp b/src/check_type.cpp index 51f472961..c0ae64baa 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -1338,14 +1338,14 @@ ParameterValue handle_parameter_value(CheckerContext *ctx, Type *in_type, Type * } } } - } else if (allow_caller_location && o.mode == Addressing_Context) { + } else if (allow_caller_location && (o.mode == Addressing_Context || are_types_identical(o.type, t_source_code_location))) { param_value.kind = ParameterValue_Value; param_value.ast_value = expr; } else if (o.value.kind != ExactValue_Invalid) { param_value.kind = ParameterValue_Constant; param_value.value = o.value; } else { - error(expr, "Default parameter must be a constant, %d", o.mode); + error(expr, "Default parameter must be a constant"); } } } else { From 3bd71229596427d43551ff6612d71505bb79c796 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 26 Apr 2022 11:42:01 +0100 Subject: [PATCH 45/79] Fix #1720 --- vendor/vulkan/_gen/create_vulkan_odin_wrapper.py | 3 +++ vendor/vulkan/core.odin | 3 +++ 2 files changed, 6 insertions(+) diff --git a/vendor/vulkan/_gen/create_vulkan_odin_wrapper.py b/vendor/vulkan/_gen/create_vulkan_odin_wrapper.py index ae1bc8d64..9a32f5796 100644 --- a/vendor/vulkan/_gen/create_vulkan_odin_wrapper.py +++ b/vendor/vulkan/_gen/create_vulkan_odin_wrapper.py @@ -626,6 +626,9 @@ with open("../core.odin", 'w', encoding='utf-8') as f: f.write(BASE) f.write(""" API_VERSION_1_0 :: (1<<22) | (0<<12) | (0) +API_VERSION_1_1 :: (1<<22) | (1<<12) | (0) +API_VERSION_1_2 :: (1<<22) | (2<<12) | (0) +API_VERSION_1_3 :: (1<<22) | (3<<12) | (0) MAKE_VERSION :: proc(major, minor, patch: u32) -> u32 { return (major<<22) | (minor<<12) | (patch) diff --git a/vendor/vulkan/core.odin b/vendor/vulkan/core.odin index 94c667910..b90bfad17 100644 --- a/vendor/vulkan/core.odin +++ b/vendor/vulkan/core.odin @@ -3,6 +3,9 @@ // package vulkan API_VERSION_1_0 :: (1<<22) | (0<<12) | (0) +API_VERSION_1_1 :: (1<<22) | (1<<12) | (0) +API_VERSION_1_2 :: (1<<22) | (2<<12) | (0) +API_VERSION_1_3 :: (1<<22) | (3<<12) | (0) MAKE_VERSION :: proc(major, minor, patch: u32) -> u32 { return (major<<22) | (minor<<12) | (patch) From c81fd2e5dd82fba0d5a1eb6771b1816cdb4ba574 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 26 Apr 2022 11:45:46 +0100 Subject: [PATCH 46/79] Fix #1644 --- core/math/linalg/specific.odin | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/math/linalg/specific.odin b/core/math/linalg/specific.odin index cb007bd91..a4aaeb012 100644 --- a/core/math/linalg/specific.odin +++ b/core/math/linalg/specific.odin @@ -479,21 +479,21 @@ angle_from_quaternion_f16 :: proc(q: Quaternionf16) -> f16 { return math.asin(q.x*q.x + q.y*q.y + q.z*q.z) * 2 } - return math.cos(q.x) * 2 + return math.acos(q.w) * 2 } angle_from_quaternion_f32 :: proc(q: Quaternionf32) -> f32 { if abs(q.w) > math.SQRT_THREE*0.5 { return math.asin(q.x*q.x + q.y*q.y + q.z*q.z) * 2 } - return math.cos(q.x) * 2 + return math.acos(q.w) * 2 } angle_from_quaternion_f64 :: proc(q: Quaternionf64) -> f64 { if abs(q.w) > math.SQRT_THREE*0.5 { return math.asin(q.x*q.x + q.y*q.y + q.z*q.z) * 2 } - return math.cos(q.x) * 2 + return math.acos(q.w) * 2 } angle_from_quaternion :: proc{ angle_from_quaternion_f16, From a5342a01267f55dec5a5b9f775cec8c8379139b1 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 26 Apr 2022 13:14:09 +0200 Subject: [PATCH 47/79] Address edge cases. --- .github/workflows/ci.yml | 11 ++---- .gitignore | 1 - Makefile | 4 +-- build_odin.sh | 4 +-- core/crypto/util/util.odin | 1 - src/build_settings.cpp | 42 ++++++++++++++++++---- src/path.cpp | 67 +++++++++++++++++++++++++++++++++-- src/string.cpp | 35 ------------------ tests/core/Makefile | 29 ++++++++------- tests/core/build.bat | 10 +++--- tests/core/math/big/build.bat | 2 +- tests/issues/run.bat | 21 ++++++----- tests/issues/run.sh | 19 +++++----- tests/vendor/Makefile | 3 +- tests/vendor/build.bat | 3 +- 15 files changed, 150 insertions(+), 102 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d72775636..3cc4283b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,9 +39,7 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: | - cd tests/issues - ./run.sh + run: tests/issues/run.sh timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 @@ -93,9 +91,7 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: | - cd tests/issues - ./run.sh + run: tests/issues/run.sh timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 @@ -167,8 +163,7 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat - cd tests\issues - call run.bat + call tests\issues\run.bat timeout-minutes: 10 - name: Odin check examples/all for Windows 32bits shell: cmd diff --git a/.gitignore b/.gitignore index d03a86fd7..e8b3d3050 100644 --- a/.gitignore +++ b/.gitignore @@ -269,7 +269,6 @@ bin/ # - Linux/MacOS odin odin.dSYM -*.bin # shared collection shared/ diff --git a/Makefile b/Makefile index 1a1c93180..82150c6a2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -all: debug +all: debug demo demo: - ./odin run examples/demo + ./odin run examples/demo/demo.odin report: ./odin report diff --git a/build_odin.sh b/build_odin.sh index 4810cafd2..aef3f2836 100755 --- a/build_odin.sh +++ b/build_odin.sh @@ -102,7 +102,7 @@ build_odin() { } run_demo() { - ./odin run examples/demo + ./odin run examples/demo/demo.odin -file } case $OS in @@ -147,4 +147,4 @@ if [[ $# -eq 1 ]]; then exit 0 else panic "Too many arguments!" -fi \ No newline at end of file +fi diff --git a/core/crypto/util/util.odin b/core/crypto/util/util.odin index 83b07e546..6273a232e 100644 --- a/core/crypto/util/util.odin +++ b/core/crypto/util/util.odin @@ -11,7 +11,6 @@ package util */ import "core:mem" -_ :: mem // @note(bp): this can replace the other two cast_slice :: #force_inline proc "contextless" ($D: typeid/[]$DE, src: $S/[]$SE) -> D { diff --git a/src/build_settings.cpp b/src/build_settings.cpp index 212ded5c8..89d370144 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -1276,16 +1276,44 @@ bool init_build_paths(String init_filename) { if (bc->out_filepath.len > 0) { bc->build_paths[BuildPath_Output] = path_from_string(ha, bc->out_filepath); + if (build_context.metrics.os == TargetOs_windows) { + String output_file = path_to_string(ha, bc->build_paths[BuildPath_Output]); + defer (gb_free(ha, output_file.text)); + if (path_is_directory(bc->build_paths[BuildPath_Output])) { + gb_printf_err("Output path %.*s is a directory.\n", LIT(output_file)); + return false; + } else if (bc->build_paths[BuildPath_Output].ext.len == 0) { + gb_printf_err("Output path %.*s must have an appropriate extension.\n", LIT(output_file)); + return false; + } + } } else { - String output_name = remove_directory_from_path(init_filename); - output_name = remove_extension_from_path(output_name); - output_name = copy_string(ha, string_trim_whitespace(output_name)); + Path output_path; - Path output_path = path_from_string(ha, output_name); + if (str_eq(init_filename, str_lit("."))) { + // We must name the output file after the current directory. + debugf("Output name will be created from current base name %.*s.\n", LIT(bc->build_paths[BuildPath_Main_Package].basename)); + String last_element = last_path_element(bc->build_paths[BuildPath_Main_Package].basename); - // Replace extension. - if (output_path.ext.len > 0) { - gb_free(ha, output_path.ext.text); + if (last_element.len == 0) { + gb_printf_err("The output name is created from the last path element. `%.*s` has none. Use `-out:output_name.ext` to set it.\n", LIT(bc->build_paths[BuildPath_Main_Package].basename)); + return false; + } + output_path.basename = copy_string(ha, bc->build_paths[BuildPath_Main_Package].basename); + output_path.name = copy_string(ha, last_element); + + } else { + // Init filename was not 'current path'. + // Contruct the output name from the path elements as usual. + String output_name = remove_directory_from_path(init_filename); + output_name = remove_extension_from_path(output_name); + output_name = copy_string(ha, string_trim_whitespace(output_name)); + output_path = path_from_string(ha, output_name); + + // Replace extension. + if (output_path.ext.len > 0) { + gb_free(ha, output_path.ext.text); + } } output_path.ext = copy_string(ha, output_extension); diff --git a/src/path.cpp b/src/path.cpp index 8d8e532b8..6f83c39ea 100644 --- a/src/path.cpp +++ b/src/path.cpp @@ -1,6 +1,46 @@ /* Path handling utilities. */ +String remove_extension_from_path(String const &s) { + for (isize i = s.len-1; i >= 0; i--) { + if (s[i] == '.') { + return substring(s, 0, i); + } + } + return s; +} + +String remove_directory_from_path(String const &s) { + isize len = 0; + for (isize i = s.len-1; i >= 0; i--) { + if (s[i] == '/' || + s[i] == '\\') { + break; + } + len += 1; + } + return substring(s, s.len-len, s.len); +} + +bool path_is_directory(String path); + +String directory_from_path(String const &s) { + if (path_is_directory(s)) { + return s; + } + + isize i = s.len-1; + for (; i >= 0; i--) { + if (s[i] == '/' || + s[i] == '\\') { + break; + } + } + if (i >= 0) { + return substring(s, 0, i); + } + return substring(s, 0, 0); +} #if defined(GB_SYSTEM_WINDOWS) bool path_is_directory(String path) { @@ -98,11 +138,15 @@ Path path_from_string(gbAllocator a, String const &path) { String fullpath = path_to_full_path(a, path); defer (gb_free(heap_allocator(), fullpath.text)); - res.basename = directory_from_path(fullpath); - res.basename = copy_string(a, res.basename); + res.basename = directory_from_path(fullpath); + res.basename = copy_string(a, res.basename); - if (string_ends_with(fullpath, '/')) { + if (path_is_directory(fullpath)) { // It's a directory. We don't need to tinker with the name and extension. + // It could have a superfluous trailing `/`. Remove it if so. + if (res.basename.len > 0 && res.basename.text[res.basename.len - 1] == '/') { + res.basename.len--; + } return res; } @@ -116,6 +160,23 @@ Path path_from_string(gbAllocator a, String const &path) { return res; } +// NOTE(Jeroen): Takes a path String and returns the last path element. +String last_path_element(String const &path) { + isize count = 0; + u8 * start = (u8 *)(&path.text[path.len - 1]); + for (isize length = path.len; length > 0 && path.text[length - 1] != '/'; length--) { + count++; + start--; + } + if (count > 0) { + start++; // Advance past the `/` and return the substring. + String res = make_string(start, count); + return res; + } + // Must be a root path like `/` or `C:/`, return empty String. + return STR_LIT(""); +} + bool path_is_directory(Path path) { String path_string = path_to_full_path(heap_allocator(), path); defer (gb_free(heap_allocator(), path_string.text)); diff --git a/src/string.cpp b/src/string.cpp index 3515df48e..616761265 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -298,41 +298,6 @@ String filename_from_path(String s) { return make_string(nullptr, 0); } -String remove_extension_from_path(String const &s) { - for (isize i = s.len-1; i >= 0; i--) { - if (s[i] == '.') { - return substring(s, 0, i); - } - } - return s; -} - -String remove_directory_from_path(String const &s) { - isize len = 0; - for (isize i = s.len-1; i >= 0; i--) { - if (s[i] == '/' || - s[i] == '\\') { - break; - } - len += 1; - } - return substring(s, s.len-len, s.len); -} - -String directory_from_path(String const &s) { - isize i = s.len-1; - for (; i >= 0; i--) { - if (s[i] == '/' || - s[i] == '\\') { - break; - } - } - if (i >= 0) { - return substring(s, 0, i); - } - return substring(s, 0, 0); -} - String concatenate_strings(gbAllocator a, String const &x, String const &y) { isize len = x.len+y.len; u8 *data = gb_alloc_array(a, u8, len+1); diff --git a/tests/core/Makefile b/tests/core/Makefile index 9bb622633..6a92b4efb 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -1,6 +1,5 @@ ODIN=../../odin PYTHON=$(shell which python3) -OUT_FILE=test_binary.bin all: download_test_assets image_test compress_test strings_test hash_test crypto_test noise_test encoding_test \ math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test @@ -9,39 +8,39 @@ download_test_assets: $(PYTHON) download_assets.py image_test: - $(ODIN) run image/test_core_image.odin -out=$(OUT_FILE) -file + $(ODIN) run image/test_core_image.odin -file compress_test: - $(ODIN) run compress/test_core_compress.odin -out=$(OUT_FILE) -file + $(ODIN) run compress/test_core_compress.odin -file strings_test: - $(ODIN) run strings/test_core_strings.odin -out=$(OUT_FILE) -file + $(ODIN) run strings/test_core_strings.odin -file hash_test: - $(ODIN) run hash -out=$(OUT_FILE) -o:speed -no-bounds-check + $(ODIN) run hash -out=test_hash -o:speed -no-bounds-check crypto_test: - $(ODIN) run crypto -out=$(OUT_FILE) -o:speed -no-bounds-check + $(ODIN) run crypto -out=test_crypto_hash -o:speed -no-bounds-check noise_test: - $(ODIN) run math/noise -out=$(OUT_FILE) + $(ODIN) run math/noise -out=test_noise encoding_test: - $(ODIN) run encoding/hxa -out=$(OUT_FILE) -collection:tests=.. - $(ODIN) run encoding/json -out=$(OUT_FILE) - $(ODIN) run encoding/varint -out=$(OUT_FILE) + $(ODIN) run encoding/hxa -out=test_hxa -collection:tests=.. + $(ODIN) run encoding/json -out=test_json + $(ODIN) run encoding/varint -out=test_varint math_test: - $(ODIN) run math/test_core_math.odin -out=$(OUT_FILE) -file -collection:tests=.. + $(ODIN) run math/test_core_math.odin -out=test_core_math -file -collection:tests=.. linalg_glsl_math_test: - $(ODIN) run math/linalg/glsl/test_linalg_glsl_math.odin -file -out=$(OUT_FILE) -collection:tests=.. + $(ODIN) run math/linalg/glsl/test_linalg_glsl_math.odin -file -out=test_linalg_glsl_math -collection:tests=.. filepath_test: - $(ODIN) run path/filepath/test_core_filepath.odin -file -out=$(OUT_FILE) -collection:tests=.. + $(ODIN) run path/filepath/test_core_filepath.odin -file -out=test_core_filepath -collection:tests=.. reflect_test: - $(ODIN) run reflect/test_core_reflect.odin -file -out=$(OUT_FILE) -collection:tests=.. + $(ODIN) run reflect/test_core_reflect.odin -file -out=test_core_reflect -collection:tests=.. os_exit_test: - $(ODIN) run os/test_core_os_exit.odin -file -out=$(OUT_FILE) && exit 1 || exit 0 + $(ODIN) run os/test_core_os_exit.odin -file -out=test_core_os_exit && exit 1 || exit 0 diff --git a/tests/core/build.bat b/tests/core/build.bat index 331a473aa..2f9ba672e 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -1,8 +1,6 @@ @echo off -set OUT_FILE=test_binary.exe -set COMMON=-show-timings -no-bounds-check -vet -strict-style -collection:tests=.. -out:%OUT_FILE% +set COMMON=-show-timings -no-bounds-check -vet -strict-style -collection:tests=.. set PATH_TO_ODIN==..\..\odin - python3 download_assets.py echo --- echo Running core:image tests @@ -37,14 +35,14 @@ echo --- echo --- echo Running core:encoding tests echo --- -%PATH_TO_ODIN% run encoding/hxa %COMMON% -%PATH_TO_ODIN% run encoding/json %COMMON% +%PATH_TO_ODIN% run encoding/hxa %COMMON% +%PATH_TO_ODIN% run encoding/json %COMMON% %PATH_TO_ODIN% run encoding/varint %COMMON% echo --- echo Running core:math/noise tests echo --- -%PATH_TO_ODIN% run math/noise %COMMON% +%PATH_TO_ODIN% run math/noise %COMMON% echo --- echo Running core:math tests diff --git a/tests/core/math/big/build.bat b/tests/core/math/big/build.bat index ad199d775..16bdbc8ca 100644 --- a/tests/core/math/big/build.bat +++ b/tests/core/math/big/build.bat @@ -4,7 +4,7 @@ set PATH_TO_ODIN==..\..\..\..\odin set TEST_ARGS=-fast-tests set TEST_ARGS=-no-random set TEST_ARGS= -set OUT_NAME=math_big_test_library.dll +set OUT_NAME=math_big_test_library set COMMON=-build-mode:shared -show-timings -no-bounds-check -define:MATH_BIG_EXE=false -vet -strict-style echo --- echo Running core:math/big tests diff --git a/tests/issues/run.bat b/tests/issues/run.bat index a936bd896..a652d9694 100644 --- a/tests/issues/run.bat +++ b/tests/issues/run.bat @@ -1,12 +1,17 @@ @echo off -set PATH_TO_ODIN==..\..\odin -set COMMON=-collection:tests=.. -out:build\test_issue -if not exist "build" mkdir build -%PATH_TO_ODIN% build test_issue_829.odin %COMMON% -file -build\test_issue +if not exist "tests\issues\build\" mkdir tests\issues\build -%PATH_TO_ODIN% build test_issue_1592.odin %COMMON% -file -build\test_issue +set COMMON=-collection:tests=tests -out:tests\issues\build\test_issue -rmdir /S /Q build +@echo on + +.\odin build tests\issues\test_issue_829.odin %COMMON% -file +tests\issues\build\test_issue + +.\odin build tests\issues\test_issue_1592.odin %COMMON% -file +tests\issues\build\test_issue + +@echo off + +rmdir /S /Q tests\issues\build diff --git a/tests/issues/run.sh b/tests/issues/run.sh index ec6d7130d..117a9a5f1 100755 --- a/tests/issues/run.sh +++ b/tests/issues/run.sh @@ -1,17 +1,18 @@ #!/bin/bash -ODIN=../../odin -COMMON="-collection:tests=.. -out:build/test_issue.bin" - set -eu -mkdir -p build + +mkdir -p tests/issues/build + +COMMON="-collection:tests=tests -out:tests/issues/build/test_issue" + set -x -$ODIN build test_issue_829.odin $COMMON -file -build/test_issue.bin +./odin build tests/issues/test_issue_829.odin $COMMON -file +tests/issues/build/test_issue -$ODIN build test_issue_1592.odin $COMMON -file -build/test_issue.bin +./odin build tests/issues/test_issue_1592.odin $COMMON -file +tests/issues/build/test_issue set +x -rm -rf build +rm -rf tests/issues/build diff --git a/tests/vendor/Makefile b/tests/vendor/Makefile index 380e64e09..341067c6e 100644 --- a/tests/vendor/Makefile +++ b/tests/vendor/Makefile @@ -1,6 +1,5 @@ ODIN=../../odin ODINFLAGS= -OUT_FILE=test_binary.bin OS=$(shell uname) @@ -11,4 +10,4 @@ endif all: botan_test botan_test: - $(ODIN) run botan -out=$(OUT_FILE) -o:speed -no-bounds-check $(ODINFLAGS) + $(ODIN) run botan -out=botan_hash -o:speed -no-bounds-check $(ODINFLAGS) diff --git a/tests/vendor/build.bat b/tests/vendor/build.bat index 4bd9a6496..e70d9f1d5 100644 --- a/tests/vendor/build.bat +++ b/tests/vendor/build.bat @@ -1,6 +1,5 @@ @echo off -set OUT_FILE=test_binary.exe -set COMMON=-show-timings -no-bounds-check -vet -strict-style -out:%OUT_FILE% +set COMMON=-show-timings -no-bounds-check -vet -strict-style set PATH_TO_ODIN==..\..\odin echo --- From 5e11ad2e1e6d0e335f47197c63a5ffdd8f405dd7 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 26 Apr 2022 14:23:23 +0200 Subject: [PATCH 48/79] Update test paths. --- .github/workflows/ci.yml | 11 ++++++++--- core/crypto/util/util.odin | 1 + tests/core/Makefile | 28 ++++++++++++++-------------- tests/core/build.bat | 28 ++++++++++++++-------------- tests/issues/run.bat | 14 +++++++------- tests/issues/run.sh | 16 ++++++++-------- tests/vendor/Makefile | 2 +- tests/vendor/build.bat | 4 ++-- 8 files changed, 55 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cc4283b0..d72775636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,9 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: tests/issues/run.sh + run: | + cd tests/issues + ./run.sh timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 @@ -91,7 +93,9 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: tests/issues/run.sh + run: | + cd tests/issues + ./run.sh timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 @@ -163,7 +167,8 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat - call tests\issues\run.bat + cd tests\issues + call run.bat timeout-minutes: 10 - name: Odin check examples/all for Windows 32bits shell: cmd diff --git a/core/crypto/util/util.odin b/core/crypto/util/util.odin index 6273a232e..83b07e546 100644 --- a/core/crypto/util/util.odin +++ b/core/crypto/util/util.odin @@ -11,6 +11,7 @@ package util */ import "core:mem" +_ :: mem // @note(bp): this can replace the other two cast_slice :: #force_inline proc "contextless" ($D: typeid/[]$DE, src: $S/[]$SE) -> D { diff --git a/tests/core/Makefile b/tests/core/Makefile index 6a92b4efb..2c24fef75 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -8,39 +8,39 @@ download_test_assets: $(PYTHON) download_assets.py image_test: - $(ODIN) run image/test_core_image.odin -file + $(ODIN) run image/test_core_image.odin -file -out:test_core_image compress_test: - $(ODIN) run compress/test_core_compress.odin -file + $(ODIN) run compress/test_core_compress.odin -file -out:test_core_compress strings_test: - $(ODIN) run strings/test_core_strings.odin -file + $(ODIN) run strings/test_core_strings.odin -file -out:test_core_strings hash_test: - $(ODIN) run hash -out=test_hash -o:speed -no-bounds-check + $(ODIN) run hash -o:speed -no-bounds-check -out:test_hash crypto_test: - $(ODIN) run crypto -out=test_crypto_hash -o:speed -no-bounds-check + $(ODIN) run crypto -o:speed -no-bounds-check -out:test_crypto_hash noise_test: - $(ODIN) run math/noise -out=test_noise + $(ODIN) run math/noise -out:test_noise encoding_test: - $(ODIN) run encoding/hxa -out=test_hxa -collection:tests=.. - $(ODIN) run encoding/json -out=test_json - $(ODIN) run encoding/varint -out=test_varint + $(ODIN) run encoding/hxa -collection:tests=.. -out:test_hxa + $(ODIN) run encoding/json -out:test_json + $(ODIN) run encoding/varint -out:test_varint math_test: - $(ODIN) run math/test_core_math.odin -out=test_core_math -file -collection:tests=.. + $(ODIN) run math/test_core_math.odin -file -collection:tests=.. -out:test_core_math linalg_glsl_math_test: - $(ODIN) run math/linalg/glsl/test_linalg_glsl_math.odin -file -out=test_linalg_glsl_math -collection:tests=.. + $(ODIN) run math/linalg/glsl/test_linalg_glsl_math.odin -file -collection:tests=.. -out:test_linalg_glsl_math filepath_test: - $(ODIN) run path/filepath/test_core_filepath.odin -file -out=test_core_filepath -collection:tests=.. + $(ODIN) run path/filepath/test_core_filepath.odin -file -collection:tests=.. -out:test_core_filepath reflect_test: - $(ODIN) run reflect/test_core_reflect.odin -file -out=test_core_reflect -collection:tests=.. + $(ODIN) run reflect/test_core_reflect.odin -file -collection:tests=.. -out:test_core_reflect os_exit_test: - $(ODIN) run os/test_core_os_exit.odin -file -out=test_core_os_exit && exit 1 || exit 0 + $(ODIN) run os/test_core_os_exit.odin -file -out:test_core_os_exit && exit 1 || exit 0 \ No newline at end of file diff --git a/tests/core/build.bat b/tests/core/build.bat index 2f9ba672e..b03fef4bb 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -5,61 +5,61 @@ python3 download_assets.py echo --- echo Running core:image tests echo --- -%PATH_TO_ODIN% run image %COMMON% +%PATH_TO_ODIN% run image %COMMON% -out:test_core_image.exe echo --- echo Running core:compress tests echo --- -%PATH_TO_ODIN% run compress %COMMON% +%PATH_TO_ODIN% run compress %COMMON% -out:test_core_compress.exe echo --- echo Running core:strings tests echo --- -%PATH_TO_ODIN% run strings %COMMON% +%PATH_TO_ODIN% run strings %COMMON% -out:test_core_strings.exe echo --- echo Running core:hash tests echo --- -%PATH_TO_ODIN% run hash %COMMON% -o:size +%PATH_TO_ODIN% run hash %COMMON% -o:size -out:test_core_hash.exe echo --- echo Running core:odin tests echo --- -%PATH_TO_ODIN% run odin %COMMON% -o:size +%PATH_TO_ODIN% run odin %COMMON% -o:size -out:test_core_odin.exe echo --- echo Running core:crypto hash tests echo --- -%PATH_TO_ODIN% run crypto %COMMON% +%PATH_TO_ODIN% run crypto %COMMON% -out:test_crypto_hash.exe echo --- echo Running core:encoding tests echo --- -%PATH_TO_ODIN% run encoding/hxa %COMMON% -%PATH_TO_ODIN% run encoding/json %COMMON% -%PATH_TO_ODIN% run encoding/varint %COMMON% +%PATH_TO_ODIN% run encoding/hxa %COMMON% -out:test_hxa.exe +%PATH_TO_ODIN% run encoding/json %COMMON% -out:test_json.exe +%PATH_TO_ODIN% run encoding/varint %COMMON% -out:test_varint.exe echo --- echo Running core:math/noise tests echo --- -%PATH_TO_ODIN% run math/noise %COMMON% +%PATH_TO_ODIN% run math/noise %COMMON% -out:test_noise.exe echo --- echo Running core:math tests echo --- -%PATH_TO_ODIN% run math %COMMON% +%PATH_TO_ODIN% run math %COMMON% -out:test_core_math.exe echo --- echo Running core:math/linalg/glsl tests echo --- -%PATH_TO_ODIN% run math/linalg/glsl %COMMON% +%PATH_TO_ODIN% run math/linalg/glsl %COMMON% -out:test_linalg_glsl.exe echo --- echo Running core:path/filepath tests echo --- -%PATH_TO_ODIN% run path/filepath %COMMON% +%PATH_TO_ODIN% run path/filepath %COMMON% -out:test_core_filepath.exe echo --- echo Running core:reflect tests echo --- -%PATH_TO_ODIN% run reflect %COMMON% +%PATH_TO_ODIN% run reflect %COMMON% -out:test_core_reflect.exe \ No newline at end of file diff --git a/tests/issues/run.bat b/tests/issues/run.bat index a652d9694..a7078ae0f 100644 --- a/tests/issues/run.bat +++ b/tests/issues/run.bat @@ -1,17 +1,17 @@ @echo off -if not exist "tests\issues\build\" mkdir tests\issues\build +if not exist "build\" mkdir build -set COMMON=-collection:tests=tests -out:tests\issues\build\test_issue +set COMMON=-collection:tests=.. -out:build\test_issue.exe @echo on -.\odin build tests\issues\test_issue_829.odin %COMMON% -file -tests\issues\build\test_issue +..\..\odin build test_issue_829.odin %COMMON% -file +build\test_issue -.\odin build tests\issues\test_issue_1592.odin %COMMON% -file -tests\issues\build\test_issue +..\..\odin build test_issue_1592.odin %COMMON% -file +build\test_issue @echo off -rmdir /S /Q tests\issues\build +rmdir /S /Q build diff --git a/tests/issues/run.sh b/tests/issues/run.sh index 117a9a5f1..ec0804bac 100755 --- a/tests/issues/run.sh +++ b/tests/issues/run.sh @@ -1,18 +1,18 @@ #!/bin/bash set -eu -mkdir -p tests/issues/build - -COMMON="-collection:tests=tests -out:tests/issues/build/test_issue" +mkdir -p build +ODIN=../../odin +COMMON="-collection:tests=.. -out:build/test_issue" set -x -./odin build tests/issues/test_issue_829.odin $COMMON -file -tests/issues/build/test_issue +$ODIN build test_issue_829.odin $COMMON -file +./build/test_issue -./odin build tests/issues/test_issue_1592.odin $COMMON -file -tests/issues/build/test_issue +$ODIN build test_issue_1592.odin $COMMON -file +./build/test_issue set +x -rm -rf tests/issues/build +rm -rf build diff --git a/tests/vendor/Makefile b/tests/vendor/Makefile index 341067c6e..6c68d7908 100644 --- a/tests/vendor/Makefile +++ b/tests/vendor/Makefile @@ -10,4 +10,4 @@ endif all: botan_test botan_test: - $(ODIN) run botan -out=botan_hash -o:speed -no-bounds-check $(ODINFLAGS) + $(ODIN) run botan -o:speed -no-bounds-check $(ODINFLAGS) -out=vendor_botan diff --git a/tests/vendor/build.bat b/tests/vendor/build.bat index e70d9f1d5..d92a5eaea 100644 --- a/tests/vendor/build.bat +++ b/tests/vendor/build.bat @@ -5,9 +5,9 @@ set PATH_TO_ODIN==..\..\odin echo --- echo Running vendor:botan tests echo --- -%PATH_TO_ODIN% run botan %COMMON% +%PATH_TO_ODIN% run botan %COMMON% -out:vendor_botan.exe echo --- echo Running vendor:glfw tests echo --- -%PATH_TO_ODIN% run glfw %COMMON% \ No newline at end of file +%PATH_TO_ODIN% run glfw %COMMON% -out:vendor_glfw.exe \ No newline at end of file From ba5e33bc3518c327057102f56ac2c1cce55dc76f Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 26 Apr 2022 14:51:16 +0200 Subject: [PATCH 49/79] Update CI workflow. --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cc4283b0..a5a6eddb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,9 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: tests/issues/run.sh + run: | + cd tests/issues + ./run.sh timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 @@ -91,7 +93,8 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - run: tests/issues/run.sh + cd tests/issues + ./run.sh timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 @@ -163,7 +166,8 @@ jobs: shell: cmd run: | call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat - call tests\issues\run.bat + cd tests\issues + call run.bat timeout-minutes: 10 - name: Odin check examples/all for Windows 32bits shell: cmd From 1c03e6805775c58044d417eea2504312b4e916ab Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 26 Apr 2022 14:56:28 +0200 Subject: [PATCH 50/79] Update CI. --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5a6eddb1..d72775636 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,8 +40,8 @@ jobs: timeout-minutes: 10 - name: Odin issues tests run: | - cd tests/issues - ./run.sh + cd tests/issues + ./run.sh timeout-minutes: 10 - name: Odin check examples/all for Linux i386 run: ./odin check examples/all -vet -strict-style -target:linux_i386 @@ -93,8 +93,9 @@ jobs: make timeout-minutes: 10 - name: Odin issues tests - cd tests/issues - ./run.sh + run: | + cd tests/issues + ./run.sh timeout-minutes: 10 - name: Odin check examples/all for Darwin arm64 run: ./odin check examples/all -vet -strict-style -target:darwin_arm64 From 40f0f5ad8dd56691ae42e9dd4b9a2f5e5395f3ee Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 26 Apr 2022 15:01:09 +0200 Subject: [PATCH 51/79] Update CI for math library. --- tests/core/math/big/build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/math/big/build.bat b/tests/core/math/big/build.bat index 16bdbc8ca..ad199d775 100644 --- a/tests/core/math/big/build.bat +++ b/tests/core/math/big/build.bat @@ -4,7 +4,7 @@ set PATH_TO_ODIN==..\..\..\..\odin set TEST_ARGS=-fast-tests set TEST_ARGS=-no-random set TEST_ARGS= -set OUT_NAME=math_big_test_library +set OUT_NAME=math_big_test_library.dll set COMMON=-build-mode:shared -show-timings -no-bounds-check -define:MATH_BIG_EXE=false -vet -strict-style echo --- echo Running core:math/big tests From d262eda91c527880a95f70a3d6fa20346c1d20d6 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Tue, 26 Apr 2022 15:10:31 +0200 Subject: [PATCH 52/79] Update Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 82150c6a2..91010a620 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -all: debug demo +all: debug demo: - ./odin run examples/demo/demo.odin + ./odin run examples/demo/demo.odin -file report: ./odin report From a6cef2e50ea4867f02bac5442debfa1de0743681 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Tue, 26 Apr 2022 21:47:45 +0100 Subject: [PATCH 53/79] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 9ca94bcdf..9a87ab8da 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2016-2021 Ginger Bill. All rights reserved. +Copyright (c) 2016-2022 Ginger Bill. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: From 96924969895d78d93e8f39158a0409a6a2c214ab Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 27 Apr 2022 12:27:53 +0100 Subject: [PATCH 54/79] Add `intrinsics.type_field_type` --- core/intrinsics/intrinsics.odin | 1 + src/check_builtin.cpp | 31 +++++++++++++++++++++++++++++++ src/checker_builtin_procs.hpp | 2 ++ 3 files changed, 34 insertions(+) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index a25e9783d..c132d4095 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -153,6 +153,7 @@ type_is_specialization_of :: proc($T, $S: typeid) -> bool --- type_is_variant_of :: proc($U, $V: typeid) -> bool where type_is_union(U) --- type_has_field :: proc($T: typeid, $name: string) -> bool --- +type_field_type :: proc($T: typeid, $name: string) -> typeid --- type_proc_parameter_count :: proc($T: typeid) -> int where type_is_proc(T) --- type_proc_return_count :: proc($T: typeid) -> int where type_is_proc(T) --- diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index e055539c5..6c7972d45 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -3926,6 +3926,37 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 break; } break; + case BuiltinProc_type_field_type: + { + Operand op = {}; + Type *bt = check_type(c, ce->args[0]); + Type *type = base_type(bt); + if (type == nullptr || type == t_invalid) { + error(ce->args[0], "Expected a type for '%.*s'", LIT(builtin_name)); + return false; + } + Operand x = {}; + check_expr(c, &x, ce->args[1]); + + if (!is_type_string(x.type) || x.mode != Addressing_Constant || x.value.kind != ExactValue_String) { + error(ce->args[1], "Expected a const string for field argument"); + return false; + } + + String field_name = x.value.value_string; + + Selection sel = lookup_field(type, field_name, false); + if (sel.index.count == 0) { + gbString t = type_to_string(type); + error(ce->args[1], "'%.*s' is not a field of type %s", LIT(field_name), t); + gb_string_free(t); + return false; + } + operand->mode = Addressing_Type; + operand->type = sel.entity->type; + break; + } + break; case BuiltinProc_type_is_specialization_of: { diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index fe14ae372..0f72f01f7 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -179,6 +179,7 @@ BuiltinProc__type_simple_boolean_begin, BuiltinProc__type_simple_boolean_end, BuiltinProc_type_has_field, + BuiltinProc_type_field_type, BuiltinProc_type_is_specialization_of, @@ -395,6 +396,7 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT(""), 0, false, Expr_Stmt, BuiltinProcPkg_intrinsics}, {STR_LIT("type_has_field"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_field_type"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("type_is_specialization_of"), 2, false, Expr_Expr, BuiltinProcPkg_intrinsics}, From 9349dfba8fec53f52f77a0c8928e115ec93ff447 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 27 Apr 2022 12:39:45 +0100 Subject: [PATCH 55/79] Add new builtin `container_of` --- core/runtime/core_builtin.odin | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/runtime/core_builtin.odin b/core/runtime/core_builtin.odin index 67b1bd37b..13e464a76 100644 --- a/core/runtime/core_builtin.odin +++ b/core/runtime/core_builtin.odin @@ -5,6 +5,16 @@ import "core:intrinsics" @builtin Maybe :: union($T: typeid) #maybe {T} + +@builtin +container_of :: #force_inline proc "contextless" (ptr: $P/^$Field_Type, $T: typeid, $field_name: string) -> ^T + where intrinsics.type_has_field(T, field_name), + intrinsics.type_field_type(T, field_name) == Field_Type { + offset :: offset_of_by_string(T, field_name) + return (^T)(uintptr(ptr) - offset) if ptr != nil else nil +} + + @thread_local global_default_temp_allocator_data: Default_Temp_Allocator @builtin From 3a9b0a22e728d1158efccb7f90e2c8aa05c486ef Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 27 Apr 2022 14:27:33 +0100 Subject: [PATCH 56/79] Add `core:container/intrusive/list` --- .../intrusive/list/intrusive_list.odin | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 core/container/intrusive/list/intrusive_list.odin diff --git a/core/container/intrusive/list/intrusive_list.odin b/core/container/intrusive/list/intrusive_list.odin new file mode 100644 index 000000000..88e21edc5 --- /dev/null +++ b/core/container/intrusive/list/intrusive_list.odin @@ -0,0 +1,173 @@ +package container_intrusive_list + +import "core:intrinsics" + +// An intrusive doubly-linked list +// +// As this is an intrusive container, a `Node` must be embedded in your own +// structure which is conventionally called a "link". The use of `push_front` +// and `push_back` take the address of this node. Retrieving the data +// associated with the node requires finding the relative offset of the node +// of the parent structure. The parent type and field name are given to +// `iterator_*` procedures, or to the built-in `container_of` procedure. +// +// This data structure is two-pointers in size: +// 8 bytes on 32-bit platforms and 16 bytes on 64-bit platforms +List :: struct { + head: ^Node, + tail: ^Node, +} + + +Node :: struct { + next, prev: ^Node, +} + +push_front :: proc(list: ^List, node: ^Node) { + if list.head != nil { + list.head.prev = node + node.prev, node.next = nil, list.head + list.head = node + } else { + list.head, list.tail = node, node + node.prev, node.next = nil, nil + } +} + +push_back :: proc(list: ^List, node: ^Node) { + if list.tail != nil { + list.tail.next = node + node.prev, node.next = list.tail, nil + list.tail = node + } else { + list.head, list.tail = node, node + node.prev, node.next = nil, nil + } +} + +remove :: proc(list: ^List, node: ^Node) { + if node != nil { + if node.next != nil { + node.next.prev = node.prev + } + if node.prev != nil { + node.prev.next = node.next + } + if list.head == node { + list.head = node.next + } + if list.tail == node { + list.tail = node.prev + } + } +} + +remove_by_proc :: proc(list: ^List, to_erase: proc(^Node) -> bool) { + for node := list.head; node != nil; { + next := node.next + if to_erase(node) { + if node.next != nil { + node.next.prev = node.prev + } + if node.prev != nil { + node.prev.next = node.next + } + if list.head == node { + list.head = node.next + } + if list.tail == node { + list.tail = node.prev + } + } + node = next + } +} + + +is_empty :: proc(list: ^List) -> bool { + return list.head == nil +} + +pop_front :: proc(list: ^List) -> ^Node { + link := list.head + if link == nil { + return nil + } + if link.next != nil { + link.next.prev = link.prev + } + if link.prev != nil { + link.prev.next = link.next + } + if link == list.head { + list.head = link.next + } + if link == list.tail { + list.tail = link.prev + } + return link + +} +pop_back :: proc(list: ^List) -> ^Node { + link := list.tail + if link == nil { + return nil + } + if link.next != nil { + link.next.prev = link.prev + } + if link.prev != nil { + link.prev.next = link.next + } + if link == list.head { + list.head = link.next + } + if link == list.tail { + list.tail = link.prev + } + return link +} + + +Iterator :: struct($T: typeid) { + curr: ^Node, + offset: uintptr, +} + +iterator_head :: proc(list: List, $T: typeid, $field_name: string) -> Iterator(T) + where intrinsics.type_has_field(T, field_name), + intrinsics.type_field_type(T, field_name) == Node { + return {list.head, offset_of_by_string(T, field_name)} +} + +iterator_tail :: proc(list: List, $T: typeid, $field_name: string) -> Iterator(T) + where intrinsics.type_has_field(T, field_name), + intrinsics.type_field_type(T, field_name) == Node { + return {list.tail, offset_of_by_string(T, field_name)} +} + +iterator_from_node :: proc(node: ^Node, $T: typeid, $field_name: string) -> Iterator(T) + where intrinsics.type_has_field(T, field_name), + intrinsics.type_field_type(T, field_name) == Node { + return {node, offset_of_by_string(T, field_name)} +} + +iterate_next :: proc(it: ^Iterator($T)) -> (ptr: ^T, ok: bool) { + node := it.curr + if node == nil { + return nil, false + } + it.curr = node.next + + return (^T)(uintptr(node) - it.offset), true +} + +iterate_prev :: proc(it: ^Iterator($T)) -> (ptr: ^T, ok: bool) { + node := it.curr + if node == nil { + return nil, false + } + it.curr = node.prev + + return (^T)(uintptr(node) - it.offset), true +} \ No newline at end of file From fbbb0d7610d2ace168df2aec65d76f51f35ec6a8 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 27 Apr 2022 14:51:04 +0100 Subject: [PATCH 57/79] Update intrinsics.odin for documentation --- core/intrinsics/intrinsics.odin | 54 +++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/core/intrinsics/intrinsics.odin b/core/intrinsics/intrinsics.odin index c132d4095..beb6f3f31 100644 --- a/core/intrinsics/intrinsics.odin +++ b/core/intrinsics/intrinsics.odin @@ -41,6 +41,10 @@ mem_copy_non_overlapping :: proc(dst, src: rawptr, len: int) --- mem_zero :: proc(ptr: rawptr, len: int) --- mem_zero_volatile :: proc(ptr: rawptr, len: int) --- +// prefer [^]T operations if possible +ptr_offset :: proc(ptr: ^$T, offset: int) -> ^T --- +ptr_sub :: proc(a, b: ^$T) -> int --- + unaligned_load :: proc(src: ^$T) -> T --- unaligned_store :: proc(dst: ^$T, val: T) -> T --- @@ -82,6 +86,7 @@ atomic_store_explicit :: proc(dst: ^$T, val: T, order: Atomic_Memory_Order) --- atomic_load :: proc(dst: ^$T) -> T --- atomic_load_explicit :: proc(dst: ^$T, order: Atomic_Memory_Order) -> T --- +// fetch then operator atomic_add :: proc(dst; ^$T, val: T) -> T --- atomic_add_explicit :: proc(dst; ^$T, val: T, order: Atomic_Memory_Order) -> T --- atomic_sub :: proc(dst; ^$T, val: T) -> T --- @@ -119,19 +124,20 @@ type_is_string :: proc($T: typeid) -> bool --- type_is_typeid :: proc($T: typeid) -> bool --- type_is_any :: proc($T: typeid) -> bool --- -type_is_endian_platform :: proc($T: typeid) -> bool --- -type_is_endian_little :: proc($T: typeid) -> bool --- -type_is_endian_big :: proc($T: typeid) -> bool --- -type_is_unsigned :: proc($T: typeid) -> bool --- -type_is_numeric :: proc($T: typeid) -> bool --- -type_is_ordered :: proc($T: typeid) -> bool --- -type_is_ordered_numeric :: proc($T: typeid) -> bool --- -type_is_indexable :: proc($T: typeid) -> bool --- -type_is_sliceable :: proc($T: typeid) -> bool --- -type_is_comparable :: proc($T: typeid) -> bool --- -type_is_simple_compare :: proc($T: typeid) -> bool --- // easily compared using memcmp (== and !=) -type_is_dereferenceable :: proc($T: typeid) -> bool --- -type_is_valid_map_key :: proc($T: typeid) -> bool --- +type_is_endian_platform :: proc($T: typeid) -> bool --- +type_is_endian_little :: proc($T: typeid) -> bool --- +type_is_endian_big :: proc($T: typeid) -> bool --- +type_is_unsigned :: proc($T: typeid) -> bool --- +type_is_numeric :: proc($T: typeid) -> bool --- +type_is_ordered :: proc($T: typeid) -> bool --- +type_is_ordered_numeric :: proc($T: typeid) -> bool --- +type_is_indexable :: proc($T: typeid) -> bool --- +type_is_sliceable :: proc($T: typeid) -> bool --- +type_is_comparable :: proc($T: typeid) -> bool --- +type_is_simple_compare :: proc($T: typeid) -> bool --- // easily compared using memcmp (== and !=) +type_is_dereferenceable :: proc($T: typeid) -> bool --- +type_is_valid_map_key :: proc($T: typeid) -> bool --- +type_is_valid_matrix_elements :: proc($T: typeid) -> bool --- type_is_named :: proc($T: typeid) -> bool --- type_is_pointer :: proc($T: typeid) -> bool --- @@ -146,6 +152,7 @@ type_is_enum :: proc($T: typeid) -> bool --- type_is_proc :: proc($T: typeid) -> bool --- type_is_bit_set :: proc($T: typeid) -> bool --- type_is_simd_vector :: proc($T: typeid) -> bool --- +type_is_matrix :: proc($T: typeid) -> bool --- type_has_nil :: proc($T: typeid) -> bool --- @@ -161,20 +168,41 @@ type_proc_return_count :: proc($T: typeid) -> int where type_is_proc(T) --- type_proc_parameter_type :: proc($T: typeid, index: int) -> typeid where type_is_proc(T) --- type_proc_return_type :: proc($T: typeid, index: int) -> typeid where type_is_proc(T) --- +type_struct_field_count :: proc($T: typeid) -> int where type_is_struct(T) --- + type_polymorphic_record_parameter_count :: proc($T: typeid) -> typeid --- type_polymorphic_record_parameter_value :: proc($T: typeid, index: int) -> $V --- +type_is_specialized_polymorphic_record :: proc($T: typeid) -> bool --- +type_is_unspecialized_polymorphic_record :: proc($T: typeid) -> bool --- + +type_is_subtype_of :: proc($T, $U: typeid) -> bool --- type_field_index_of :: proc($T: typeid, $name: string) -> uintptr --- type_equal_proc :: proc($T: typeid) -> (equal: proc "contextless" (rawptr, rawptr) -> bool) where type_is_comparable(T) --- type_hasher_proc :: proc($T: typeid) -> (hasher: proc "contextless" (data: rawptr, seed: uintptr) -> uintptr) where type_is_comparable(T) --- +constant_utf16_cstring :: proc($literal: string) -> [^]u16 --- // WASM targets only wasm_memory_grow :: proc(index, delta: uintptr) -> int --- wasm_memory_size :: proc(index: uintptr) -> int --- + +// Darwin targets only +objc_object :: struct{} +objc_selector :: struct{} +objc_class :: struct{} +objc_id :: ^objc_object +objc_SEL :: ^objc_selector +objc_Class :: ^objc_class + +objc_find_selector :: proc($name: string) -> objc_SEL --- +objc_register_selector :: proc($name: string) -> objc_SEL --- +objc_find_class :: proc($name: string) -> objc_Class --- +objc_register_class :: proc($name: string) -> objc_Class --- + // Internal compiler use only __entry_point :: proc() --- \ No newline at end of file From 904f0407f8e19a419dfe313181c2b47216121e11 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 27 Apr 2022 14:53:26 +0100 Subject: [PATCH 58/79] Add `intrinsics.type_is_multi_pointer` --- src/check_builtin.cpp | 2 ++ src/checker_builtin_procs.hpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/check_builtin.cpp b/src/check_builtin.cpp index 6c7972d45..9a5d1c554 100644 --- a/src/check_builtin.cpp +++ b/src/check_builtin.cpp @@ -29,6 +29,7 @@ BuiltinTypeIsProc *builtin_type_is_procs[BuiltinProc__type_simple_boolean_end - is_type_named, is_type_pointer, + is_type_multi_pointer, is_type_array, is_type_enumerated_array, is_type_slice, @@ -3866,6 +3867,7 @@ bool check_builtin_procedure(CheckerContext *c, Operand *operand, Ast *call, i32 case BuiltinProc_type_is_valid_matrix_elements: case BuiltinProc_type_is_named: case BuiltinProc_type_is_pointer: + case BuiltinProc_type_is_multi_pointer: case BuiltinProc_type_is_array: case BuiltinProc_type_is_enumerated_array: case BuiltinProc_type_is_slice: diff --git a/src/checker_builtin_procs.hpp b/src/checker_builtin_procs.hpp index 0f72f01f7..d301cae0c 100644 --- a/src/checker_builtin_procs.hpp +++ b/src/checker_builtin_procs.hpp @@ -158,6 +158,7 @@ BuiltinProc__type_simple_boolean_begin, BuiltinProc_type_is_named, BuiltinProc_type_is_pointer, + BuiltinProc_type_is_multi_pointer, BuiltinProc_type_is_array, BuiltinProc_type_is_enumerated_array, BuiltinProc_type_is_slice, @@ -376,6 +377,7 @@ gb_global BuiltinProc builtin_procs[BuiltinProc_COUNT] = { {STR_LIT("type_is_named"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("type_is_pointer"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, + {STR_LIT("type_is_multi_pointer"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("type_is_array"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("type_is_enumerated_array"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, {STR_LIT("type_is_slice"), 1, false, Expr_Expr, BuiltinProcPkg_intrinsics}, From da0f722aad98afc94de09903a74802f43e6f5961 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Wed, 27 Apr 2022 15:56:45 +0200 Subject: [PATCH 59/79] Move Odin CI test assets over to its own repository. --- tests/core/download_assets.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/core/download_assets.py b/tests/core/download_assets.py index d86f7f1e7..50137f563 100644 --- a/tests/core/download_assets.py +++ b/tests/core/download_assets.py @@ -5,8 +5,9 @@ import sys import os import zipfile +TEST_SUITES = ['PNG', 'XML'] DOWNLOAD_BASE_PATH = "assets/{}" -ASSETS_BASE_URL = "https://raw.githubusercontent.com/Kelimion/compress-odin/master/tests/assets/{}/{}" +ASSETS_BASE_URL = "https://raw.githubusercontent.com/odin-lang/test-assets/master/{}/{}" PNG_IMAGES = [ "basi0g01.png", "basi0g02.png", "basi0g04.png", "basi0g08.png", "basi0g16.png", "basi2c08.png", "basi2c16.png", "basi3p01.png", "basi3p02.png", "basi3p04.png", "basi3p08.png", "basi4a08.png", @@ -73,25 +74,27 @@ def try_download_and_unpack_zip(suite): print("Could not extract ZIP file") return 2 - def main(): - print("Downloading PNG assets") + for suite in TEST_SUITES: + print("Downloading {} assets".format(suite)) - # Make PNG assets path - try: - path = DOWNLOAD_BASE_PATH.format("PNG") - os.makedirs(path) - except FileExistsError: - pass + # Make assets path + try: + path = DOWNLOAD_BASE_PATH.format(suite) + os.makedirs(path) + except FileExistsError: + pass + + # Try downloading and unpacking the assets + r = try_download_and_unpack_zip(suite) + if r is not None: + return r + + # We could fall back on downloading the PNG files individually, but it's slow + print("Done downloading {} assets.".format(suite)) - # Try downloading and unpacking the PNG assets - r = try_download_and_unpack_zip("PNG") - if r is not None: - return r - # We could fall back on downloading the PNG files individually, but it's slow - print("Done downloading PNG assets") return 0 if __name__ == '__main__': From bd73b2845b2e7cadf38c441babeeb9765896b2be Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 27 Apr 2022 15:03:48 +0100 Subject: [PATCH 60/79] Unify default `Sema` and `Atomic_Sema` behaviour --- core/sync/primitives_atomic.odin | 65 +++++++++++++++----------------- core/sync/sema_internal.odin | 42 ++------------------- 2 files changed, 34 insertions(+), 73 deletions(-) diff --git a/core/sync/primitives_atomic.odin b/core/sync/primitives_atomic.odin index 11fff4e60..22771443d 100644 --- a/core/sync/primitives_atomic.odin +++ b/core/sync/primitives_atomic.odin @@ -400,30 +400,28 @@ atomic_cond_broadcast :: proc(c: ^Atomic_Cond) { // // An Atomic_Sema must not be copied after first use Atomic_Sema :: struct { - mutex: Atomic_Mutex, - cond: Atomic_Cond, - count: int, + count: Futex, } atomic_sema_post :: proc(s: ^Atomic_Sema, count := 1) { - atomic_mutex_lock(&s.mutex) - defer atomic_mutex_unlock(&s.mutex) - - s.count += count - atomic_cond_signal(&s.cond) + atomic_add_explicit(&s.count, Futex(count), .Release) + if count == 1 { + futex_signal(&s.count) + } else { + futex_broadcast(&s.count) + } } atomic_sema_wait :: proc(s: ^Atomic_Sema) { - atomic_mutex_lock(&s.mutex) - defer atomic_mutex_unlock(&s.mutex) - - for s.count == 0 { - atomic_cond_wait(&s.cond, &s.mutex) - } - - s.count -= 1 - if s.count > 0 { - atomic_cond_signal(&s.cond) + for { + original_count := atomic_load_explicit(&s.count, .Relaxed) + for original_count == 0 { + futex_wait(&s.count, u32(original_count)) + original_count = s.count + } + if original_count == atomic_compare_exchange_strong_explicit(&s.count, original_count, original_count-1, .Acquire, .Acquire) { + return + } } } @@ -431,25 +429,22 @@ atomic_sema_wait_with_timeout :: proc(s: ^Atomic_Sema, duration: time.Duration) if duration <= 0 { return false } - atomic_mutex_lock(&s.mutex) - defer atomic_mutex_unlock(&s.mutex) - - start := time.tick_now() + for { - for s.count == 0 { - remaining := duration - time.tick_since(start) - if remaining < 0 { - return false + original_count := atomic_load_explicit(&s.count, .Relaxed) + for start := time.tick_now(); original_count == 0; /**/ { + remaining := duration - time.tick_since(start) + if remaining < 0 { + return false + } + + if !futex_wait_with_timeout(&s.count, u32(original_count), remaining) { + return false + } + original_count = s.count } - - if !atomic_cond_wait_with_timeout(&s.cond, &s.mutex, remaining) { - return false + if original_count == atomic_compare_exchange_strong_explicit(&s.count, original_count, original_count-1, .Acquire, .Acquire) { + return true } } - - s.count -= 1 - if s.count > 0 { - atomic_cond_signal(&s.cond) - } - return true } diff --git a/core/sync/sema_internal.odin b/core/sync/sema_internal.odin index e4a3c0bfc..5e2203c34 100644 --- a/core/sync/sema_internal.odin +++ b/core/sync/sema_internal.odin @@ -6,53 +6,19 @@ import "core:time" when #config(ODIN_SYNC_SEMA_USE_FUTEX, true) { _Sema :: struct { - count: Futex, + atomic: Atomic_Sema, } _sema_post :: proc(s: ^Sema, count := 1) { - atomic_add_explicit(&s.impl.count, Futex(count), .Release) - if count == 1 { - futex_signal(&s.impl.count) - } else { - futex_broadcast(&s.impl.count) - } + atomic_sema_post(&s.impl.atomic, count) } _sema_wait :: proc(s: ^Sema) { - for { - original_count := atomic_load_explicit(&s.impl.count, .Relaxed) - for original_count == 0 { - futex_wait(&s.impl.count, u32(original_count)) - original_count = s.impl.count - } - if original_count == atomic_compare_exchange_strong_explicit(&s.impl.count, original_count, original_count-1, .Acquire, .Acquire) { - return - } - } + atomic_sema_wait(&s.impl.atomic) } _sema_wait_with_timeout :: proc(s: ^Sema, duration: time.Duration) -> bool { - if duration <= 0 { - return false - } - for { - - original_count := atomic_load_explicit(&s.impl.count, .Relaxed) - for start := time.tick_now(); original_count == 0; /**/ { - remaining := duration - time.tick_since(start) - if remaining < 0 { - return false - } - - if !futex_wait_with_timeout(&s.impl.count, u32(original_count), remaining) { - return false - } - original_count = s.impl.count - } - if original_count == atomic_compare_exchange_strong_explicit(&s.impl.count, original_count, original_count-1, .Acquire, .Acquire) { - return true - } - } + return atomic_sema_wait_with_timeout(&s.impl.atomic, duration) } } else { _Sema :: struct { From cd13dedb36799d1b6e2f674d95fd820babff5e16 Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Wed, 27 Apr 2022 20:06:43 +0200 Subject: [PATCH 61/79] D3D12 binding fixes: Added dxgi.IID versions of all the UUIDs in d3d12.odin so that the IIDs can be used to fetch interfaces properly. Also fixed an issue where GetDesc, GetCPUDescriptorHandleForHeapStart and GetGPUDescriptorHandleForHeapStart had the wrong signature due to an old D3D12 header bug, more info: https://stackoverflow.com/questions/34118929/getcpudescriptorhandleforheapstart-stack-corruption --- vendor/directx/d3d12/d3d12.odin | 184 +++++++++++++++++++++----------- 1 file changed, 120 insertions(+), 64 deletions(-) diff --git a/vendor/directx/d3d12/d3d12.odin b/vendor/directx/d3d12/d3d12.odin index f3885ed63..739606249 100644 --- a/vendor/directx/d3d12/d3d12.odin +++ b/vendor/directx/d3d12/d3d12.odin @@ -189,7 +189,8 @@ SRV_DIMENSION :: enum i32 { PFN_DESTRUCTION_CALLBACK :: #type proc "c" (a0: rawptr) -ID3DDestructionNotifier_UUID :: "a06eb39a-50da-425b-8c31-4eecd6c270f3" +ID3DDestructionNotifier_UUID_STRING :: "a06eb39a-50da-425b-8c31-4eecd6c270f3" +ID3DDestructionNotifier_UUID := &IID{0xa06eb39a, 0x50da, 0x425b, {0x8c, 0x31, 0x4e, 0xec, 0xd6, 0xc2, 0x70, 0xf3}} ID3DDestructionNotifier :: struct #raw_union { #subtype iunknown: IUnknown, using id3ddestructionnotifier_vtable: ^ID3DDestructionNotifier_VTable, @@ -658,7 +659,8 @@ RASTERIZER_DESC :: struct { } -IObject_UUID :: "c4fec28f-7966-4e95-9f94-f431cb56c3b8" +IObject_UUID_STRING :: "c4fec28f-7966-4e95-9f94-f431cb56c3b8" +IObject_UUID := &IID{0xc4fec28f, 0x7966, 0x4e95, {0x9f, 0x94, 0xf4, 0x31, 0xcb, 0x56, 0xc3, 0xb8}} IObject :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12object_vtable: ^IObject_VTable, @@ -672,7 +674,8 @@ IObject_VTable :: struct { } -IDeviceChild_UUID :: "905db94b-a00c-4140-9df5-2b64ca9ea357" +IDeviceChild_UUID_STRING :: "905db94b-a00c-4140-9df5-2b64ca9ea357" +IDeviceChild_UUID := &IID{0x905db94b, 0xa00c, 0x4140, {0x9d, 0xf5, 0x2b, 0x64, 0xca, 0x9e, 0xa3, 0x57}} IDeviceChild :: struct #raw_union { #subtype id3d12object: IObject, using id3d12devicechild_vtable: ^IDeviceChild_VTable, @@ -683,7 +686,8 @@ IDeviceChild_VTable :: struct { } -IRootSignature_UUID :: "c54a6b66-72df-4ee8-8be5-a946a1429214" +IRootSignature_UUID_STRING :: "c54a6b66-72df-4ee8-8be5-a946a1429214" +IRootSignature_UUID := &IID{0xc54a6b66, 0x72df, 0x4ee8, {0x8b, 0xe5, 0xa9, 0x46, 0xa1, 0x42, 0x92, 0x14}} IRootSignature :: struct { using id3d12devicechild: IDeviceChild, } @@ -2058,7 +2062,8 @@ VERSIONED_ROOT_SIGNATURE_DESC :: struct { } -IRootSignatureDeserializer_UUID :: "34AB647B-3CC8-46AC-841B-C0965645C046" +IRootSignatureDeserializer_UUID_STRING :: "34AB647B-3CC8-46AC-841B-C0965645C046" +IRootSignatureDeserializer_UUID := &IID{0x34AB647B, 0x3CC8, 0x46AC, {0x84, 0x1B, 0xC0, 0x96, 0x56, 0x45, 0xC0, 0x46}} IRootSignatureDeserializer :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12rootsignaturedeserializer_vtable: ^IRootSignatureDeserializer_VTable, @@ -2069,7 +2074,8 @@ IRootSignatureDeserializer_VTable :: struct { } -IVersionedRootSignatureDeserializer_UUID :: "7F91CE67-090C-4BB7-B78E-ED8FF2E31DA0" +IVersionedRootSignatureDeserializer_UUID_STRING :: "7F91CE67-090C-4BB7-B78E-ED8FF2E31DA0" +IVersionedRootSignatureDeserializer_UUID := &IID{0x7F91CE67, 0x090C, 0x4BB7, {0xB7, 0x8E, 0xED, 0x8F, 0xF2, 0xE3, 0x1D, 0xA0}} IVersionedRootSignatureDeserializer :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12versionedrootsignaturedeserializer_vtable: ^IVersionedRootSignatureDeserializer_VTable, @@ -2236,13 +2242,15 @@ COMMAND_SIGNATURE_DESC :: struct { } -IPageable_UUID :: "63ee58fb-1268-4835-86da-f008ce62f0d6" +IPageable_UUID_STRING :: "63ee58fb-1268-4835-86da-f008ce62f0d6" +IPageable_UUID := &IID{0x63ee58fb, 0x1268, 0x4835, {0x86, 0xda, 0xf0, 0x08, 0xce, 0x62, 0xf0, 0xd6}} IPageable :: struct { using id3d12devicechild: IDeviceChild, } -IHeap_UUID :: "6b3b2502-6e51-45b3-90ee-9884265e8df3" +IHeap_UUID_STRING :: "6b3b2502-6e51-45b3-90ee-9884265e8df3" +IHeap_UUID := &IID{0x6b3b2502, 0x6e51, 0x45b3, {0x90, 0xee, 0x98, 0x84, 0x26, 0x5e, 0x8d, 0xf3}} IHeap :: struct #raw_union { #subtype id3d12pageable: IPageable, using id3d12heap_vtable: ^IHeap_VTable, @@ -2253,7 +2261,8 @@ IHeap_VTable :: struct { } -IResource_UUID :: "696442be-a72e-4059-bc79-5b5c98040fad" +IResource_UUID_STRING :: "696442be-a72e-4059-bc79-5b5c98040fad" +IResource_UUID := &IID{0x696442be, 0xa72e, 0x4059, {0xbc, 0x79, 0x5b, 0x5c, 0x98, 0x04, 0x0f, 0xad}} IResource :: struct #raw_union { #subtype id3d12pageable: IPageable, using id3d12resource_vtable: ^IResource_VTable, @@ -2270,7 +2279,8 @@ IResource_VTable :: struct { } -ICommandAllocator_UUID :: "6102dee4-af59-4b09-b999-b44d73f09b24" +ICommandAllocator_UUID_STRING :: "6102dee4-af59-4b09-b999-b44d73f09b24" +ICommandAllocator_UUID := &IID{0x6102dee4, 0xaf59, 0x4b09, {0xb9, 0x99, 0xb4, 0x4d, 0x73, 0xf0, 0x9b, 0x24}} ICommandAllocator :: struct #raw_union { #subtype id3d12pageable: IPageable, using id3d12commandallocator_vtable: ^ICommandAllocator_VTable, @@ -2281,7 +2291,8 @@ ICommandAllocator_VTable :: struct { } -IFence_UUID :: "0a753dcf-c4d8-4b91-adf6-be5a60d95a76" +IFence_UUID_STRING :: "0a753dcf-c4d8-4b91-adf6-be5a60d95a76" +IFence_UUID := &IID {0x0a753dcf, 0xc4d8, 0x4b91, {0xad, 0xf6, 0xbe, 0x5a, 0x60, 0xd9, 0x5a, 0x76}} IFence :: struct #raw_union { #subtype id3d12pageable: IPageable, using id3d12fence_vtable: ^IFence_VTable, @@ -2294,7 +2305,8 @@ IFence_VTable :: struct { } -IFence1_UUID :: "433685fe-e22b-4ca0-a8db-b5b4f4dd0e4a" +IFence1_UUID_STRING :: "433685fe-e22b-4ca0-a8db-b5b4f4dd0e4a" +IFence1_UUID := &IID{0x433685fe, 0xe22b, 0x4ca0, {0xa8, 0xdb, 0xb5, 0xb4, 0xf4, 0xdd, 0x0e, 0x4a}} IFence1 :: struct #raw_union { #subtype id3d12fence: IFence, using id3d12fence1_vtable: ^IFence1_VTable, @@ -2305,7 +2317,8 @@ IFence1_VTable :: struct { } -IPipelineState_UUID :: "765a30f3-f624-4c6f-a828-ace948622445" +IPipelineState_UUID_STRING :: "765a30f3-f624-4c6f-a828-ace948622445" +IPipelineState_UUID := &IID{0x765a30f3, 0xf624, 0x4c6f, {0xa8, 0x28, 0xac, 0xe9, 0x48, 0x62, 0x24, 0x45}} IPipelineState :: struct #raw_union { #subtype id3d12pageable: IPageable, using id3d12pipelinestate_vtable: ^IPipelineState_VTable, @@ -2316,32 +2329,35 @@ IPipelineState_VTable :: struct { } -IDescriptorHeap_UUID :: "8efb471d-616c-4f49-90f7-127bb763fa51" +IDescriptorHeap_UUID_STRING :: "8efb471d-616c-4f49-90f7-127bb763fa51" +IDescriptorHeap_UUID := &IID{0x8efb471d, 0x616c, 0x4f49, { 0x90, 0xf7, 0x12, 0x7b, 0xb7, 0x63, 0xfa, 0x51}} IDescriptorHeap :: struct #raw_union { #subtype id3d12pageable: IPageable, using id3d12descriptorheap_vtable: ^IDescriptorHeap_VTable, } IDescriptorHeap_VTable :: struct { using id3d12devicechild_vtable: IDeviceChild_VTable, - GetDesc: proc "stdcall" (this: ^IDescriptorHeap) -> DESCRIPTOR_HEAP_DESC, - GetCPUDescriptorHandleForHeapStart: proc "stdcall" (this: ^IDescriptorHeap) -> CPU_DESCRIPTOR_HANDLE, - GetGPUDescriptorHandleForHeapStart: proc "stdcall" (this: ^IDescriptorHeap) -> GPU_DESCRIPTOR_HANDLE, -} + GetDesc: proc "stdcall" (this: ^IDescriptorHeap, desc: ^DESCRIPTOR_HEAP_DESC), + GetCPUDescriptorHandleForHeapStart: proc "stdcall" (this: ^IDescriptorHeap, handle: ^CPU_DESCRIPTOR_HANDLE), + GetGPUDescriptorHandleForHeapStart: proc "stdcall" (this: ^IDescriptorHeap, handle: ^GPU_DESCRIPTOR_HANDLE), +} - -IQueryHeap_UUID :: "0d9658ae-ed45-469e-a61d-970ec583cab4" +IQueryHeap_UUID_STRING :: "0d9658ae-ed45-469e-a61d-970ec583cab4" +IQueryHeap_UUID := &IID{0x0d9658ae, 0xed45, 0x469e, {0xa6, 0x1d, 0x97, 0x0e, 0xc5, 0x83, 0xca, 0xb4}} IQueryHeap :: struct { #subtype id3d12pageable: IPageable, } -ICommandSignature_UUID :: "c36a797c-ec80-4f0a-8985-a7b2475082d1" +ICommandSignature_UUID_STRING :: "c36a797c-ec80-4f0a-8985-a7b2475082d1" +ICommandSignature_UUID := &IID{0xc36a797c, 0xec80, 0x4f0a, {0x89, 0x85, 0xa7, 0xb2, 0x47, 0x50, 0x82, 0xd1}} ICommandSignature :: struct { #subtype id3d12pageable: IPageable, } -ICommandList_UUID :: "7116d91c-e7e4-47ce-b8c6-ec8168f437e5" +ICommandList_UUID_STRING :: "7116d91c-e7e4-47ce-b8c6-ec8168f437e5" +ICommandList_UUID := &IID {0x7116d91c, 0xe7e4, 0x47ce, {0xb8, 0xc6, 0xec, 0x81, 0x68, 0xf4, 0x37, 0xe5}} ICommandList :: struct #raw_union { #subtype id3d12devicechild: IDeviceChild, using id3d12commandlist_vtable: ^ICommandList_VTable, @@ -2352,7 +2368,8 @@ ICommandList_VTable :: struct { } -IGraphicsCommandList_UUID :: "5b160d0f-ac1b-4185-8ba8-b3ae42a5a455" +IGraphicsCommandList_UUID_STRING :: "5b160d0f-ac1b-4185-8ba8-b3ae42a5a455" +IGraphicsCommandList_UUID := &IID{0x5b160d0f, 0xac1b, 0x4185, {0x8b, 0xa8, 0xb3, 0xae, 0x42, 0xa5, 0xa4, 0x55}} IGraphicsCommandList :: struct #raw_union { #subtype id3d12commandlist: ICommandList, using id3d12graphicscommandlist_vtable: ^IGraphicsCommandList_VTable, @@ -2413,7 +2430,8 @@ IGraphicsCommandList_VTable :: struct { } -IGraphicsCommandList1_UUID :: "553103fb-1fe7-4557-bb38-946d7d0e7ca7" +IGraphicsCommandList1_UUID_STRING :: "553103fb-1fe7-4557-bb38-946d7d0e7ca7" +IGraphicsCommandList1_UUID := &IID{0x553103fb, 0x1fe7, 0x4557, {0xbb, 0x38, 0x94, 0x6d, 0x7d, 0x0e, 0x7c, 0xa7}} IGraphicsCommandList1 :: struct #raw_union { #subtype id3d12graphicscommandlist: IGraphicsCommandList, using id3d12graphicscommandlist1_vtable: ^IGraphicsCommandList1_VTable, @@ -2440,7 +2458,8 @@ WRITEBUFFERIMMEDIATE_MODE :: enum i32 { } -IGraphicsCommandList2_UUID :: "38C3E585-FF17-412C-9150-4FC6F9D72A28" +IGraphicsCommandList2_UUID_STRING :: "38C3E585-FF17-412C-9150-4FC6F9D72A28" +IGraphicsCommandList2_UUID := &IID{0x38C3E585, 0xFF17, 0x412C, {0x91, 0x50, 0x4F, 0xC6, 0xF9, 0xD7, 0x2A, 0x28}} IGraphicsCommandList2 :: struct #raw_union { #subtype id3d12graphicscommandlist1: IGraphicsCommandList1, using id3d12graphicscommandlist2_vtable: ^IGraphicsCommandList2_VTable, @@ -2451,7 +2470,8 @@ IGraphicsCommandList2_VTable :: struct { } -ICommandQueue_UUID :: "0ec870a6-5d7e-4c22-8cfc-5baae07616ed" +ICommandQueue_UUID_STRING :: "0ec870a6-5d7e-4c22-8cfc-5baae07616ed" +ICommandQueue_UUID := &IID{0x0ec870a6, 0x5d7e, 0x4c22, { 0x8c, 0xfc, 0x5b, 0xaa, 0xe0, 0x76, 0x16, 0xed}} ICommandQueue :: struct #raw_union { #subtype id3d12pageable: IPageable, using id3d12commandqueue_vtable: ^ICommandQueue_VTable, @@ -2472,7 +2492,8 @@ ICommandQueue_VTable :: struct { } -IDevice_UUID :: "189819f1-1db6-4b57-be54-1821339b85f7" +IDevice_UUID_STRING :: "189819f1-1db6-4b57-be54-1821339b85f7" +IDevice_UUID := &IID{0x189819f1, 0x1db6, 0x4b57, { 0xbe, 0x54, 0x18, 0x21, 0x33, 0x9b, 0x85, 0xf7}} IDevice :: struct #raw_union { #subtype id3d12object: IObject, using id3d12device_vtable: ^IDevice_VTable, @@ -2519,7 +2540,8 @@ IDevice_VTable :: struct { } -IPipelineLibrary_UUID :: "c64226a8-9201-46af-b4cc-53fb9ff7414f" +IPipelineLibrary_UUID_STRING :: "c64226a8-9201-46af-b4cc-53fb9ff7414f" +IPipelineLibrary_UUID := &IID{0xc64226a8, 0x9201, 0x46af, {0xb4, 0xcc, 0x53, 0xfb, 0x9f, 0xf7, 0x41, 0x4f}} IPipelineLibrary :: struct #raw_union { #subtype id3d12devicechild: IDeviceChild, using id3d12pipelinelibrary_vtable: ^IPipelineLibrary_VTable, @@ -2534,7 +2556,8 @@ IPipelineLibrary_VTable :: struct { } -IPipelineLibrary1_UUID :: "80eabf42-2568-4e5e-bd82-c37f86961dc3" +IPipelineLibrary1_UUID_STRING :: "80eabf42-2568-4e5e-bd82-c37f86961dc3" +IPipelineLibrary1_UUID := &IID{0x80eabf42, 0x2568, 0x4e5e, {0xbd, 0x82, 0xc3, 0x7f, 0x86, 0x96, 0x1d, 0xc3}} IPipelineLibrary1 :: struct #raw_union { #subtype id3d12pipelinelibrary: IPipelineLibrary, using id3d12pipelinelibrary1_vtable: ^IPipelineLibrary1_VTable, @@ -2559,7 +2582,8 @@ RESIDENCY_PRIORITY :: enum i32 { } -IDevice1_UUID :: "77acce80-638e-4e65-8895-c1f23386863e" +IDevice1_UUID_STRING :: "77acce80-638e-4e65-8895-c1f23386863e" +IDevice1_UUID := &IID{0x77acce80, 0x638e, 0x4e65, {0x88, 0x95, 0xc1, 0xf2, 0x33, 0x86, 0x86, 0x3e}} IDevice1 :: struct #raw_union { #subtype id3d12device: IDevice, using id3d12device1_vtable: ^IDevice1_VTable, @@ -2572,7 +2596,8 @@ IDevice1_VTable :: struct { } -IDevice2_UUID :: "30baa41e-b15b-475c-a0bb-1af5c5b64328" +IDevice2_UUID_STRING :: "30baa41e-b15b-475c-a0bb-1af5c5b64328" +IDevice2_UUID := &IID{0x30baa41e, 0xb15b, 0x475c, {0xa0, 0xbb, 0x1a, 0xf5, 0xc5, 0xb6, 0x43, 0x28}} IDevice2 :: struct #raw_union { #subtype id3d12device1: IDevice1, using id3d12device2_vtable: ^IDevice2_VTable, @@ -2588,7 +2613,8 @@ RESIDENCY_FLAGS :: enum u32 { // TODO: make bit_set } -IDevice3_UUID :: "81dadc15-2bad-4392-93c5-101345c4aa98" +IDevice3_UUID_STRING :: "81dadc15-2bad-4392-93c5-101345c4aa98" +IDevice3_UUID := &IID{0x81dadc15, 0x2bad, 0x4392, {0x93, 0xc5, 0x10, 0x13, 0x45, 0xc4, 0xaa, 0x98}} IDevice3 :: struct #raw_union { #subtype id3d12device2: IDevice2, using id3d12device3_vtable: ^IDevice3_VTable, @@ -2618,7 +2644,8 @@ PROTECTED_SESSION_STATUS :: enum i32 { } -IProtectedSession_UUID :: "A1533D18-0AC1-4084-85B9-89A96116806B" +IProtectedSession_UUID_STRING :: "A1533D18-0AC1-4084-85B9-89A96116806B" +IProtectedSession_UUID := &IID{0xA1533D18, 0x0AC1, 0x4084, {0x85, 0xB9, 0x89, 0xA9, 0x61, 0x16, 0x80, 0x6B}} IProtectedSession :: struct #raw_union { #subtype id3d12devicechild: IDeviceChild, using id3d12protectedsession_vtable: ^IProtectedSession_VTable, @@ -2649,7 +2676,8 @@ PROTECTED_RESOURCE_SESSION_DESC :: struct { } -IProtectedResourceSession_UUID :: "6CD696F4-F289-40CC-8091-5A6C0A099C3D" +IProtectedResourceSession_UUID_STRING :: "6CD696F4-F289-40CC-8091-5A6C0A099C3D" +IProtectedResourceSession_UUID := &IID{0x6CD696F4, 0xF289, 0x40CC, {0x80, 0x91, 0x5A, 0x6C, 0x0A, 0x09, 0x9C, 0x3D}} IProtectedResourceSession :: struct #raw_union { #subtype id3d12protectedsession: IProtectedSession, using id3d12protectedresourcesession_vtable: ^IProtectedResourceSession_VTable, @@ -2660,7 +2688,8 @@ IProtectedResourceSession_VTable :: struct { } -IDevice4_UUID :: "e865df17-a9ee-46f9-a463-3098315aa2e5" +IDevice4_UUID_STRING :: "e865df17-a9ee-46f9-a463-3098315aa2e5" +IDevice4_UUID := &IID{0xe865df17, 0xa9ee, 0x46f9, {0xa4, 0x63, 0x30, 0x98, 0x31, 0x5a, 0xa2, 0xe5}} IDevice4 :: struct #raw_union { #subtype id3d12device3: IDevice3, using id3d12device4_vtable: ^IDevice4_VTable, @@ -2681,7 +2710,8 @@ LIFETIME_STATE :: enum i32 { } -ILifetimeOwner_UUID :: "e667af9f-cd56-4f46-83ce-032e595d70a8" +ILifetimeOwner_UUID_STRING :: "e667af9f-cd56-4f46-83ce-032e595d70a8" +ILifetimeOwner_UUID := &IID{0xe667af9f, 0xcd56, 0x4f46, {0x83, 0xce, 0x03, 0x2e, 0x59, 0x5d, 0x70, 0xa8}} ILifetimeOwner :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12lifetimeowner_vtable: ^ILifetimeOwner_VTable, @@ -2692,7 +2722,8 @@ ILifetimeOwner_VTable :: struct { } -ISwapChainAssistant_UUID :: "f1df64b6-57fd-49cd-8807-c0eb88b45c8f" +ISwapChainAssistant_UUID_STRING :: "f1df64b6-57fd-49cd-8807-c0eb88b45c8f" +ISwapChainAssistant_UUID := &IID{0xf1df64b6, 0x57fd, 0x49cd, {0x88, 0x07, 0xc0, 0xeb, 0x88, 0xb4, 0x5c, 0x8f}} ISwapChainAssistant :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12swapchainassistant_vtable: ^ISwapChainAssistant_VTable, @@ -2706,7 +2737,8 @@ ISwapChainAssistant_VTable :: struct { } -ILifetimeTracker_UUID :: "3fd03d36-4eb1-424a-a582-494ecb8ba813" +ILifetimeTracker_UUID_STRING :: "3fd03d36-4eb1-424a-a582-494ecb8ba813" +ILifetimeTracker_UUID := &IID{0x3fd03d36, 0x4eb1, 0x424a, {0xa5, 0x82, 0x49, 0x4e, 0xcb, 0x8b, 0xa8, 0x13}} ILifetimeTracker :: struct #raw_union { #subtype id3d12devicechild: IDeviceChild, using id3d12lifetimetracker_vtable: ^ILifetimeTracker_VTable, @@ -2772,13 +2804,15 @@ META_COMMAND_DESC :: struct { } -IStateObject_UUID :: "47016943-fca8-4594-93ea-af258b55346d" +IStateObject_UUID_STRING :: "47016943-fca8-4594-93ea-af258b55346d" +IStateObject_UUID := &IID{0x47016943, 0xfca8, 0x4594, {0x93, 0xea, 0xaf, 0x25, 0x8b, 0x55, 0x34, 0x6d}} IStateObject :: struct #raw_union { #subtype id3d12pageable: IPageable, } -IStateObjectProperties_UUID :: "de5fa827-9bf9-4f26-89ff-d7f56fde3860" +IStateObjectProperties_UUID_STRING :: "de5fa827-9bf9-4f26-89ff-d7f56fde3860" +IStateObjectProperties_IID := &IID{0xde5fa827, 0x9bf9, 0x4f26, {0x89, 0xff, 0xd7, 0xf5, 0x6f, 0xde, 0x38, 0x60}} IStateObjectProperties :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12stateobjectproperties_vtable: ^IStateObjectProperties_VTable, @@ -3119,7 +3153,8 @@ HIT_KIND :: enum i32 { } -IDevice5_UUID :: "8b4f173b-2fea-4b80-8f58-4307191ab95d" +IDevice5_UUID_STRING :: "8b4f173b-2fea-4b80-8f58-4307191ab95d" +IDevice5_UUID := &IID{0x8b4f173b, 0x2fea, 0x4b80, {0x8f, 0x58, 0x43, 0x07, 0x19, 0x1a, 0xb9, 0x5d}} IDevice5 :: struct #raw_union { #subtype id3d12device4: IDevice4, using id3d12device5_vtable: ^IDevice5_VTable, @@ -3325,7 +3360,8 @@ VERSIONED_DEVICE_REMOVED_EXTENDED_DATA :: struct { } -IDeviceRemovedExtendedDataSettings_UUID :: "82BC481C-6B9B-4030-AEDB-7EE3D1DF1E63" +IDeviceRemovedExtendedDataSettings_UUID_SRING :: "82BC481C-6B9B-4030-AEDB-7EE3D1DF1E63" +IDeviceRemovedExtendedDataSettings_UUID := &IID{0x82BC481C, 0x6B9B, 0x4030, {0xAE, 0xDB, 0x7E, 0xE3, 0xD1, 0xDF, 0x1E, 0x63}} IDeviceRemovedExtendedDataSettings :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12deviceremovedextendeddatasettings_vtable: ^IDeviceRemovedExtendedDataSettings_VTable, @@ -3338,7 +3374,8 @@ IDeviceRemovedExtendedDataSettings_VTable :: struct { } -IDeviceRemovedExtendedDataSettings1_UUID :: "DBD5AE51-3317-4F0A-ADF9-1D7CEDCAAE0B" +IDeviceRemovedExtendedDataSettings1_UUID_STRING :: "DBD5AE51-3317-4F0A-ADF9-1D7CEDCAAE0B" +IDeviceRemovedExtendedDataSettings1_UUID := &IID{0xDBD5AE51, 0x3317, 0x4F0A, {0xAD, 0xF9, 0x1D, 0x7C, 0xED, 0xCA, 0xAE, 0x0B}} IDeviceRemovedExtendedDataSettings1 :: struct #raw_union { #subtype id3d12deviceremovedextendeddatasettings: IDeviceRemovedExtendedDataSettings, using id3d12deviceremovedextendeddatasettings1_vtable: ^IDeviceRemovedExtendedDataSettings1_VTable, @@ -3349,7 +3386,8 @@ IDeviceRemovedExtendedDataSettings1_VTable :: struct { } -IDeviceRemovedExtendedData_UUID :: "98931D33-5AE8-4791-AA3C-1A73A2934E71" +IDeviceRemovedExtendedData_UUID_STRING :: "98931D33-5AE8-4791-AA3C-1A73A2934E71" +IDeviceRemovedExtendedData_UUID := &IID{0x98931D33, 0x5AE8, 0x4791, {0xAA, 0x3C, 0x1A, 0x73, 0xA2, 0x93, 0x4E, 0x71}} IDeviceRemovedExtendedData :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12deviceremovedextendeddata_vtable: ^IDeviceRemovedExtendedData_VTable, @@ -3361,7 +3399,8 @@ IDeviceRemovedExtendedData_VTable :: struct { } -IDeviceRemovedExtendedData1_UUID :: "9727A022-CF1D-4DDA-9EBA-EFFA653FC506" +IDeviceRemovedExtendedData1_UUID_STRING :: "9727A022-CF1D-4DDA-9EBA-EFFA653FC506" +IDeviceRemovedExtendedData1_UUID := &IID{0x9727A022, 0xCF1D, 0x4DDA, {0x9E, 0xBA, 0xEF, 0xFA, 0x65, 0x3F, 0xC5, 0x06}} IDeviceRemovedExtendedData1 :: struct #raw_union { #subtype id3d12deviceremovedextendeddata: IDeviceRemovedExtendedData, using id3d12deviceremovedextendeddata1_vtable: ^IDeviceRemovedExtendedData1_VTable, @@ -3387,7 +3426,8 @@ MEASUREMENTS_ACTION :: enum i32 { } -IDevice6_UUID :: "c70b221b-40e4-4a17-89af-025a0727a6dc" +IDevice6_UUID_STRING :: "c70b221b-40e4-4a17-89af-025a0727a6dc" +IDevice6_UUID := &IID{0xc70b221b, 0x40e4, 0x4a17, {0x89, 0xaf, 0x02, 0x5a, 0x07, 0x27, 0xa6, 0xdc}} IDevice6 :: struct #raw_union { #subtype id3d12device5: IDevice5, using id3d12device6_vtable: ^IDevice6_VTable, @@ -3415,7 +3455,8 @@ PROTECTED_RESOURCE_SESSION_DESC1 :: struct { } -IProtectedResourceSession1_UUID :: "D6F12DD6-76FB-406E-8961-4296EEFC0409" +IProtectedResourceSession1_UUID_STRING :: "D6F12DD6-76FB-406E-8961-4296EEFC0409" +IProtectedResourceSession1_UUID := &IID{0xD6F12DD6, 0x76FB, 0x406E, {0x89, 0x61, 0x42, 0x96, 0xEE, 0xFC, 0x04, 0x09}} IProtectedResourceSession1 :: struct #raw_union { #subtype id3d12protectedresourcesession: IProtectedResourceSession, using id3d12protectedresourcesession1_vtable: ^IProtectedResourceSession1_VTable, @@ -3426,7 +3467,8 @@ IProtectedResourceSession1_VTable :: struct { } -IDevice7_UUID :: "5c014b53-68a1-4b9b-8bd1-dd6046b9358b" +IDevice7_UUID_STRING :: "5c014b53-68a1-4b9b-8bd1-dd6046b9358b" +IDevice7_UUID := &IID{0x5c014b53, 0x68a1, 0x4b9b, {0x8b, 0xd1, 0xdd, 0x60, 0x46, 0xb9, 0x35, 0x8b}} IDevice7 :: struct #raw_union { #subtype id3d12device6: IDevice6, using id3d12device7_vtable: ^IDevice7_VTable, @@ -3438,7 +3480,8 @@ IDevice7_VTable :: struct { } -IDevice8_UUID :: "9218E6BB-F944-4F7E-A75C-B1B2C7B701F3" +IDevice8_UUID_STRING :: "9218E6BB-F944-4F7E-A75C-B1B2C7B701F3" +IDevice8_UUID := &IID{0x9218E6BB, 0xF944, 0x4F7E, {0xA7, 0x5C, 0xB1, 0xB2, 0xC7, 0xB7, 0x01, 0xF3}} IDevice8 :: struct #raw_union { #subtype id3d12device7: IDevice7, using id3d12device8_vtable: ^IDevice8_VTable, @@ -3453,7 +3496,8 @@ IDevice8_VTable :: struct { } -IResource1_UUID :: "9D5E227A-4430-4161-88B3-3ECA6BB16E19" +IResource1_UUID_STRING :: "9D5E227A-4430-4161-88B3-3ECA6BB16E19" +IResource1_UUID := &IID{0x9D5E227A, 0x4430, 0x4161, {0x88, 0xB3, 0x3E, 0xCA, 0x6B, 0xB1, 0x6E, 0x19}} IResource1 :: struct #raw_union { #subtype id3d12resource: IResource, using id3d12resource1_vtable: ^IResource1_VTable, @@ -3464,7 +3508,8 @@ IResource1_VTable :: struct { } -IResource2_UUID :: "BE36EC3B-EA85-4AEB-A45A-E9D76404A495" +IResource2_UUID_STRING :: "BE36EC3B-EA85-4AEB-A45A-E9D76404A495" +IResource2_UUID := &IID{0xBE36EC3B, 0xEA85, 0x4AEB, {0xA4, 0x5A, 0xE9, 0xD7, 0x64, 0x04, 0xA4, 0x95}} IResource2 :: struct #raw_union { #subtype id3d12resource1: IResource1, using id3d12resource2_vtable: ^IResource2_VTable, @@ -3475,7 +3520,8 @@ IResource2_VTable :: struct { } -IHeap1_UUID :: "572F7389-2168-49E3-9693-D6DF5871BF6D" +IHeap1_UUID_STRING :: "572F7389-2168-49E3-9693-D6DF5871BF6D" +IHeap1_UUID := &IID{0x572F7389, 0x2168, 0x49E3, {0x96, 0x93, 0xD6, 0xDF, 0x58, 0x71, 0xBF, 0x6D}} IHeap1 :: struct #raw_union { #subtype id3d12heap: IHeap, using id3d12heap1_vtable: ^IHeap1_VTable, @@ -3486,7 +3532,8 @@ IHeap1_VTable :: struct { } -IGraphicsCommandList3_UUID :: "6FDA83A7-B84C-4E38-9AC8-C7BD22016B3D" +IGraphicsCommandList3_UUID_STRING :: "6FDA83A7-B84C-4E38-9AC8-C7BD22016B3D" +IGraphicsCommandList3_UUID := &IID{0x6FDA83A7, 0xB84C, 0x4E38, {0x9A, 0xC8, 0xC7, 0xBD, 0x22, 0x01, 0x6B, 0x3D}} IGraphicsCommandList3 :: struct #raw_union { #subtype id3d12graphicscommandlist2: IGraphicsCommandList2, using id3d12graphicscommandlist3_vtable: ^IGraphicsCommandList3_VTable, @@ -3568,7 +3615,8 @@ RENDER_PASS_FLAGS :: enum u32 { // TODO: make bit_set } -IMetaCommand_UUID :: "DBB84C27-36CE-4FC9-B801-F048C46AC570" +IMetaCommand_UUID_STRING :: "DBB84C27-36CE-4FC9-B801-F048C46AC570" +IMetaCommand_UUID := &IID{0xDBB84C27, 0x36CE, 0x4FC9, {0xB8, 0x01, 0xF0, 0x48, 0xC4, 0x6A, 0xC5, 0x70}} IMetaCommand :: struct #raw_union { #subtype id3d12pageable: IPageable, using id3d12metacommand_vtable: ^IMetaCommand_VTable, @@ -3589,7 +3637,8 @@ DISPATCH_RAYS_DESC :: struct { } -IGraphicsCommandList4_UUID :: "8754318e-d3a9-4541-98cf-645b50dc4874" +IGraphicsCommandList4_UUID_STRING :: "8754318e-d3a9-4541-98cf-645b50dc4874" +IGraphicsCommandList4_UUID := &IID{0x8754318e, 0xd3a9, 0x4541, {0x98, 0xcf, 0x64, 0x5b, 0x50, 0xdc, 0x48, 0x74}} IGraphicsCommandList4 :: struct #raw_union { #subtype id3d12graphicscommandlist3: IGraphicsCommandList3, using id3d12graphicscommandlist4_vtable: ^IGraphicsCommandList4_VTable, @@ -3608,7 +3657,8 @@ IGraphicsCommandList4_VTable :: struct { } -ITools_UUID :: "7071e1f0-e84b-4b33-974f-12fa49de65c5" +ITools_UUID_STRING :: "7071e1f0-e84b-4b33-974f-12fa49de65c5" +ITools_UUID := &IID{0x7071e1f0, 0xe84b, 0x4b33, {0x97, 0x4f, 0x12, 0xfa, 0x49, 0xde, 0x65, 0xc5}} ITools :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12tools_vtable: ^ITools_VTable, @@ -3632,7 +3682,8 @@ MEMCPY_DEST :: struct { } -IDebug_UUID :: "344488b7-6846-474b-b989-f027448245e0" +IDebug_UUID_STRING :: "344488b7-6846-474b-b989-f027448245e0" +IDebug_UUID := &IID{0x344488b7, 0x6846, 0x474b, {0xb9, 0x89, 0xf0, 0x27, 0x44, 0x82, 0x45, 0xe0}} IDebug :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12debug_vtable: ^IDebug_VTable, @@ -3648,7 +3699,8 @@ GPU_BASED_VALIDATION_FLAGS :: enum u32 { // TODO: make bit_set } -IDebug1_UUID :: "affaa4ca-63fe-4d8e-b8ad-159000af4304" +IDebug1_UUID_STRING :: "affaa4ca-63fe-4d8e-b8ad-159000af4304" +IDebug1_UUID := &IID{0xaffaa4ca, 0x63fe, 0x4d8e, {0xb8, 0xad, 0x15, 0x90, 0x00, 0xaf, 0x43, 0x04}} IDebug1 :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12debug1_vtable: ^IDebug1_VTable, @@ -3672,7 +3724,8 @@ IDebug2_VTable :: struct { } -IDebug3_UUID :: "5cf4e58f-f671-4ff1-a542-3686e3d153d1" +IDebug3_UUID_STRING :: "5cf4e58f-f671-4ff1-a542-3686e3d153d1" +IDebug3_UUID := &IID{0x5cf4e58f, 0xf671, 0x4ff1, {0xa5, 0x42, 0x36, 0x86, 0xe3, 0xd1, 0x53, 0xd1}} IDebug3 :: struct #raw_union { #subtype id3d12debug: IDebug, using id3d12debug3_vtable: ^IDebug3_VTable, @@ -3732,7 +3785,8 @@ DEBUG_DEVICE_GPU_SLOWDOWN_PERFORMANCE_FACTOR :: struct { } -IDebugDevice1_UUID :: "a9b71770-d099-4a65-a698-3dee10020f88" +IDebugDevice1_UUID_STRING :: "a9b71770-d099-4a65-a698-3dee10020f88" +IDebugDevice1_UUID := &IID{0xa9b71770, 0xd099, 0x4a65, {0xa6, 0x98, 0x3d, 0xee, 0x10, 0x02, 0x0f, 0x88}} IDebugDevice1 :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12debugdevice1_vtable: ^IDebugDevice1_VTable, @@ -3745,7 +3799,8 @@ IDebugDevice1_VTable :: struct { } -IDebugDevice_UUID :: "3febd6dd-4973-4787-8194-e45f9e28923e" +IDebugDevice_UUID_STRING :: "3febd6dd-4973-4787-8194-e45f9e28923e" +IDebugDevice_UUID := &IID{0x3febd6dd, 0x4973, 0x4787, {0x81, 0x94, 0xe4, 0x5f, 0x9e, 0x28, 0x92, 0x3e}} IDebugDevice :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12debugdevice_vtable: ^IDebugDevice_VTable, @@ -3758,7 +3813,8 @@ IDebugDevice_VTable :: struct { } -IDebugDevice2_UUID :: "60eccbc1-378d-4df1-894c-f8ac5ce4d7dd" +IDebugDevice2_UUID_STRING :: "60eccbc1-378d-4df1-894c-f8ac5ce4d7dd" +IDebugDevice2_UUID := &IID{0x60eccbc1, 0x378d, 0x4df1, {0x89, 0x4c, 0xf8, 0xac, 0x5c, 0xe4, 0xd7, 0xdd}} IDebugDevice2 :: struct #raw_union { #subtype id3d12debugdevice: IDebugDevice, using id3d12debugdevice2_vtable: ^IDebugDevice2_VTable, @@ -3770,8 +3826,8 @@ IDebugDevice2_VTable :: struct { } - -IDebugCommandQueue_UUID :: "09e0bf36-54ac-484f-8847-4baeeab6053a" +IDebugCommandQueue_UUID_STRING :: "09e0bf36-54ac-484f-8847-4baeeab6053a" +IDebugCommandQueue_UUID := &IID{0x09e0bf36, 0x54ac, 0x484f, {0x88, 0x47, 0x4b, 0xae, 0xea, 0xb6, 0x05, 0x3a}} IDebugCommandQueue :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12debugcommandqueue_vtable: ^IDebugCommandQueue_VTable, From 67689ecb21bc1735871b36fdf1411fbffb02e8fb Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Wed, 27 Apr 2022 20:47:05 +0200 Subject: [PATCH 62/79] Typo fix in d3d12.odin --- vendor/directx/d3d12/d3d12.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/directx/d3d12/d3d12.odin b/vendor/directx/d3d12/d3d12.odin index 739606249..9b3ebbe78 100644 --- a/vendor/directx/d3d12/d3d12.odin +++ b/vendor/directx/d3d12/d3d12.odin @@ -3360,7 +3360,7 @@ VERSIONED_DEVICE_REMOVED_EXTENDED_DATA :: struct { } -IDeviceRemovedExtendedDataSettings_UUID_SRING :: "82BC481C-6B9B-4030-AEDB-7EE3D1DF1E63" +IDeviceRemovedExtendedDataSettings_UUID_STRING :: "82BC481C-6B9B-4030-AEDB-7EE3D1DF1E63" IDeviceRemovedExtendedDataSettings_UUID := &IID{0x82BC481C, 0x6B9B, 0x4030, {0xAE, 0xDB, 0x7E, 0xE3, 0xD1, 0xDF, 0x1E, 0x63}} IDeviceRemovedExtendedDataSettings :: struct #raw_union { #subtype iunknown: IUnknown, From 03c921260091d7dd8b7834163ed3fbdaa4cd46e2 Mon Sep 17 00:00:00 2001 From: Karl Zylinski Date: Wed, 27 Apr 2022 22:30:28 +0200 Subject: [PATCH 63/79] Added some additional IID versions of Direct3D 12 UUIDs that I missed during my first pass. All are accounted for now. --- vendor/directx/d3d12/d3d12.odin | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/vendor/directx/d3d12/d3d12.odin b/vendor/directx/d3d12/d3d12.odin index 9b3ebbe78..c61b6cfb5 100644 --- a/vendor/directx/d3d12/d3d12.odin +++ b/vendor/directx/d3d12/d3d12.odin @@ -3846,7 +3846,8 @@ DEBUG_COMMAND_LIST_GPU_BASED_VALIDATION_SETTINGS :: struct { } -IDebugCommandList1_UUID :: "102ca951-311b-4b01-b11f-ecb83e061b37" +IDebugCommandList1_UUID_STRING :: "102ca951-311b-4b01-b11f-ecb83e061b37" +IDebugCommandList1_UUID := &IID{0x102ca951, 0x311b, 0x4b01, {0xb1, 0x1f, 0xec, 0xb8, 0x3e, 0x06, 0x1b, 0x37}} IDebugCommandList1 :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12debugcommandlist1_vtable: ^IDebugCommandList1_VTable, @@ -3859,7 +3860,8 @@ IDebugCommandList1_VTable :: struct { } -IDebugCommandList_UUID :: "09e0bf36-54ac-484f-8847-4baeeab6053f" +IDebugCommandList_UUID_STRING :: "09e0bf36-54ac-484f-8847-4baeeab6053f" +IDebugCommandList_UUID := &IID{0x09e0bf36, 0x54ac, 0x484f, {0x88, 0x47, 0x4b, 0xae, 0xea, 0xb6, 0x05, 0x3f}} IDebugCommandList :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12debugcommandlist_vtable: ^IDebugCommandList_VTable, @@ -3872,7 +3874,8 @@ IDebugCommandList_VTable :: struct { } -IDebugCommandList2_UUID :: "aeb575cf-4e06-48be-ba3b-c450fc96652e" +IDebugCommandList2_UUID_STRING :: "aeb575cf-4e06-48be-ba3b-c450fc96652e" +IDebugCommandList2_UUID := &IID{0xaeb575cf, 0x4e06, 0x48be, {0xba, 0x3b, 0xc4, 0x50, 0xfc, 0x96, 0x65, 0x2e}} IDebugCommandList2 :: struct #raw_union { #subtype id3d12debugcommandlist: IDebugCommandList, using id3d12debugcommandlist2_vtable: ^IDebugCommandList2_VTable, @@ -3884,7 +3887,8 @@ IDebugCommandList2_VTable :: struct { } -ISharingContract_UUID :: "0adf7d52-929c-4e61-addb-ffed30de66ef" +ISharingContract_UUID_STRING :: "0adf7d52-929c-4e61-addb-ffed30de66ef" +ISharingContract_UUID := &IID{0x0adf7d52, 0x929c, 0x4e61, {0xad, 0xdb, 0xff, 0xed, 0x30, 0xde, 0x66, 0xef}} ISharingContract :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12sharingcontract_vtable: ^ISharingContract_VTable, @@ -4789,7 +4793,8 @@ INFO_QUEUE_FILTER :: struct { } -IInfoQueue_UUID :: "0742a90b-c387-483f-b946-30a7e4e61458" +IInfoQueue_UUID_STRING :: "0742a90b-c387-483f-b946-30a7e4e61458" +IInfoQueue_UUID := &IID{0x0742a90b, 0xc387, 0x483f, {0xb9, 0x46, 0x30, 0xa7, 0xe4, 0xe6, 0x14, 0x58}} IInfoQueue :: struct #raw_union { #subtype iunknown: IUnknown, using id3d12infoqueue_vtable: ^IInfoQueue_VTable, @@ -4861,7 +4866,8 @@ SHADING_RATE_COMBINER :: enum i32 { } -IGraphicsCommandList5_UUID :: "55050859-4024-474c-87f5-6472eaee44ea" +IGraphicsCommandList5_UUID_STRING :: "55050859-4024-474c-87f5-6472eaee44ea" +IGraphicsCommandList5_UUID := &IID{0x55050859, 0x4024, 0x474c, {0x87, 0xf5, 0x64, 0x72, 0xea, 0xee, 0x44, 0xea}} IGraphicsCommandList5 :: struct #raw_union { #subtype id3d12graphicscommandlist4: IGraphicsCommandList4, using id3d12graphicscommandlist5_vtable: ^IGraphicsCommandList5_VTable, @@ -4879,7 +4885,8 @@ DISPATCH_MESH_ARGUMENTS :: struct { } -IGraphicsCommandList6_UUID :: "c3827890-e548-4cfa-96cf-5689a9370f80" +IGraphicsCommandList6_UUID_STRING :: "c3827890-e548-4cfa-96cf-5689a9370f80" +IGraphicsCommandList6_UUID := &IID{0xc3827890, 0xe548, 0x4cfa, {0x96, 0xcf, 0x56, 0x89, 0xa9, 0x37, 0x0f, 0x80}} IGraphicsCommandList6 :: struct #raw_union { #subtype id3d12graphicscommandlist5: IGraphicsCommandList5, using id3d12graphicscommandlist6_vtable: ^IGraphicsCommandList6_VTable, From 80878264b63cd8476def629526b294b8e129791a Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 28 Apr 2022 15:29:00 +0200 Subject: [PATCH 64/79] [xml] Speedup. --- core/encoding/xml/debug_print.odin | 18 +- core/encoding/xml/example/xml_example.odin | 77 ++++-- core/encoding/xml/helpers.odin | 28 +-- core/encoding/xml/tokenizer.odin | 11 + core/encoding/xml/xml_reader.odin | 278 +++++++++++---------- tests/core/encoding/xml/test_core_xml.odin | 17 +- 6 files changed, 246 insertions(+), 183 deletions(-) diff --git a/core/encoding/xml/debug_print.odin b/core/encoding/xml/debug_print.odin index e6a8c9433..7c20ac123 100644 --- a/core/encoding/xml/debug_print.odin +++ b/core/encoding/xml/debug_print.odin @@ -1,8 +1,7 @@ -package xml /* An XML 1.0 / 1.1 parser - Copyright 2021 Jeroen van Rijn . + Copyright 2021-2022 Jeroen van Rijn . Made available under Odin's BSD-3 license. A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816). @@ -10,6 +9,8 @@ package xml List of contributors: Jeroen van Rijn: Initial implementation. */ +package xml + import "core:io" import "core:fmt" @@ -40,17 +41,16 @@ print :: proc(writer: io.Writer, doc: ^Document) -> (written: int, err: io.Error written += wprintf(writer, "[Pre-root comment] %v\n", comment) } - if doc.root != nil { + if len(doc.elements) > 0 { + wprintln(writer, " --- ") + print_element(writer, doc, 0) wprintln(writer, " --- ") - print_element(writer, doc.root) - wprintln(writer, " --- ") } return written, .None } -print_element :: proc(writer: io.Writer, element: ^Element, indent := 0) -> (written: int, err: io.Error) { - if element == nil { return } +print_element :: proc(writer: io.Writer, doc: ^Document, element_id: Element_ID, indent := 0) -> (written: int, err: io.Error) { using fmt tab :: proc(writer: io.Writer, indent: int) { @@ -61,6 +61,8 @@ print_element :: proc(writer: io.Writer, element: ^Element, indent := 0) -> (wri tab(writer, indent) + element := doc.elements[element_id] + if element.kind == .Element { wprintf(writer, "<%v>\n", element.ident) if len(element.value) > 0 { @@ -74,7 +76,7 @@ print_element :: proc(writer: io.Writer, element: ^Element, indent := 0) -> (wri } for child in element.children { - print_element(writer, child, indent + 1) + print_element(writer, doc, child, indent + 1) } } else if element.kind == .Comment { wprintf(writer, "[COMMENT] %v\n", element.value) diff --git a/core/encoding/xml/example/xml_example.odin b/core/encoding/xml/example/xml_example.odin index daa3c5dab..cadfcfb43 100644 --- a/core/encoding/xml/example/xml_example.odin +++ b/core/encoding/xml/example/xml_example.odin @@ -1,52 +1,85 @@ package xml_example import "core:encoding/xml" -import "core:os" import "core:mem" import "core:fmt" import "core:time" import "core:strings" import "core:hash" +N :: 1 + example :: proc() { using fmt - doc: ^xml.Document - err: xml.Error + docs: [N]^xml.Document + errs: [N]xml.Error + times: [N]time.Duration + + defer for round in 0..` tag.") - os.exit(1) + eprintln("Could not locate top-level `` tag.") + return } - printf("Found `` with %v children.\n", len(charlist.children)) + printf("Found `` with %v children, %v elements total\n", len(docs[0].elements[charlist].children), docs[0].element_count) - crc32 := doc_hash(doc) + crc32 := doc_hash(docs[0]) printf("[%v] CRC32: 0x%08x\n", "🎉" if crc32 == 0xcaa042b9 else "🤬", crc32) + + for round in 0.. (crc32: u32) { diff --git a/core/encoding/xml/helpers.odin b/core/encoding/xml/helpers.odin index 14597ddbd..48f058334 100644 --- a/core/encoding/xml/helpers.odin +++ b/core/encoding/xml/helpers.odin @@ -1,22 +1,20 @@ -package xml /* An XML 1.0 / 1.1 parser - Copyright 2021 Jeroen van Rijn . + Copyright 2021-2022 Jeroen van Rijn . Made available under Odin's BSD-3 license. This file contains helper functions. */ +package xml - -/* - Find `tag`'s nth child with a given ident. -*/ -find_child_by_ident :: proc(tag: ^Element, ident: string, nth := 0) -> (res: ^Element, found: bool) { - if tag == nil { return nil, false } +// Find parent's nth child with a given ident. +find_child_by_ident :: proc(doc: ^Document, parent_id: Element_ID, ident: string, nth := 0) -> (res: Element_ID, found: bool) { + tag := doc.elements[parent_id] count := 0 - for child in tag.children { + for child_id in tag.children { + child := doc.elements[child_id] /* Skip commments. They have no name. */ @@ -26,18 +24,16 @@ find_child_by_ident :: proc(tag: ^Element, ident: string, nth := 0) -> (res: ^El If the ident matches and it's the nth such child, return it. */ if child.ident == ident { - if count == nth { return child, true } + if count == nth { return child_id, true } count += 1 } } - return nil, false + return 0, false } -/* - Find an attribute by key. -*/ -find_attribute_val_by_key :: proc(tag: ^Element, key: string) -> (val: string, found: bool) { - if tag == nil { return "", false } +// Find an attribute by key. +find_attribute_val_by_key :: proc(doc: ^Document, parent_id: Element_ID, key: string) -> (val: string, found: bool) { + tag := doc.elements[parent_id] for attr in tag.attribs { /* diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin index 2da3b7683..c3fece76e 100644 --- a/core/encoding/xml/tokenizer.odin +++ b/core/encoding/xml/tokenizer.odin @@ -1,3 +1,14 @@ +/* + An XML 1.0 / 1.1 parser + + Copyright 2021-2022 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + A from-scratch XML implementation, loosely modeled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816). + + List of contributors: + Jeroen van Rijn: Initial implementation. +*/ package xml import "core:fmt" diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 0315b0e05..636dd0ae4 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -1,8 +1,7 @@ -package xml /* An XML 1.0 / 1.1 parser - Copyright 2021 Jeroen van Rijn . + Copyright 2021-2022 Jeroen van Rijn . Made available under Odin's BSD-3 license. A from-scratch XML implementation, loosely modelled on the [spec](https://www.w3.org/TR/2006/REC-xml11-20060816). @@ -25,12 +24,17 @@ package xml List of contributors: Jeroen van Rijn: Initial implementation. */ +package xml +// An XML 1.0 / 1.1 parser import "core:bytes" -import "core:strings" import "core:encoding/entity" +import "core:intrinsics" import "core:mem" import "core:os" +import "core:strings" + +likely :: intrinsics.expect DEFAULT_Options :: Options{ flags = { @@ -88,7 +92,9 @@ Option_Flag :: enum { Option_Flags :: bit_set[Option_Flag; u16] Document :: struct { - root: ^Element, + elements: [dynamic]Element, + element_count: Element_ID, + prolog: Attributes, encoding: Encoding, @@ -129,8 +135,8 @@ Element :: struct { Comment, }, - parent: ^Element, - children: [dynamic]^Element, + parent: Element_ID, + children: [dynamic]Element_ID, } Attr :: struct { @@ -185,7 +191,7 @@ Error :: enum { No_DocType, Too_Many_DocTypes, - DocType_Must_Proceed_Elements, + DocType_Must_Preceed_Elements, /* If a DOCTYPE is present _or_ the caller @@ -237,12 +243,16 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err doc.tokenizer = t doc.input = data + doc.elements = make([dynamic]Element, 1024, 1024, allocator) + // strings.intern_init(&doc.intern, allocator, allocator) err = .Unexpected_Token - element, parent: ^Element + element, parent: Element_ID - tag_is_open := false + tag_is_open := false + first_element := true + open: Token /* If a DOCTYPE is present, the root tag has to match. @@ -252,6 +262,7 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err loop: for { skip_whitespace(t) + // NOTE(Jeroen): This is faster as a switch. switch t.ch { case '<': /* @@ -259,35 +270,85 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err */ advance_rune(t) - open := scan(t) - #partial switch open.kind { - - case .Question: + open = scan(t) + // NOTE(Jeroen): We're not using a switch because this if-else chain ordered by likelihood is 2.5% faster at -o:size and -o:speed. + if likely(open.kind, Token_Kind.Ident) == .Ident { /* - 0 { - /* - We've already seen a prolog. - */ - return doc, .Too_Many_Prologs - } else { - /* - Could be ` 0 && expected_doctype != open.text { + error(t, t.offset, "Root Tag doesn't match DOCTYPE. Expected: %v, got: %v\n", expected_doctype, open.text) + return doc, .Invalid_DocType } + } + + /* + One of these should follow: + - `>`, which means we've just opened this tag and expect a later element to close it. + - `/>`, which means this is an 'empty' or self-closing tag. + */ + end_token := scan(t) + #partial switch end_token.kind { + case .Gt: + /* + We're now the new parent. + */ + parent = element + + case .Slash: + /* + Empty tag. Close it. + */ + expect(t, .Gt) or_return + parent = doc.elements[element].parent + element = parent + tag_is_open = false + case: - error(t, t.offset, "Expected \" 0 { return doc, .Too_Many_DocTypes } - if doc.root != nil { - return doc, .DocType_Must_Proceed_Elements + if doc.element_count > 0 { + return doc, .DocType_Must_Preceed_Elements } parse_doctype(doc) or_return @@ -327,14 +388,14 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err comment := scan_comment(t) or_return if .Intern_Comments in opts.flags { - if doc.root == nil { + if len(doc.elements) == 0 { append(&doc.comments, comment) } else { - el := new(Element) - el.parent = element - el.kind = .Comment - el.value = comment - append(&element.children, el) + el := new_element(doc) + doc.elements[el].parent = element + doc.elements[el].kind = .Comment + doc.elements[el].value = comment + append(&doc.elements[element].children, el) } } @@ -343,83 +404,32 @@ parse_from_slice :: proc(data: []u8, options := DEFAULT_Options, path := "", err return } - case .Ident: + } else if open.kind == .Question { /* - e.g. 0 && expected_doctype != open.text { - error(t, t.offset, "Root Tag doesn't match DOCTYPE. Expected: %v, got: %v\n", expected_doctype, open.text) - return doc, .Invalid_DocType + next := scan(t) + #partial switch next.kind { + case .Ident: + if len(next.text) == 3 && strings.to_lower(next.text, context.temp_allocator) == "xml" { + parse_prolog(doc) or_return + } else if len(doc.prolog) > 0 { + /* + We've already seen a prolog. + */ + return doc, .Too_Many_Prologs + } else { + /* + Could be ``, which means we've just opened this tag and expect a later element to close it. - - `/>`, which means this is an 'empty' or self-closing tag. - */ - end_token := scan(t) - #partial switch end_token.kind { - case .Gt: - /* - We're now the new parent. - */ - parent = element - - case .Slash: - /* - Empty tag. Close it. - */ - expect(t, .Gt) or_return - parent = element.parent - element = parent - tag_is_open = false - case: - error(t, t.offset, "Expected close tag, got: %#v\n", end_token) + error(t, t.offset, "Expected \" (err: Error) { */ doc.doctype.rest = string(t.src[offset : t.offset - 1]) return .None +} + +Element_ID :: u32 + +new_element :: proc(doc: ^Document) -> (id: Element_ID) { + element_space := len(doc.elements) + + // Need to resize + if int(doc.element_count) + 1 > element_space { + if element_space < 65536 { + element_space *= 2 + } else { + element_space += 65536 + } + resize(&doc.elements, element_space) + } + + cur := doc.element_count + doc.element_count += 1 + + return cur } \ No newline at end of file diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin index 7669afe97..82386b2bb 100644 --- a/tests/core/encoding/xml/test_core_xml.odin +++ b/tests/core/encoding/xml/test_core_xml.odin @@ -224,7 +224,7 @@ doc_to_string :: proc(doc: ^xml.Document) -> (result: string) { written += wprintf(writer, "[DOCTYPE] %v\n", doc.doctype.ident) if len(doc.doctype.rest) > 0 { - wprintf(writer, "\t%v\n", doc.doctype.rest) + wprintf(writer, "\t%v\n", doc.doctype.rest) } } @@ -232,17 +232,16 @@ doc_to_string :: proc(doc: ^xml.Document) -> (result: string) { written += wprintf(writer, "[Pre-root comment] %v\n", comment) } - if doc.root != nil { - wprintln(writer, " --- ") - print_element(writer, doc.root) - wprintln(writer, " --- ") + if doc.element_count > 0 { + wprintln(writer, " --- ") + print_element(writer, doc, 0) + wprintln(writer, " --- ") } return written, .None } - print_element :: proc(writer: io.Writer, element: ^xml.Element, indent := 0) -> (written: int, err: io.Error) { - if element == nil { return } + print_element :: proc(writer: io.Writer, doc: ^xml.Document, element_id: xml.Element_ID, indent := 0) -> (written: int, err: io.Error) { using fmt tab :: proc(writer: io.Writer, indent: int) { @@ -253,6 +252,8 @@ doc_to_string :: proc(doc: ^xml.Document) -> (result: string) { tab(writer, indent) + element := doc.elements[element_id] + if element.kind == .Element { wprintf(writer, "<%v>\n", element.ident) if len(element.value) > 0 { @@ -266,7 +267,7 @@ doc_to_string :: proc(doc: ^xml.Document) -> (result: string) { } for child in element.children { - print_element(writer, child, indent + 1) + print_element(writer, doc, child, indent + 1) } } else if element.kind == .Comment { wprintf(writer, "[COMMENT] %v\n", element.value) From 127b0ba65e473ae54fe514ea3e695959cefc3b68 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 28 Apr 2022 15:46:36 +0200 Subject: [PATCH 65/79] [xml] Enable tests. --- tests/core/build.bat | 16 +++++++++------- tests/core/encoding/xml/test_core_xml.odin | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/core/build.bat b/tests/core/build.bat index b03fef4bb..8e4ba1d15 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -1,5 +1,6 @@ @echo off -set COMMON=-show-timings -no-bounds-check -vet -strict-style -collection:tests=.. +set COMMON=-no-bounds-check -vet -strict-style +set COLLECTION=-collection:tests=.. set PATH_TO_ODIN==..\..\odin python3 download_assets.py echo --- @@ -35,9 +36,10 @@ echo --- echo --- echo Running core:encoding tests echo --- -%PATH_TO_ODIN% run encoding/hxa %COMMON% -out:test_hxa.exe -%PATH_TO_ODIN% run encoding/json %COMMON% -out:test_json.exe +%PATH_TO_ODIN% run encoding/hxa %COMMON% %COLLECTION% -out:test_hxa.exe +%PATH_TO_ODIN% run encoding/json %COMMON% -out:test_json.exe %PATH_TO_ODIN% run encoding/varint %COMMON% -out:test_varint.exe +%PATH_TO_ODIN% run encoding/xml %COMMON% -out:test_xml.exe echo --- echo Running core:math/noise tests @@ -47,19 +49,19 @@ echo --- echo --- echo Running core:math tests echo --- -%PATH_TO_ODIN% run math %COMMON% -out:test_core_math.exe +%PATH_TO_ODIN% run math %COMMON% %COLLECTION% -out:test_core_math.exe echo --- echo Running core:math/linalg/glsl tests echo --- -%PATH_TO_ODIN% run math/linalg/glsl %COMMON% -out:test_linalg_glsl.exe +%PATH_TO_ODIN% run math/linalg/glsl %COMMON% %COLLECTION% -out:test_linalg_glsl.exe echo --- echo Running core:path/filepath tests echo --- -%PATH_TO_ODIN% run path/filepath %COMMON% -out:test_core_filepath.exe +%PATH_TO_ODIN% run path/filepath %COMMON% %COLLECTION% -out:test_core_filepath.exe echo --- echo Running core:reflect tests echo --- -%PATH_TO_ODIN% run reflect %COMMON% -out:test_core_reflect.exe \ No newline at end of file +%PATH_TO_ODIN% run reflect %COMMON% %COLLECTION% -out:test_core_reflect.exe \ No newline at end of file diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin index 82386b2bb..a79c939c8 100644 --- a/tests/core/encoding/xml/test_core_xml.odin +++ b/tests/core/encoding/xml/test_core_xml.odin @@ -168,6 +168,21 @@ TESTS :: []TEST{ err = .Invalid_DocType, crc32 = 0x49b83d0a, }, + + /* + Parse the 8.2 MiB unicode.xml for good measure. + */ + { + filename = "unicode.xml", + options = { + flags = { + .Ignore_Unsupported, + }, + expected_doctype = "", + }, + err = .None, + crc32 = 0xcaa042b9, + }, } when ODIN_TEST { From a1002e69606cf1d60a5936ddd8ab9d069b5d499b Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 28 Apr 2022 17:46:36 +0200 Subject: [PATCH 66/79] Fix -error-pos-style:unix --- src/main.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main.cpp b/src/main.cpp index 7b0364149..c9e908f11 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -630,6 +630,7 @@ enum BuildFlagKind { BuildFlag_IgnoreWarnings, BuildFlag_WarningsAsErrors, BuildFlag_VerboseErrors, + BuildFlag_ErrorPosStyle, // internal use only BuildFlag_InternalIgnoreLazy, @@ -793,6 +794,7 @@ bool parse_build_flags(Array args) { add_flag(&build_flags, BuildFlag_IgnoreWarnings, str_lit("ignore-warnings"), BuildFlagParam_None, Command_all); add_flag(&build_flags, BuildFlag_WarningsAsErrors, str_lit("warnings-as-errors"), BuildFlagParam_None, Command_all); add_flag(&build_flags, BuildFlag_VerboseErrors, str_lit("verbose-errors"), BuildFlagParam_None, Command_all); + add_flag(&build_flags, BuildFlag_ErrorPosStyle, str_lit("error-pos-style"), BuildFlagParam_String, Command_all); add_flag(&build_flags, BuildFlag_InternalIgnoreLazy, str_lit("internal-ignore-lazy"), BuildFlagParam_None, Command_all); @@ -1472,6 +1474,15 @@ bool parse_build_flags(Array args) { case BuildFlag_VerboseErrors: build_context.show_error_line = true; break; + + case BuildFlag_ErrorPosStyle: + GB_ASSERT(value.kind == ExactValue_String); + build_context.ODIN_ERROR_POS_STYLE = ErrorPosStyle_Default; + if (value.value_string == "unix" || value.value_string == "UNIX") { + build_context.ODIN_ERROR_POS_STYLE = ErrorPosStyle_Unix; + } + break; + case BuildFlag_InternalIgnoreLazy: build_context.ignore_lazy = true; break; From 1ed84a064bd0d64ba4deba56d5a2704350866d1f Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 28 Apr 2022 18:12:55 +0200 Subject: [PATCH 67/79] Make -error-pos-style case-insensitive. --- src/main.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index c9e908f11..ff7bb9f67 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1477,9 +1477,14 @@ bool parse_build_flags(Array args) { case BuildFlag_ErrorPosStyle: GB_ASSERT(value.kind == ExactValue_String); - build_context.ODIN_ERROR_POS_STYLE = ErrorPosStyle_Default; - if (value.value_string == "unix" || value.value_string == "UNIX") { + + if (str_eq_ignore_case(value.value_string, str_lit("odin"))) { + build_context.ODIN_ERROR_POS_STYLE = ErrorPosStyle_Default; + } else if (str_eq_ignore_case(value.value_string, str_lit("unix"))) { build_context.ODIN_ERROR_POS_STYLE = ErrorPosStyle_Unix; + } else { + gb_printf_err("-error-pos-style options are 'unix' and 'odin'\n"); + bad_flags = true; } break; From e53ba3b11612db5c52ecf9b523e4d0ed87f7b1ad Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 28 Apr 2022 18:18:25 +0200 Subject: [PATCH 68/79] Allow -error-pos-style:default as an alias for odin --- src/main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index ff7bb9f67..818a783e1 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1478,12 +1478,12 @@ bool parse_build_flags(Array args) { case BuildFlag_ErrorPosStyle: GB_ASSERT(value.kind == ExactValue_String); - if (str_eq_ignore_case(value.value_string, str_lit("odin"))) { + if (str_eq_ignore_case(value.value_string, str_lit("odin")) || str_eq_ignore_case(value.value_string, str_lit("default"))) { build_context.ODIN_ERROR_POS_STYLE = ErrorPosStyle_Default; } else if (str_eq_ignore_case(value.value_string, str_lit("unix"))) { build_context.ODIN_ERROR_POS_STYLE = ErrorPosStyle_Unix; } else { - gb_printf_err("-error-pos-style options are 'unix' and 'odin'\n"); + gb_printf_err("-error-pos-style options are 'unix', 'odin' and 'default' (odin)\n"); bad_flags = true; } break; From 2fae6eda2321881ccf8d942e2c27e6a7c29aebfd Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 28 Apr 2022 18:58:49 +0200 Subject: [PATCH 69/79] [i18n] Initial i18n support. - Add initial GetText .MO parser - Add translation struct and helpers - Pluralized lookup TODO: - Support for more translation catalog file formats. --- core/i18n/example/i18n_example.odin | 64 +++++++++++ core/i18n/example/messages.pot | 30 +++++ core/i18n/example/nl_NL.mo | Bin 0 -> 672 bytes core/i18n/example/nl_NL.po | 33 ++++++ core/i18n/gettext.odin | 163 ++++++++++++++++++++++++++++ core/i18n/i18n.odin | 116 ++++++++++++++++++++ 6 files changed, 406 insertions(+) create mode 100644 core/i18n/example/i18n_example.odin create mode 100644 core/i18n/example/messages.pot create mode 100644 core/i18n/example/nl_NL.mo create mode 100644 core/i18n/example/nl_NL.po create mode 100644 core/i18n/gettext.odin create mode 100644 core/i18n/i18n.odin diff --git a/core/i18n/example/i18n_example.odin b/core/i18n/example/i18n_example.odin new file mode 100644 index 000000000..f9fb2a353 --- /dev/null +++ b/core/i18n/example/i18n_example.odin @@ -0,0 +1,64 @@ +package i18n_example + +import "core:mem" +import "core:fmt" +import "core:i18n" + +LOC :: i18n.get + +_main :: proc() { + using fmt + + err: i18n.Error + + /* + Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter. + */ + i18n.ACTIVE, err = i18n.parse_mo(#load("nl_NL.mo")) + defer i18n.destroy() + + if err != .None { return } + + /* + These are in the .MO catalog. + */ + println("-----") + println(LOC("")) + println("-----") + println(LOC("There are 69,105 leaves here.")) + println("-----") + println(LOC("Hellope, World!")) + + /* + For ease of use, pluralized lookup can use both singular and plural form as key for the same translation. + */ + println("-----") + printf(LOC("There is %d leaf.\n", 1), 1) + printf(LOC("There is %d leaf.\n", 42), 42) + + printf(LOC("There are %d leaves.\n", 1), 1) + printf(LOC("There are %d leaves.\n", 42), 42) + + /* + This isn't. + */ + println("-----") + println(LOC("Come visit us on Discord!")) +} + +main :: proc() { + using fmt + + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + _main() + + if len(track.allocation_map) > 0 { + println() + for _, v in track.allocation_map { + printf("%v Leaked %v bytes.\n", v.location, v.size) + } + } +} \ No newline at end of file diff --git a/core/i18n/example/messages.pot b/core/i18n/example/messages.pot new file mode 100644 index 000000000..53d521b6b --- /dev/null +++ b/core/i18n/example/messages.pot @@ -0,0 +1,30 @@ +# Odin i18n Example +# Copyright (C) 2021 Jeroen van Rijn +# This file is distributed under the same license as the PACKAGE package. +# Jeroen van Rijn , 2021. +# +#, fuzzy +msgid "" +msgstr "Project-Id-Version: Example 0.0.1\n" + "Report-Msgid-Bugs-To: Jeroen van Rijn \n" + "POT-Creation-Date: 2021-11-27 19:23+0100\n" + "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" + "Last-Translator: FULL NAME \n" + "Language: en-GB\n" + "MIME-Version: 1.0\n" + "Content-Type: text/plain; charset=UTF-8\n" + "Content-Transfer-Encoding: 8bit\n" + +#: i18n_example.odin:28 +msgid "There are 69,105 leaves here." +msgstr "Er zijn hier 69.105 bladeren." + +#: i18n_example.odin:30 +msgid "Hellope, World!" +msgstr "Hallo, Wereld!" + +#: i18n_example.odin:36 +msgid "There is %d leaf.\n" +msgid_plural "There are %d leaves.\n" +msgstr[0] "Er is %d blad.\n" +msgstr[1] "Er zijn %d bladeren.\n" \ No newline at end of file diff --git a/core/i18n/example/nl_NL.mo b/core/i18n/example/nl_NL.mo new file mode 100644 index 0000000000000000000000000000000000000000..0b1a668f4d225e8695e479d6135779288870ac39 GIT binary patch literal 672 zcmZ9J&u$Yj5XKiM7au_44AS&asfgO_2DJ*Crb0kU3ki*)<+lpj3*#GPkbdF5#NYS;wSO_ zR+9Xp^&T;~-RZ})ZqWIHR+5uDL<~A!zL6ZD*0#Yd_+*_f*OOw24nYz-&$qI48#Kxl zdWdQ9S5+Q1$|#%&_D@5XqhiWVojt`x;D;suh|Vi(au}V-y3q)!NJYkG*jN|%#8;~1 zFWbuV!shT6okasHX<(*K4eY){t!iq2(R#W^80)an=b}f6x^+_Xed}tUgJ~|0e%Kp8w;gU8SRX?6cyK)hmFX|<$1FLJ r#*sB9{CC6VbdO literal 0 HcmV?d00001 diff --git a/core/i18n/example/nl_NL.po b/core/i18n/example/nl_NL.po new file mode 100644 index 000000000..1b8acbcc1 --- /dev/null +++ b/core/i18n/example/nl_NL.po @@ -0,0 +1,33 @@ +# Odin i18n Example +# Copyright (C) 2021 Jeroen van Rijn +# This file is distributed under the same license as the PACKAGE package. +# Jeroen van Rijn , 2021. +# +msgid "" +msgstr "" +"Project-Id-Version: Example 0.0.1\n" +"Report-Msgid-Bugs-To: Jeroen van Rijn \n" +"POT-Creation-Date: 2021-11-27 19:23+0100\n" +"PO-Revision-Date: 2021-11-28 02:56+0100\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language-Team: Odin Language Team\n" +"X-Generator: Poedit 3.0\n" +"Last-Translator: Jeroen van Rijn\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" +"Language: nl_NL\n" + +#: i18n_example.odin:28 +msgid "There are 69,105 leaves here." +msgstr "Er zijn hier 69.105 bladeren." + +#: i18n_example.odin:30 +msgid "Hellope, World!" +msgstr "Hallo, Wereld!" + +#: i18n_example.odin:36 +msgid "There is %d leaf.\n" +msgid_plural "There are %d leaves.\n" +msgstr[0] "Er is %d blad.\n" +msgstr[1] "Er zijn %d bladeren.\n" diff --git a/core/i18n/gettext.odin b/core/i18n/gettext.odin new file mode 100644 index 000000000..7918e217e --- /dev/null +++ b/core/i18n/gettext.odin @@ -0,0 +1,163 @@ +package i18n +/* + A parser for GNU GetText .MO files. + + Copyright 2021 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + A from-scratch implementation based after the specification found here: + https://www.gnu.org/software/gettext/manual/html_node/MO-Files.html + + List of contributors: + Jeroen van Rijn: Initial implementation. +*/ +import "core:os" +import "core:strings" +import "core:bytes" + +parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { + context.allocator = allocator + /* + An MO file should have at least a 4-byte magic, 2 x 2 byte version info, + a 4-byte number of strings value, and 2 x 4-byte offsets. + */ + if len(data) < 20 { + return {}, .MO_File_Invalid + } + + /* + Check magic. Should be 0x950412de in native Endianness. + */ + native := true + magic := read_u32(data, native) or_return + + if magic != 0x950412de { + native = false + magic = read_u32(data, native) or_return + + if magic != 0x950412de { return {}, .MO_File_Invalid_Signature } + } + + /* + We can ignore version_minor at offset 6. + */ + version_major := read_u16(data[4:]) or_return + if version_major > 1 { return {}, .MO_File_Unsupported_Version } + + count := read_u32(data[ 8:]) or_return + original_offset := read_u32(data[12:]) or_return + translated_offset := read_u32(data[16:]) or_return + + if count == 0 { return {}, .Empty_Translation_Catalog } + + /* + Initalize Translation, interner and optional pluralizer. + */ + translation = new(Translation) + translation.pluralize = pluralizer + strings.intern_init(&translation.intern, allocator, allocator) + + for n := u32(0); n < count; n += 1 { + /* + Grab string's original length and offset. + */ + offset := original_offset + 8 * n + if len(data) < int(offset + 8) { return translation, .MO_File_Invalid } + + o_length := read_u32(data[offset :], native) or_return + o_offset := read_u32(data[offset + 4:], native) or_return + + offset = translated_offset + 8 * n + if len(data) < int(offset + 8) { return translation, .MO_File_Invalid } + + t_length := read_u32(data[offset :], native) or_return + t_offset := read_u32(data[offset + 4:], native) or_return + + max_offset := int(max(o_offset + o_length + 1, t_offset + t_length + 1)) + if len(data) < max_offset { return translation, .Premature_EOF } + + key := data[o_offset:][:o_length] + val := data[t_offset:][:t_length] + + /* + Could be a pluralized string. + */ + zero := []byte{0} + + keys := bytes.split(key, zero) + vals := bytes.split(val, zero) + + if len(keys) != len(vals) || max(len(keys), len(vals)) > MAX_PLURALS { + return translation, .MO_File_Incorrect_Plural_Count + } + + for k in keys { + interned_key := strings.intern_get(&translation.intern, string(k)) + + interned_vals: [MAX_PLURALS]string = {} + last_val: string + + i := 0 + for v in vals { + interned_vals[i] = strings.intern_get(&translation.intern, string(v)) + last_val = interned_vals[i] + i += 1 + } + for ; i < MAX_PLURALS; i += 1 { + interned_vals[i] = last_val + } + translation.k_v[interned_key] = interned_vals + } + delete(vals) + delete(keys) + } + return +} + +parse_mo_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { + context.allocator = allocator + + data, data_ok := os.read_entire_file(filename) + defer delete(data) + + if !data_ok { return {}, .File_Error } + + return parse_mo_from_slice(data, pluralizer) +} + +parse_mo :: proc { parse_mo_file, parse_mo_from_slice } + +/* + Helpers. +*/ +read_u32 :: proc(data: []u8, native_endian := true) -> (res: u32, err: Error) { + if len(data) < size_of(u32) { return 0, .Premature_EOF } + + val := (^u32)(raw_data(data))^ + + if native_endian { + return val, .None + } else { + when ODIN_ENDIAN == .Little { + return u32(transmute(u32be)val), .None + } else { + return u32(transmute(u32le)val), .None + } + } +} + +read_u16 :: proc(data: []u8, native_endian := true) -> (res: u16, err: Error) { + if len(data) < size_of(u16) { return 0, .Premature_EOF } + + val := (^u16)(raw_data(data))^ + + if native_endian { + return val, .None + } else { + when ODIN_ENDIAN == .Little { + return u16(transmute(u16be)val), .None + } else { + return u16(transmute(u16le)val), .None + } + } +} \ No newline at end of file diff --git a/core/i18n/i18n.odin b/core/i18n/i18n.odin new file mode 100644 index 000000000..7c72f9858 --- /dev/null +++ b/core/i18n/i18n.odin @@ -0,0 +1,116 @@ +package i18n +/* + Internationalization helpers. + + Copyright 2021 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + List of contributors: + Jeroen van Rijn: Initial implementation. +*/ +import "core:strings" + +/* + TODO: + - Support for more translation catalog file formats. +*/ + +MAX_PLURALS :: 10 + +/* + Currently active catalog. +*/ +ACTIVE: ^Translation + +/* + The main data structure. This can be generated from various different file formats, as long as we have a parser for them. +*/ +Translation :: struct { + k_v: map[string][MAX_PLURALS]string, + intern: strings.Intern, + + pluralize: proc(number: int) -> int, +} + +Error :: enum { + /* + General return values. + */ + None = 0, + Empty_Translation_Catalog, + + /* + Couldn't find, open or read file. + */ + File_Error, + + /* + File too short. + */ + Premature_EOF, + + /* + GNU Gettext *.MO file errors. + */ + MO_File_Invalid_Signature, + MO_File_Unsupported_Version, + MO_File_Invalid, + MO_File_Incorrect_Plural_Count, +} + +/* + Several ways to use: + - get(key), which defaults to the singular form and i18n.ACTIVE catalog, or + - get(key, number), which returns the appropriate plural from the active catalog, or + - get(key, number, catalog) to grab text from a specific one. +*/ +get :: proc(key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) { + /* + A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule. + */ + plural := 1 if number != 1 else 0 + + if catalog.pluralize != nil { + plural = catalog.pluralize(number) + } + return get_by_slot(key, plural, catalog) +} + +/* + Several ways to use: + - get_by_slot(key), which defaults to the singular form and i18n.ACTIVE catalog, or + - get_by_slot(key, slot), which returns the requested plural from the active catalog, or + - get_by_slot(key, slot, catalog) to grab text from a specific one. + + If a file format parser doesn't (yet) support plural slots, each of the slots will point at the same string. +*/ +get_by_slot :: proc(key: string, slot := 0, catalog: ^Translation = ACTIVE) -> (value: string) { + if catalog == nil { + /* + Return the key if the catalog catalog hasn't been initialized yet. + */ + return key + } + + /* + Return the translation from the requested slot if this key is known, else return the key. + */ + if translations, ok := catalog.k_v[key]; ok { + plural := min(max(0, slot), MAX_PLURALS - 1) + return translations[plural] + } + return key +} + +/* + Same for destroy: + - destroy(), to clean up the currently active catalog catalog i18n.ACTIVE + - destroy(catalog), to clean up a specific catalog. +*/ +destroy :: proc(catalog: ^Translation = ACTIVE) { + if catalog != nil { + strings.intern_destroy(&catalog.intern) + delete(catalog.k_v) + free(catalog) + } +} \ No newline at end of file From ba23bfb7b9eb32eb9bcf22b0364a25b6ae32203e Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Thu, 28 Apr 2022 20:12:32 +0200 Subject: [PATCH 70/79] [i18n] Allow multiple sections. --- core/i18n/gettext.odin | 11 ++++---- core/i18n/i18n.odin | 57 +++++++++++++++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/core/i18n/gettext.odin b/core/i18n/gettext.odin index 7918e217e..70c922cfb 100644 --- a/core/i18n/gettext.odin +++ b/core/i18n/gettext.odin @@ -57,6 +57,10 @@ parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allo translation.pluralize = pluralizer strings.intern_init(&translation.intern, allocator, allocator) + // Gettext MO files only have one section. + translation.k_v[""] = {} + section := &translation.k_v[""] + for n := u32(0); n < count; n += 1 { /* Grab string's original length and offset. @@ -94,7 +98,7 @@ parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allo for k in keys { interned_key := strings.intern_get(&translation.intern, string(k)) - interned_vals: [MAX_PLURALS]string = {} + interned_vals := make([]string, len(keys)) last_val: string i := 0 @@ -103,10 +107,7 @@ parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allo last_val = interned_vals[i] i += 1 } - for ; i < MAX_PLURALS; i += 1 { - interned_vals[i] = last_val - } - translation.k_v[interned_key] = interned_vals + section[interned_key] = interned_vals } delete(vals) delete(keys) diff --git a/core/i18n/i18n.odin b/core/i18n/i18n.odin index 7c72f9858..1ee19c2b4 100644 --- a/core/i18n/i18n.odin +++ b/core/i18n/i18n.odin @@ -15,18 +15,19 @@ import "core:strings" - Support for more translation catalog file formats. */ -MAX_PLURALS :: 10 - /* Currently active catalog. */ ACTIVE: ^Translation +// Allow between 1 and 255 plural forms. Default: 10. +MAX_PLURALS :: min(max(#config(ODIN_i18N_MAX_PLURAL_FORMS, 10), 1), 255) + /* The main data structure. This can be generated from various different file formats, as long as we have a parser for them. */ Translation :: struct { - k_v: map[string][MAX_PLURALS]string, + k_v: map[string]map[string][]string, intern: strings.Intern, pluralize: proc(number: int) -> int, @@ -64,7 +65,7 @@ Error :: enum { - get(key, number), which returns the appropriate plural from the active catalog, or - get(key, number, catalog) to grab text from a specific one. */ -get :: proc(key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) { +get_single_section :: proc(key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) { /* A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule. */ @@ -76,6 +77,25 @@ get :: proc(key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: return get_by_slot(key, plural, catalog) } +/* + Several ways to use: + - get(section, key), which defaults to the singular form and i18n.ACTIVE catalog, or + - get(section, key, number), which returns the appropriate plural from the active catalog, or + - get(section, key, number, catalog) to grab text from a specific one. +*/ +get_by_section :: proc(section, key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) { + /* + A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule. + */ + plural := 1 if number != 1 else 0 + + if catalog.pluralize != nil { + plural = catalog.pluralize(number) + } + return get_by_slot(key, plural, catalog) +} +get :: proc{get_single_section, get_by_section} + /* Several ways to use: - get_by_slot(key), which defaults to the singular form and i18n.ACTIVE catalog, or @@ -84,10 +104,22 @@ get :: proc(key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: If a file format parser doesn't (yet) support plural slots, each of the slots will point at the same string. */ -get_by_slot :: proc(key: string, slot := 0, catalog: ^Translation = ACTIVE) -> (value: string) { - if catalog == nil { +get_by_slot_single_section :: proc(key: string, slot := 0, catalog: ^Translation = ACTIVE) -> (value: string) { + return get_by_slot_by_section("", key, slot, catalog) +} + +/* + Several ways to use: + - get_by_slot(key), which defaults to the singular form and i18n.ACTIVE catalog, or + - get_by_slot(key, slot), which returns the requested plural from the active catalog, or + - get_by_slot(key, slot, catalog) to grab text from a specific one. + + If a file format parser doesn't (yet) support plural slots, each of the slots will point at the same string. +*/ +get_by_slot_by_section :: proc(section, key: string, slot := 0, catalog: ^Translation = ACTIVE) -> (value: string) { + if catalog == nil || section not_in catalog.k_v { /* - Return the key if the catalog catalog hasn't been initialized yet. + Return the key if the catalog catalog hasn't been initialized yet, or the section is not present. */ return key } @@ -95,12 +127,13 @@ get_by_slot :: proc(key: string, slot := 0, catalog: ^Translation = ACTIVE) -> ( /* Return the translation from the requested slot if this key is known, else return the key. */ - if translations, ok := catalog.k_v[key]; ok { - plural := min(max(0, slot), MAX_PLURALS - 1) + if translations, ok := catalog.k_v[section][key]; ok { + plural := min(max(0, slot), len(catalog.k_v[section][key]) - 1) return translations[plural] } return key } +get_by_slot :: proc{get_by_slot_single_section, get_by_slot_by_section} /* Same for destroy: @@ -110,6 +143,12 @@ get_by_slot :: proc(key: string, slot := 0, catalog: ^Translation = ACTIVE) -> ( destroy :: proc(catalog: ^Translation = ACTIVE) { if catalog != nil { strings.intern_destroy(&catalog.intern) + for section in &catalog.k_v { + for key in &catalog.k_v[section] { + delete(catalog.k_v[section][key]) + } + delete(catalog.k_v[section]) + } delete(catalog.k_v) free(catalog) } From 1289c96e2cf9fdcdb9c4fb4988f73c0e319fb329 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 00:29:55 +0200 Subject: [PATCH 71/79] [i18n] QT Linguist TS reader. --- core/encoding/xml/xml_reader.odin | 1 - core/i18n/example/i18n_example.odin | 56 ++++++++-- core/i18n/gettext.odin | 2 +- core/i18n/i18n.odin | 21 +++- core/i18n/qt_linguist.odin | 153 +++++++++++++++++++++++++++ tests/core/assets/XML/nl_NL-qt-ts.ts | 52 ++++----- 6 files changed, 243 insertions(+), 42 deletions(-) create mode 100644 core/i18n/qt_linguist.odin diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index 636dd0ae4..6d0d4e1aa 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -87,7 +87,6 @@ Option_Flag :: enum { If a tag body has a comment, it will be stripped unless this option is given. */ Keep_Tag_Body_Comments, - } Option_Flags :: bit_set[Option_Flag; u16] diff --git a/core/i18n/example/i18n_example.odin b/core/i18n/example/i18n_example.odin index f9fb2a353..8c173ee4a 100644 --- a/core/i18n/example/i18n_example.odin +++ b/core/i18n/example/i18n_example.odin @@ -4,9 +4,9 @@ import "core:mem" import "core:fmt" import "core:i18n" -LOC :: i18n.get +_T :: i18n.get -_main :: proc() { +mo :: proc() { using fmt err: i18n.Error @@ -23,27 +23,60 @@ _main :: proc() { These are in the .MO catalog. */ println("-----") - println(LOC("")) + println(_T("")) println("-----") - println(LOC("There are 69,105 leaves here.")) + println(_T("There are 69,105 leaves here.")) println("-----") - println(LOC("Hellope, World!")) + println(_T("Hellope, World!")) /* For ease of use, pluralized lookup can use both singular and plural form as key for the same translation. */ println("-----") - printf(LOC("There is %d leaf.\n", 1), 1) - printf(LOC("There is %d leaf.\n", 42), 42) + printf(_T("There is %d leaf.\n", 1), 1) + printf(_T("There is %d leaf.\n", 42), 42) - printf(LOC("There are %d leaves.\n", 1), 1) - printf(LOC("There are %d leaves.\n", 42), 42) + printf(_T("There are %d leaves.\n", 1), 1) + printf(_T("There are %d leaves.\n", 42), 42) /* This isn't. */ println("-----") - println(LOC("Come visit us on Discord!")) + println(_T("Come visit us on Discord!")) +} + +qt :: proc() { + using fmt + + err: i18n.Error + + /* + Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter. + */ + i18n.ACTIVE, err = i18n.parse_qt(#load("../../../tests/core/assets/XML/nl_NL-qt-ts.ts")) + defer i18n.destroy() + + fmt.printf("parse_qt returned %v\n", err) + if err != .None { + return + } + + /* + These are in the .TS catalog. + */ + println("--- Page section ---") + println("Page:Text for translation =", _T("Page", "Text for translation")) + println("-----") + println("Page:Also text to translate =", _T("Page", "Also text to translate")) + println("-----") + println("--- installscript section ---") + println("installscript:99 bottles of beer on the wall =", _T("installscript", "99 bottles of beer on the wall")) + println("-----") + println("--- apple_count section ---") + println("apple_count:%d apple(s) =") + println("\t 1 =", _T("apple_count", "%d apple(s)", 1)) + println("\t 42 =", _T("apple_count", "%d apple(s)", 42)) } main :: proc() { @@ -53,7 +86,8 @@ main :: proc() { mem.tracking_allocator_init(&track, context.allocator) context.allocator = mem.tracking_allocator(&track) - _main() + // mo() + qt() if len(track.allocation_map) > 0 { println() diff --git a/core/i18n/gettext.odin b/core/i18n/gettext.odin index 70c922cfb..54c5a1111 100644 --- a/core/i18n/gettext.odin +++ b/core/i18n/gettext.odin @@ -2,7 +2,7 @@ package i18n /* A parser for GNU GetText .MO files. - Copyright 2021 Jeroen van Rijn . + Copyright 2021-2022 Jeroen van Rijn . Made available under Odin's BSD-3 license. A from-scratch implementation based after the specification found here: diff --git a/core/i18n/i18n.odin b/core/i18n/i18n.odin index 1ee19c2b4..36204efd9 100644 --- a/core/i18n/i18n.odin +++ b/core/i18n/i18n.odin @@ -2,7 +2,7 @@ package i18n /* Internationalization helpers. - Copyright 2021 Jeroen van Rijn . + Copyright 2021-2022 Jeroen van Rijn . Made available under Odin's BSD-3 license. List of contributors: @@ -26,8 +26,11 @@ MAX_PLURALS :: min(max(#config(ODIN_i18N_MAX_PLURAL_FORMS, 10), 1), 255) /* The main data structure. This can be generated from various different file formats, as long as we have a parser for them. */ + +Section :: map[string][]string + Translation :: struct { - k_v: map[string]map[string][]string, + k_v: map[string]Section, // k_v[section][key][plural_form] = ... intern: strings.Intern, pluralize: proc(number: int) -> int, @@ -39,6 +42,7 @@ Error :: enum { */ None = 0, Empty_Translation_Catalog, + Duplicate_Key, /* Couldn't find, open or read file. @@ -57,6 +61,17 @@ Error :: enum { MO_File_Unsupported_Version, MO_File_Invalid, MO_File_Incorrect_Plural_Count, + + /* + Qt Linguist *.TS file errors. + */ + TS_File_Parse_Error, + TS_File_Expected_Context, + TS_File_Expected_Context_Name, + TS_File_Expected_Source, + TS_File_Expected_Translation, + TS_File_Expected_NumerusForm, + } /* @@ -92,7 +107,7 @@ get_by_section :: proc(section, key: string, number := 0, catalog: ^Translation if catalog.pluralize != nil { plural = catalog.pluralize(number) } - return get_by_slot(key, plural, catalog) + return get_by_slot(section, key, plural, catalog) } get :: proc{get_single_section, get_by_section} diff --git a/core/i18n/qt_linguist.odin b/core/i18n/qt_linguist.odin new file mode 100644 index 000000000..65d51444e --- /dev/null +++ b/core/i18n/qt_linguist.odin @@ -0,0 +1,153 @@ +package i18n +/* + A parser for Qt Linguist TS files. + + Copyright 2022 Jeroen van Rijn . + Made available under Odin's BSD-3 license. + + A from-scratch implementation based after the specification found here: + https://doc.qt.io/qt-5/linguist-ts-file-format.html + + List of contributors: + Jeroen van Rijn: Initial implementation. +*/ +import "core:os" +import "core:encoding/xml" +import "core:strings" + +TS_XML_Options := xml.Options{ + flags = { + .Input_May_Be_Modified, + .Must_Have_Prolog, + .Must_Have_DocType, + .Ignore_Unsupported, + .Unbox_CDATA, + .Decode_SGML_Entities, + }, + expected_doctype = "TS", +} + +parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { + context.allocator = allocator + + ts, xml_err := xml.parse(data, TS_XML_Options) + defer xml.destroy(ts) + + if xml_err != .None || ts.element_count < 1 || ts.elements[0].ident != "TS" || len(ts.elements[0].children) == 0 { + return nil, .TS_File_Parse_Error + } + + /* + Initalize Translation, interner and optional pluralizer. + */ + translation = new(Translation) + translation.pluralize = pluralizer + strings.intern_init(&translation.intern, allocator, allocator) + + section: ^Section + + for child_id in ts.elements[0].children { + // These should be s. + child := ts.elements[child_id] + if child.ident != "context" { + return translation, .TS_File_Expected_Context + } + + // Find section name. + section_name_id, section_name_found := xml.find_child_by_ident(ts, child_id, "name") + if !section_name_found { + return translation, .TS_File_Expected_Context_Name, + } + + section_name := ts.elements[section_name_id].value + + if section_name not_in translation.k_v { + translation.k_v[section_name] = {} + } + section = &translation.k_v[section_name] + + // Find messages in section. + nth: int + for { + message_id, message_found := xml.find_child_by_ident(ts, child_id, "message", nth) + if !message_found { + break + } + + numerus_tag, _ := xml.find_attribute_val_by_key(ts, message_id, "numerus") + has_plurals := numerus_tag == "yes" + + // We must have a = key + source_id, source_found := xml.find_child_by_ident(ts, message_id, "source") + if !source_found { + return translation, .TS_File_Expected_Source + } + + // We must have a + translation_id, translation_found := xml.find_child_by_ident(ts, message_id, "translation") + if !translation_found { + return translation, .TS_File_Expected_Translation + } + + source := ts.elements[source_id] + xlat := ts.elements[translation_id] + + if source.value in section { + return translation, .Duplicate_Key + } + + if has_plurals { + if xlat.value != "" { + return translation, .TS_File_Expected_NumerusForm + } + + num_plurals: int + for { + numerus_id, numerus_found := xml.find_child_by_ident(ts, translation_id, "numerusform", num_plurals) + if !numerus_found { + break + } + num_plurals += 1 + } + + if num_plurals < 2 { + return translation, .TS_File_Expected_NumerusForm + } + section[source.value] = make([]string, num_plurals) + + num_plurals = 0 + for { + numerus_id, numerus_found := xml.find_child_by_ident(ts, translation_id, "numerusform", num_plurals) + if !numerus_found { + break + } + numerus := ts.elements[numerus_id] + section[source.value][num_plurals] = strings.intern_get(&translation.intern, numerus.value) + + num_plurals += 1 + } + } else { + // Single translation + section[source.value] = make([]string, 1) + section[source.value][0] = strings.intern_get(&translation.intern, xlat.value) + } + + nth += 1 + } + } + + return +} + +parse_qt_linguist_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { + context.allocator = allocator + + data, data_ok := os.read_entire_file(filename) + defer delete(data) + + if !data_ok { return {}, .File_Error } + + return parse_qt_linguist_from_slice(data, pluralizer) +} + +parse_qt :: proc { parse_qt_linguist_file, parse_qt_linguist_from_slice } \ No newline at end of file diff --git a/tests/core/assets/XML/nl_NL-qt-ts.ts b/tests/core/assets/XML/nl_NL-qt-ts.ts index 6ec3f2f47..36c95ce2e 100644 --- a/tests/core/assets/XML/nl_NL-qt-ts.ts +++ b/tests/core/assets/XML/nl_NL-qt-ts.ts @@ -2,34 +2,34 @@ - Page - - Text for translation - commenting - Tekst om te vertalen - - - Also text to translate - some text - Ook tekst om te vertalen - + Page + + Text for translation + commenting + Tekst om te vertalen + + + Also text to translate + some text + Ook tekst om te vertalen + - installscript - - 99 bottles of beer on the wall - some new comments here - 99 flessen bier op de muur - + installscript + + 99 bottles of beer on the wall + some new comments here + 99 flessen bier op de muur + - apple_count - - %d apple(s) - - %d appel - %d appels - - - + apple_count + + %d apple(s) + + %d appel + %d appels + + + From 9ce64916e6f6db558d53b9a852e2169b937317f8 Mon Sep 17 00:00:00 2001 From: sduman Date: Thu, 28 Apr 2022 17:08:48 -0600 Subject: [PATCH 72/79] Add missing result parameter names This adds some missing result parameters names back to pop_front_safe. Currently it the procedure won't compile since it's referencing missing variable names. --- core/container/small_array/small_array.odin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/container/small_array/small_array.odin b/core/container/small_array/small_array.odin index 5cd421c84..4dd16f30c 100644 --- a/core/container/small_array/small_array.odin +++ b/core/container/small_array/small_array.odin @@ -86,7 +86,7 @@ pop_back_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (item: T, ok: bool) { return } -pop_front_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (T, bool) { +pop_front_safe :: proc(a: ^$A/Small_Array($N, $T)) -> (item: T, ok: bool) { if N > 0 && a.len > 0 { item = a.data[0] s := slice(a) @@ -114,4 +114,4 @@ push_back_elems :: proc(a: ^$A/Small_Array($N, $T), items: ..T) { append_elem :: push_back append_elems :: push_back_elems push :: proc{push_back, push_back_elems} -append :: proc{push_back, push_back_elems} \ No newline at end of file +append :: proc{push_back, push_back_elems} From 2e11a8da5b3d0031f99a7534b029f08dc0fe5a36 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 13:02:40 +0200 Subject: [PATCH 73/79] [i18n] Move to `core:text/i18n`. --- core/{ => text}/i18n/example/i18n_example.odin | 5 +++-- core/{ => text}/i18n/example/messages.pot | 0 core/{ => text}/i18n/example/nl_NL.mo | Bin core/{ => text}/i18n/example/nl_NL.po | 0 core/{ => text}/i18n/gettext.odin | 0 core/{ => text}/i18n/i18n.odin | 0 core/{ => text}/i18n/qt_linguist.odin | 0 7 files changed, 3 insertions(+), 2 deletions(-) rename core/{ => text}/i18n/example/i18n_example.odin (91%) rename core/{ => text}/i18n/example/messages.pot (100%) rename core/{ => text}/i18n/example/nl_NL.mo (100%) rename core/{ => text}/i18n/example/nl_NL.po (100%) rename core/{ => text}/i18n/gettext.odin (100%) rename core/{ => text}/i18n/i18n.odin (100%) rename core/{ => text}/i18n/qt_linguist.odin (100%) diff --git a/core/i18n/example/i18n_example.odin b/core/text/i18n/example/i18n_example.odin similarity index 91% rename from core/i18n/example/i18n_example.odin rename to core/text/i18n/example/i18n_example.odin index 8c173ee4a..32eb38a7d 100644 --- a/core/i18n/example/i18n_example.odin +++ b/core/text/i18n/example/i18n_example.odin @@ -2,7 +2,7 @@ package i18n_example import "core:mem" import "core:fmt" -import "core:i18n" +import "core:text/i18n" _T :: i18n.get @@ -31,6 +31,7 @@ mo :: proc() { /* For ease of use, pluralized lookup can use both singular and plural form as key for the same translation. + This is a quirk of the GetText format which has separate keys for their different plurals. */ println("-----") printf(_T("There is %d leaf.\n", 1), 1) @@ -54,7 +55,7 @@ qt :: proc() { /* Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter. */ - i18n.ACTIVE, err = i18n.parse_qt(#load("../../../tests/core/assets/XML/nl_NL-qt-ts.ts")) + i18n.ACTIVE, err = i18n.parse_qt(#load("../../../../tests/core/assets/XML/nl_NL-qt-ts.ts")) defer i18n.destroy() fmt.printf("parse_qt returned %v\n", err) diff --git a/core/i18n/example/messages.pot b/core/text/i18n/example/messages.pot similarity index 100% rename from core/i18n/example/messages.pot rename to core/text/i18n/example/messages.pot diff --git a/core/i18n/example/nl_NL.mo b/core/text/i18n/example/nl_NL.mo similarity index 100% rename from core/i18n/example/nl_NL.mo rename to core/text/i18n/example/nl_NL.mo diff --git a/core/i18n/example/nl_NL.po b/core/text/i18n/example/nl_NL.po similarity index 100% rename from core/i18n/example/nl_NL.po rename to core/text/i18n/example/nl_NL.po diff --git a/core/i18n/gettext.odin b/core/text/i18n/gettext.odin similarity index 100% rename from core/i18n/gettext.odin rename to core/text/i18n/gettext.odin diff --git a/core/i18n/i18n.odin b/core/text/i18n/i18n.odin similarity index 100% rename from core/i18n/i18n.odin rename to core/text/i18n/i18n.odin diff --git a/core/i18n/qt_linguist.odin b/core/text/i18n/qt_linguist.odin similarity index 100% rename from core/i18n/qt_linguist.odin rename to core/text/i18n/qt_linguist.odin From 957ef8e8fe885fed32b62d532e642be5e756ea67 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 13:16:30 +0200 Subject: [PATCH 74/79] [i18n/xml] Move I18N XML files to their own assets directory. --- .../core/assets/I18N}/messages.pot | 0 .../core/assets/{XML => I18N}/nl_NL-qt-ts.ts | 0 .../{XML => I18N}/nl_NL-xliff-1.2.xliff | 0 .../{XML => I18N}/nl_NL-xliff-2.0.xliff | 0 .../core/assets/I18N}/nl_NL.mo | Bin .../core/assets/I18N}/nl_NL.po | 0 tests/core/encoding/xml/test_core_xml.odin | 33 ++++++------------ 7 files changed, 11 insertions(+), 22 deletions(-) rename {core/text/i18n/example => tests/core/assets/I18N}/messages.pot (100%) rename tests/core/assets/{XML => I18N}/nl_NL-qt-ts.ts (100%) rename tests/core/assets/{XML => I18N}/nl_NL-xliff-1.2.xliff (100%) rename tests/core/assets/{XML => I18N}/nl_NL-xliff-2.0.xliff (100%) rename {core/text/i18n/example => tests/core/assets/I18N}/nl_NL.mo (100%) rename {core/text/i18n/example => tests/core/assets/I18N}/nl_NL.po (100%) diff --git a/core/text/i18n/example/messages.pot b/tests/core/assets/I18N/messages.pot similarity index 100% rename from core/text/i18n/example/messages.pot rename to tests/core/assets/I18N/messages.pot diff --git a/tests/core/assets/XML/nl_NL-qt-ts.ts b/tests/core/assets/I18N/nl_NL-qt-ts.ts similarity index 100% rename from tests/core/assets/XML/nl_NL-qt-ts.ts rename to tests/core/assets/I18N/nl_NL-qt-ts.ts diff --git a/tests/core/assets/XML/nl_NL-xliff-1.2.xliff b/tests/core/assets/I18N/nl_NL-xliff-1.2.xliff similarity index 100% rename from tests/core/assets/XML/nl_NL-xliff-1.2.xliff rename to tests/core/assets/I18N/nl_NL-xliff-1.2.xliff diff --git a/tests/core/assets/XML/nl_NL-xliff-2.0.xliff b/tests/core/assets/I18N/nl_NL-xliff-2.0.xliff similarity index 100% rename from tests/core/assets/XML/nl_NL-xliff-2.0.xliff rename to tests/core/assets/I18N/nl_NL-xliff-2.0.xliff diff --git a/core/text/i18n/example/nl_NL.mo b/tests/core/assets/I18N/nl_NL.mo similarity index 100% rename from core/text/i18n/example/nl_NL.mo rename to tests/core/assets/I18N/nl_NL.mo diff --git a/core/text/i18n/example/nl_NL.po b/tests/core/assets/I18N/nl_NL.po similarity index 100% rename from core/text/i18n/example/nl_NL.po rename to tests/core/assets/I18N/nl_NL.po diff --git a/tests/core/encoding/xml/test_core_xml.odin b/tests/core/encoding/xml/test_core_xml.odin index a79c939c8..07cbc1779 100644 --- a/tests/core/encoding/xml/test_core_xml.odin +++ b/tests/core/encoding/xml/test_core_xml.odin @@ -27,7 +27,7 @@ TEST :: struct { /* Relative to ODIN_ROOT */ -TEST_FILE_PATH_PREFIX :: "tests/core/assets/XML" +TEST_FILE_PATH_PREFIX :: "tests/core/assets" TESTS :: []TEST{ /* @@ -35,23 +35,12 @@ TESTS :: []TEST{ */ { - /* - - - <恥ずべきフクロウ 올빼미_id="Foozle Hello, world!"]]>Barzle"> - <부끄러운:barzle> - ရှက်စရာ ဇီးကွက် - Owl of Shame - More CDATA Hello, world! Nonsense. - - */ - /* Tests UTF-8 idents and values. Test namespaced ident. Tests that nested partial CDATA start doesn't trip up parser. */ - filename = "utf8.xml", + filename = "XML/utf8.xml", options = { flags = { .Ignore_Unsupported, .Intern_Comments, @@ -66,7 +55,7 @@ TESTS :: []TEST{ Same as above. Unbox CDATA in data tag. */ - filename = "utf8.xml", + filename = "XML/utf8.xml", options = { flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, @@ -81,7 +70,7 @@ TESTS :: []TEST{ Simple Qt TS translation file. `core:i18n` requires it to be parsed properly. */ - filename = "nl_NL-qt-ts.ts", + filename = "I18N/nl_NL-qt-ts.ts", options = { flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities, @@ -96,7 +85,7 @@ TESTS :: []TEST{ Simple XLiff 1.2 file. `core:i18n` requires it to be parsed properly. */ - filename = "nl_NL-xliff-1.2.xliff", + filename = "I18N/nl_NL-xliff-1.2.xliff", options = { flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities, @@ -111,7 +100,7 @@ TESTS :: []TEST{ Simple XLiff 2.0 file. `core:i18n` requires it to be parsed properly. */ - filename = "nl_NL-xliff-2.0.xliff", + filename = "I18N/nl_NL-xliff-2.0.xliff", options = { flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities, @@ -122,7 +111,7 @@ TESTS :: []TEST{ }, { - filename = "entities.html", + filename = "XML/entities.html", options = { flags = { .Ignore_Unsupported, .Intern_Comments, @@ -133,7 +122,7 @@ TESTS :: []TEST{ }, { - filename = "entities.html", + filename = "XML/entities.html", options = { flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, @@ -144,7 +133,7 @@ TESTS :: []TEST{ }, { - filename = "entities.html", + filename = "XML/entities.html", options = { flags = { .Ignore_Unsupported, .Intern_Comments, .Unbox_CDATA, .Decode_SGML_Entities, @@ -158,7 +147,7 @@ TESTS :: []TEST{ Then we test that certain errors are returned as expected. */ { - filename = "utf8.xml", + filename = "XML/utf8.xml", options = { flags = { .Ignore_Unsupported, .Intern_Comments, @@ -173,7 +162,7 @@ TESTS :: []TEST{ Parse the 8.2 MiB unicode.xml for good measure. */ { - filename = "unicode.xml", + filename = "XML/unicode.xml", options = { flags = { .Ignore_Unsupported, From 09e1c0fa27a262d7fbfaa5b3e305054304847e75 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 16:19:13 +0200 Subject: [PATCH 75/79] [i18n] Add tests. --- core/text/i18n/doc.odin | 111 ++++++++++++ core/text/i18n/example/i18n_example.odin | 99 ----------- core/text/i18n/gettext.odin | 9 +- core/text/i18n/i18n.odin | 8 + core/text/i18n/qt_linguist.odin | 8 +- examples/all/all_main.odin | 4 + tests/core/Makefile | 5 +- tests/core/assets/I18N/duplicate-key.ts | 22 +++ tests/core/build.bat | 7 +- tests/core/text/i18n/test_core_text_i18n.odin | 165 ++++++++++++++++++ 10 files changed, 329 insertions(+), 109 deletions(-) create mode 100644 core/text/i18n/doc.odin delete mode 100644 core/text/i18n/example/i18n_example.odin create mode 100644 tests/core/assets/I18N/duplicate-key.ts create mode 100644 tests/core/text/i18n/test_core_text_i18n.odin diff --git a/core/text/i18n/doc.odin b/core/text/i18n/doc.odin new file mode 100644 index 000000000..cff1ce11f --- /dev/null +++ b/core/text/i18n/doc.odin @@ -0,0 +1,111 @@ +//+ignore +package i18n + +/* + The i18n package is flexible and easy to use. + + It has one call to get a translation: `get`, which the user can alias into something like `T`. + + `get`, referred to as `T` here, has a few different signatures. + All of them will return the key if the entry can't be found in the active translation catalog. + + - `T(key)` returns the translation of `key`. + - `T(key, n)` returns a pluralized translation of `key` according to value `n`. + + - `T(section, key)` returns the translation of `key` in `section`. + - `T(section, key, n)` returns a pluralized translation of `key` in `section` according to value `n`. + + By default lookup take place in the global `i18n.ACTIVE` catalog for ease of use. + If you want to override which translation to use, for example in a language preview dialog, you can use the following: + + - `T(key, n, catalog)` returns the pluralized version of `key` from explictly supplied catalog. + - `T(section, key, n, catalog)` returns the pluralized version of `key` in `section` from explictly supplied catalog. + + If a catalog has translation contexts or sections, then ommitting it in the above calls looks up in section "". + + The default pluralization rule is n != 1, which is to say that passing n == 1 (or not passing n) returns the singular form. + Passing n != 1 returns plural form 1. + + Should a language not conform to this rule, you can pass a pluralizer procedure to the catalog parser. + This is a procedure that maps an integer to an integer, taking a value and returning which plural slot should be used. + + You can also assign it to a loaded catalog after parsing, of course. + + Some code examples follow. +*/ + +/* +```cpp +import "core:fmt" +import "core:text/i18n" + +T :: i18n.get + +mo :: proc() { + using fmt + + err: i18n.Error + + /* + Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter. + */ + i18n.ACTIVE, err = i18n.parse_mo(#load("translations/nl_NL.mo")) + defer i18n.destroy() + + if err != .None { return } + + /* + These are in the .MO catalog. + */ + println("-----") + println(T("")) + println("-----") + println(T("There are 69,105 leaves here.")) + println("-----") + println(T("Hellope, World!")) + println("-----") + // We pass 1 into `T` to get the singular format string, then 1 again into printf. + printf(T("There is %d leaf.\n", 1), 1) + // We pass 42 into `T` to get the plural format string, then 42 again into printf. + printf(T("There is %d leaf.\n", 42), 42) + + /* + This isn't in the translation catalog, so the key is passed back untranslated. + */ + println("-----") + println(T("Come visit us on Discord!")) +} + +qt :: proc() { + using fmt + + err: i18n.Error + + /* + Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter. + */ + i18n.ACTIVE, err = i18n.parse_qt(#load("translations/nl_NL-qt-ts.ts")) + defer i18n.destroy() + + if err != .None { + return + } + + /* + These are in the .TS catalog. As you can see they have sections. + */ + println("--- Page section ---") + println("Page:Text for translation =", T("Page", "Text for translation")) + println("-----") + println("Page:Also text to translate =", T("Page", "Also text to translate")) + println("-----") + println("--- installscript section ---") + println("installscript:99 bottles of beer on the wall =", T("installscript", "99 bottles of beer on the wall")) + println("-----") + println("--- apple_count section ---") + println("apple_count:%d apple(s) =") + println("\t 1 =", T("apple_count", "%d apple(s)", 1)) + println("\t 42 =", T("apple_count", "%d apple(s)", 42)) +} +``` +*/ \ No newline at end of file diff --git a/core/text/i18n/example/i18n_example.odin b/core/text/i18n/example/i18n_example.odin deleted file mode 100644 index 32eb38a7d..000000000 --- a/core/text/i18n/example/i18n_example.odin +++ /dev/null @@ -1,99 +0,0 @@ -package i18n_example - -import "core:mem" -import "core:fmt" -import "core:text/i18n" - -_T :: i18n.get - -mo :: proc() { - using fmt - - err: i18n.Error - - /* - Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter. - */ - i18n.ACTIVE, err = i18n.parse_mo(#load("nl_NL.mo")) - defer i18n.destroy() - - if err != .None { return } - - /* - These are in the .MO catalog. - */ - println("-----") - println(_T("")) - println("-----") - println(_T("There are 69,105 leaves here.")) - println("-----") - println(_T("Hellope, World!")) - - /* - For ease of use, pluralized lookup can use both singular and plural form as key for the same translation. - This is a quirk of the GetText format which has separate keys for their different plurals. - */ - println("-----") - printf(_T("There is %d leaf.\n", 1), 1) - printf(_T("There is %d leaf.\n", 42), 42) - - printf(_T("There are %d leaves.\n", 1), 1) - printf(_T("There are %d leaves.\n", 42), 42) - - /* - This isn't. - */ - println("-----") - println(_T("Come visit us on Discord!")) -} - -qt :: proc() { - using fmt - - err: i18n.Error - - /* - Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter. - */ - i18n.ACTIVE, err = i18n.parse_qt(#load("../../../../tests/core/assets/XML/nl_NL-qt-ts.ts")) - defer i18n.destroy() - - fmt.printf("parse_qt returned %v\n", err) - if err != .None { - return - } - - /* - These are in the .TS catalog. - */ - println("--- Page section ---") - println("Page:Text for translation =", _T("Page", "Text for translation")) - println("-----") - println("Page:Also text to translate =", _T("Page", "Also text to translate")) - println("-----") - println("--- installscript section ---") - println("installscript:99 bottles of beer on the wall =", _T("installscript", "99 bottles of beer on the wall")) - println("-----") - println("--- apple_count section ---") - println("apple_count:%d apple(s) =") - println("\t 1 =", _T("apple_count", "%d apple(s)", 1)) - println("\t 42 =", _T("apple_count", "%d apple(s)", 42)) -} - -main :: proc() { - using fmt - - track: mem.Tracking_Allocator - mem.tracking_allocator_init(&track, context.allocator) - context.allocator = mem.tracking_allocator(&track) - - // mo() - qt() - - if len(track.allocation_map) > 0 { - println() - for _, v in track.allocation_map { - printf("%v Leaked %v bytes.\n", v.location, v.size) - } - } -} \ No newline at end of file diff --git a/core/text/i18n/gettext.odin b/core/text/i18n/gettext.odin index 54c5a1111..eed73855b 100644 --- a/core/text/i18n/gettext.odin +++ b/core/text/i18n/gettext.odin @@ -8,6 +8,9 @@ package i18n A from-scratch implementation based after the specification found here: https://www.gnu.org/software/gettext/manual/html_node/MO-Files.html + Options are ignored as they're not applicable to this format. + They're part of the signature for consistency with other catalog formats. + List of contributors: Jeroen van Rijn: Initial implementation. */ @@ -15,7 +18,7 @@ import "core:os" import "core:strings" import "core:bytes" -parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { +parse_mo_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { context.allocator = allocator /* An MO file should have at least a 4-byte magic, 2 x 2 byte version info, @@ -115,7 +118,7 @@ parse_mo_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allo return } -parse_mo_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { +parse_mo_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { context.allocator = allocator data, data_ok := os.read_entire_file(filename) @@ -123,7 +126,7 @@ parse_mo_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allo if !data_ok { return {}, .File_Error } - return parse_mo_from_slice(data, pluralizer) + return parse_mo_from_slice(data, options, pluralizer, allocator) } parse_mo :: proc { parse_mo_file, parse_mo_from_slice } diff --git a/core/text/i18n/i18n.odin b/core/text/i18n/i18n.odin index 36204efd9..e007401af 100644 --- a/core/text/i18n/i18n.odin +++ b/core/text/i18n/i18n.odin @@ -74,6 +74,14 @@ Error :: enum { } +Parse_Options :: struct { + merge_sections: bool, +} + +DEFAULT_PARSE_OPTIONS :: Parse_Options{ + merge_sections = false, +} + /* Several ways to use: - get(key), which defaults to the singular form and i18n.ACTIVE catalog, or diff --git a/core/text/i18n/qt_linguist.odin b/core/text/i18n/qt_linguist.odin index 65d51444e..0a241c1aa 100644 --- a/core/text/i18n/qt_linguist.odin +++ b/core/text/i18n/qt_linguist.odin @@ -27,7 +27,7 @@ TS_XML_Options := xml.Options{ expected_doctype = "TS", } -parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { +parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { context.allocator = allocator ts, xml_err := xml.parse(data, TS_XML_Options) @@ -59,7 +59,7 @@ parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = return translation, .TS_File_Expected_Context_Name, } - section_name := ts.elements[section_name_id].value + section_name := "" if options.merge_sections else ts.elements[section_name_id].value if section_name not_in translation.k_v { translation.k_v[section_name] = {} @@ -139,7 +139,7 @@ parse_qt_linguist_from_slice :: proc(data: []u8, pluralizer: proc(int) -> int = return } -parse_qt_linguist_file :: proc(filename: string, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { +parse_qt_linguist_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) { context.allocator = allocator data, data_ok := os.read_entire_file(filename) @@ -147,7 +147,7 @@ parse_qt_linguist_file :: proc(filename: string, pluralizer: proc(int) -> int = if !data_ok { return {}, .File_Error } - return parse_qt_linguist_from_slice(data, pluralizer) + return parse_qt_linguist_from_slice(data, options, pluralizer, allocator) } parse_qt :: proc { parse_qt_linguist_file, parse_qt_linguist_from_slice } \ No newline at end of file diff --git a/examples/all/all_main.odin b/examples/all/all_main.odin index 27f199062..36acf7714 100644 --- a/examples/all/all_main.odin +++ b/examples/all/all_main.odin @@ -56,6 +56,7 @@ import csv "core:encoding/csv" import hxa "core:encoding/hxa" import json "core:encoding/json" import varint "core:encoding/varint" +import xml "core:encoding/xml" import fmt "core:fmt" import hash "core:hash" @@ -100,6 +101,7 @@ import strings "core:strings" import sync "core:sync" import testing "core:testing" import scanner "core:text/scanner" +import i18n "core:text/i18n" import thread "core:thread" import time "core:time" @@ -158,6 +160,7 @@ _ :: csv _ :: hxa _ :: json _ :: varint +_ :: xml _ :: fmt _ :: hash _ :: image @@ -192,6 +195,7 @@ _ :: strings _ :: sync _ :: testing _ :: scanner +_ :: i18n _ :: thread _ :: time _ :: unicode diff --git a/tests/core/Makefile b/tests/core/Makefile index 2c24fef75..1405ae5c6 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -26,9 +26,10 @@ noise_test: $(ODIN) run math/noise -out:test_noise encoding_test: - $(ODIN) run encoding/hxa -collection:tests=.. -out:test_hxa - $(ODIN) run encoding/json -out:test_json + $(ODIN) run encoding/hxa -out:test_hxa -collection:tests=.. + $(ODIN) run encoding/json -out:test_json $(ODIN) run encoding/varint -out:test_varint + $(ODIN) run encoding/xml -out:test_xml math_test: $(ODIN) run math/test_core_math.odin -file -collection:tests=.. -out:test_core_math diff --git a/tests/core/assets/I18N/duplicate-key.ts b/tests/core/assets/I18N/duplicate-key.ts new file mode 100644 index 000000000..a38824d01 --- /dev/null +++ b/tests/core/assets/I18N/duplicate-key.ts @@ -0,0 +1,22 @@ + + + + + Page + + %d apple(s) + commenting + Tekst om te vertalen + + + + apple_count + + %d apple(s) + + %d appel + %d appels + + + + diff --git a/tests/core/build.bat b/tests/core/build.bat index 8e4ba1d15..77ff38038 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -64,4 +64,9 @@ echo --- echo --- echo Running core:reflect tests echo --- -%PATH_TO_ODIN% run reflect %COMMON% %COLLECTION% -out:test_core_reflect.exe \ No newline at end of file +%PATH_TO_ODIN% run reflect %COMMON% %COLLECTION% -out:test_core_reflect.exe + +echo --- +echo Running core:text/i18n tests +echo --- +%PATH_TO_ODIN% run text\i18n %COMMON% -out:test_core_i18n.exe \ No newline at end of file diff --git a/tests/core/text/i18n/test_core_text_i18n.odin b/tests/core/text/i18n/test_core_text_i18n.odin new file mode 100644 index 000000000..ba668c4fd --- /dev/null +++ b/tests/core/text/i18n/test_core_text_i18n.odin @@ -0,0 +1,165 @@ +package test_core_text_i18n + +import "core:mem" +import "core:fmt" +import "core:os" +import "core:testing" +import "core:text/i18n" + +TEST_count := 0 +TEST_fail := 0 + +when ODIN_TEST { + expect :: testing.expect + log :: testing.log +} else { + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] ", loc) + fmt.printf("log: %v\n", v) + } +} +T :: i18n.get + +Test :: struct { + section: string, + key: string, + val: string, + n: int, +} + +Test_Suite :: struct { + file: string, + loader: proc(string, i18n.Parse_Options, proc(int) -> int, mem.Allocator) -> (^i18n.Translation, i18n.Error), + err: i18n.Error, + options: i18n.Parse_Options, + tests: []Test, +} + +TESTS := []Test_Suite{ + { + file = "assets/I18N/nl_NL.mo", + loader = i18n.parse_mo_file, + tests = { + // These are in the catalog. + { "", "There are 69,105 leaves here.", "Er zijn hier 69.105 bladeren.", 1 }, + { "", "Hellope, World!", "Hallo, Wereld!", 1 }, + { "", "There is %d leaf.\n", "Er is %d blad.\n", 1 }, + { "", "There are %d leaves.\n", "Er is %d blad.\n", 1 }, + { "", "There is %d leaf.\n", "Er zijn %d bladeren.\n", 42 }, + { "", "There are %d leaves.\n", "Er zijn %d bladeren.\n", 42 }, + + // This isn't in the catalog, so should ruturn the key. + { "", "Come visit us on Discord!", "Come visit us on Discord!", 1 }, + }, + }, + + // QT Linguist with default loader options. + { + file = "assets/I18N/nl_NL-qt-ts.ts", + loader = i18n.parse_qt_linguist_file, + tests = { + // These are in the catalog. + { "Page", "Text for translation", "Tekst om te vertalen", 1}, + { "Page", "Also text to translate", "Ook tekst om te vertalen", 1}, + { "installscript", "99 bottles of beer on the wall", "99 flessen bier op de muur", 1}, + { "apple_count", "%d apple(s)", "%d appel", 1}, + { "apple_count", "%d apple(s)", "%d appels", 42}, + + // These aren't in the catalog, so should ruturn the key. + { "", "Come visit us on Discord!", "Come visit us on Discord!", 1 }, + { "Fake_Section", "Come visit us on Discord!", "Come visit us on Discord!", 1 }, + }, + }, + + // QT Linguist, merging sections. + { + file = "assets/I18N/nl_NL-qt-ts.ts", + loader = i18n.parse_qt_linguist_file, + options = {merge_sections = true}, + tests = { + // All of them are now in section "", lookup with original section should return the key. + { "", "Text for translation", "Tekst om te vertalen", 1}, + { "", "Also text to translate", "Ook tekst om te vertalen", 1}, + { "", "99 bottles of beer on the wall", "99 flessen bier op de muur", 1}, + { "", "%d apple(s)", "%d appel", 1}, + { "", "%d apple(s)", "%d appels", 42}, + + // All of them are now in section "", lookup with original section should return the key. + { "Page", "Text for translation", "Text for translation", 1}, + { "Page", "Also text to translate", "Also text to translate", 1}, + { "installscript", "99 bottles of beer on the wall", "99 bottles of beer on the wall", 1}, + { "apple_count", "%d apple(s)", "%d apple(s)", 1}, + { "apple_count", "%d apple(s)", "%d apple(s)", 42}, + }, + }, + + // QT Linguist, merging sections. Expecting .Duplicate_Key error because same key exists in more than 1 section. + { + file = "assets/I18N/duplicate-key.ts", + loader = i18n.parse_qt_linguist_file, + options = {merge_sections = true}, + err = .Duplicate_Key, + }, + + // QT Linguist, not merging sections. Shouldn't return error despite same key existing in more than 1 section. + { + file = "assets/I18N/duplicate-key.ts", + loader = i18n.parse_qt_linguist_file, + }, +} + +@test +tests :: proc(t: ^testing.T) { + using fmt + + cat: ^i18n.Translation + err: i18n.Error + + for suite in TESTS { + cat, err = suite.loader(suite.file, suite.options, nil, context.allocator) + + msg := fmt.tprintf("Expected loading %v to return %v, got %v", suite.file, suite.err, err) + expect(t, err == suite.err, msg) + + if err == .None { + for test in suite.tests { + val := T(test.section, test.key, test.n, cat) + + msg = fmt.tprintf("Expected key `%v` from section `%v`'s form for value `%v` to equal `%v`, got `%v`", test.key, test.section, test.n, test.val, val) + expect(t, val == test.val, msg) + } + } + i18n.destroy(cat) + } +} + +main :: proc() { + using fmt + + track: mem.Tracking_Allocator + mem.tracking_allocator_init(&track, context.allocator) + context.allocator = mem.tracking_allocator(&track) + + t := testing.T{} + tests(&t) + + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + if TEST_fail > 0 { + os.exit(1) + } + + if len(track.allocation_map) > 0 { + println() + for _, v in track.allocation_map { + printf("%v Leaked %v bytes.\n", v.location, v.size) + } + } +} \ No newline at end of file From ff0f0c447fb52b5836edb603e1fb8c3ae4ed4b50 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 18:10:13 +0200 Subject: [PATCH 76/79] [i18n] Fix segfault on destroy on Linux Forgot to intern the section string in QT TS loader. --- core/text/i18n/i18n.odin | 26 ++++++++++++++----------- core/text/i18n/qt_linguist.odin | 5 ++++- tests/core/assets/I18N/duplicate-key.ts | 18 ++++++++--------- 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/core/text/i18n/i18n.odin b/core/text/i18n/i18n.odin index e007401af..9d030db16 100644 --- a/core/text/i18n/i18n.odin +++ b/core/text/i18n/i18n.odin @@ -163,16 +163,20 @@ get_by_slot :: proc{get_by_slot_single_section, get_by_slot_by_section} - destroy(), to clean up the currently active catalog catalog i18n.ACTIVE - destroy(catalog), to clean up a specific catalog. */ -destroy :: proc(catalog: ^Translation = ACTIVE) { - if catalog != nil { - strings.intern_destroy(&catalog.intern) - for section in &catalog.k_v { - for key in &catalog.k_v[section] { - delete(catalog.k_v[section][key]) - } - delete(catalog.k_v[section]) - } - delete(catalog.k_v) - free(catalog) +destroy :: proc(catalog: ^Translation = ACTIVE, allocator := context.allocator) { + context.allocator = allocator + + if catalog == nil { + return } + + for section in &catalog.k_v { + for key in &catalog.k_v[section] { + delete(catalog.k_v[section][key]) + } + delete(catalog.k_v[section]) + } + delete(catalog.k_v) + strings.intern_destroy(&catalog.intern) + free(catalog) } \ No newline at end of file diff --git a/core/text/i18n/qt_linguist.odin b/core/text/i18n/qt_linguist.odin index 0a241c1aa..e23e9aef4 100644 --- a/core/text/i18n/qt_linguist.odin +++ b/core/text/i18n/qt_linguist.odin @@ -59,7 +59,10 @@ parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTION return translation, .TS_File_Expected_Context_Name, } - section_name := "" if options.merge_sections else ts.elements[section_name_id].value + section_name := strings.intern_get(&translation.intern, "") + if !options.merge_sections { + section_name = strings.intern_get(&translation.intern, ts.elements[section_name_id].value) + } if section_name not_in translation.k_v { translation.k_v[section_name] = {} diff --git a/tests/core/assets/I18N/duplicate-key.ts b/tests/core/assets/I18N/duplicate-key.ts index a38824d01..44c09d91d 100644 --- a/tests/core/assets/I18N/duplicate-key.ts +++ b/tests/core/assets/I18N/duplicate-key.ts @@ -10,13 +10,13 @@ - apple_count - - %d apple(s) - - %d appel - %d appels - - - + apple_count + + %d apple(s) + + %d appel + %d appels + + + From 36263399a0159d9ca3a109b50542638264e5642b Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 18:13:46 +0200 Subject: [PATCH 77/79] [i18n] Enable i18n test on Linux. --- tests/core/Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/core/Makefile b/tests/core/Makefile index 1405ae5c6..5c2918e30 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -2,7 +2,7 @@ ODIN=../../odin PYTHON=$(shell which python3) all: download_test_assets image_test compress_test strings_test hash_test crypto_test noise_test encoding_test \ - math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test + math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test i18n_test download_test_assets: $(PYTHON) download_assets.py @@ -44,4 +44,7 @@ reflect_test: $(ODIN) run reflect/test_core_reflect.odin -file -collection:tests=.. -out:test_core_reflect os_exit_test: - $(ODIN) run os/test_core_os_exit.odin -file -out:test_core_os_exit && exit 1 || exit 0 \ No newline at end of file + $(ODIN) run os/test_core_os_exit.odin -file -out:test_core_os_exit && exit 1 || exit 0 + +i18n_test: + $(ODIN) run text/i18n -out:test_core_i18n \ No newline at end of file From 819345caa6dccb983e496682a78829074f777011 Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 18:28:42 +0200 Subject: [PATCH 78/79] Disable i18n test for *nix for now. --- tests/core/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/Makefile b/tests/core/Makefile index 5c2918e30..2e1e4f78d 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -2,7 +2,7 @@ ODIN=../../odin PYTHON=$(shell which python3) all: download_test_assets image_test compress_test strings_test hash_test crypto_test noise_test encoding_test \ - math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test i18n_test + math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test download_test_assets: $(PYTHON) download_assets.py From 9c9c2b483c93be2f6e78911d997229a4b543c66d Mon Sep 17 00:00:00 2001 From: Jeroen van Rijn Date: Fri, 29 Apr 2022 18:41:21 +0200 Subject: [PATCH 79/79] [i18n] Enable *nix tests again. --- core/text/i18n/qt_linguist.odin | 18 +++++++++--------- tests/core/Makefile | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/text/i18n/qt_linguist.odin b/core/text/i18n/qt_linguist.odin index e23e9aef4..15a88a42f 100644 --- a/core/text/i18n/qt_linguist.odin +++ b/core/text/i18n/qt_linguist.odin @@ -92,15 +92,15 @@ parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTION return translation, .TS_File_Expected_Translation } - source := ts.elements[source_id] - xlat := ts.elements[translation_id] + source := strings.intern_get(&translation.intern, ts.elements[source_id].value) + xlat := strings.intern_get(&translation.intern, ts.elements[translation_id].value) - if source.value in section { + if source in section { return translation, .Duplicate_Key } if has_plurals { - if xlat.value != "" { + if xlat != "" { return translation, .TS_File_Expected_NumerusForm } @@ -116,7 +116,7 @@ parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTION if num_plurals < 2 { return translation, .TS_File_Expected_NumerusForm } - section[source.value] = make([]string, num_plurals) + section[source] = make([]string, num_plurals) num_plurals = 0 for { @@ -124,15 +124,15 @@ parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTION if !numerus_found { break } - numerus := ts.elements[numerus_id] - section[source.value][num_plurals] = strings.intern_get(&translation.intern, numerus.value) + numerus := strings.intern_get(&translation.intern, ts.elements[numerus_id].value) + section[source][num_plurals] = numerus num_plurals += 1 } } else { // Single translation - section[source.value] = make([]string, 1) - section[source.value][0] = strings.intern_get(&translation.intern, xlat.value) + section[source] = make([]string, 1) + section[source][0] = xlat } nth += 1 diff --git a/tests/core/Makefile b/tests/core/Makefile index 2e1e4f78d..5c2918e30 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -2,7 +2,7 @@ ODIN=../../odin PYTHON=$(shell which python3) all: download_test_assets image_test compress_test strings_test hash_test crypto_test noise_test encoding_test \ - math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test + math_test linalg_glsl_math_test filepath_test reflect_test os_exit_test i18n_test download_test_assets: $(PYTHON) download_assets.py