Merge pull request #1336 from Kelimion/i18n

[i18n] Initial i18n support.

core/encoding/xml/xml_reader.odin

@@ -87,7 +87,6 @@ Option_Flag :: enum {
 		If a tag body has a comment, it will be stripped unless this option is given.
 	*/
 	Keep_Tag_Body_Comments,
-
 }
 Option_Flags :: bit_set[Option_Flag; u16]
 

core/text/i18n/doc.odin

@@ -0,0 +1,111 @@
+//+ignore
+package i18n
+
+/*
+	The i18n package is flexible and easy to use.
+
+	It has one call to get a translation: `get`, which the user can alias into something like `T`.
+
+	`get`, referred to as `T` here, has a few different signatures.
+	All of them will return the key if the entry can't be found in the active translation catalog.
+
+	- `T(key)`              returns the translation of `key`.
+	- `T(key, n)`           returns a pluralized translation of `key` according to value `n`.
+
+	- `T(section, key)`     returns the translation of `key` in `section`.
+	- `T(section, key, n)`  returns a pluralized translation of `key` in `section` according to value `n`.
+
+	By default lookup take place in the global `i18n.ACTIVE` catalog for ease of use.
+	If you want to override which translation to use, for example in a language preview dialog, you can use the following:
+
+	- `T(key, n, catalog)`           returns the pluralized version of `key` from explictly supplied catalog.
+	- `T(section, key, n, catalog)`  returns the pluralized version of `key` in `section` from explictly supplied catalog.
+
+	If a catalog has translation contexts or sections, then ommitting it in the above calls looks up in section "".
+
+	The default pluralization rule is n != 1, which is to say that passing n == 1 (or not passing n) returns the singular form.
+	Passing n != 1 returns plural form 1.
+
+	Should a language not conform to this rule, you can pass a pluralizer procedure to the catalog parser.
+	This is a procedure that maps an integer to an integer, taking a value and returning which plural slot should be used.
+
+	You can also assign it to a loaded catalog after parsing, of course.
+
+	Some code examples follow.
+*/
+
+/*
+```cpp
+import "core:fmt"
+import "core:text/i18n"
+
+T :: i18n.get
+
+mo :: proc() {
+	using fmt
+
+	err: i18n.Error
+
+	/*
+		Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter.
+	*/
+	i18n.ACTIVE, err = i18n.parse_mo(#load("translations/nl_NL.mo"))
+	defer i18n.destroy()
+
+	if err != .None { return }
+
+	/*
+		These are in the .MO catalog.
+	*/
+	println("-----")
+	println(T(""))
+	println("-----")
+	println(T("There are 69,105 leaves here."))
+	println("-----")
+	println(T("Hellope, World!"))
+	println("-----")
+	// We pass 1 into `T` to get the singular format string, then 1 again into printf.
+	printf(T("There is %d leaf.\n", 1), 1)
+	// We pass 42 into `T` to get the plural format string, then 42 again into printf.
+	printf(T("There is %d leaf.\n", 42), 42)
+
+	/*
+		This isn't in the translation catalog, so the key is passed back untranslated.
+	*/
+	println("-----")
+	println(T("Come visit us on Discord!"))
+}
+
+qt :: proc() {
+	using fmt
+
+	err: i18n.Error
+
+	/*
+		Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter.
+	*/
+	i18n.ACTIVE, err = i18n.parse_qt(#load("translations/nl_NL-qt-ts.ts"))
+	defer i18n.destroy()
+
+	if err != .None {
+		return
+	}
+
+	/*
+		These are in the .TS catalog. As you can see they have sections.
+	*/
+	println("--- Page section ---")
+	println("Page:Text for translation =", T("Page", "Text for translation"))
+	println("-----")
+	println("Page:Also text to translate =", T("Page", "Also text to translate"))
+	println("-----")
+	println("--- installscript section ---")
+	println("installscript:99 bottles of beer on the wall =", T("installscript", "99 bottles of beer on the wall"))
+	println("-----")
+	println("--- apple_count section ---")
+	println("apple_count:%d apple(s) =")
+	println("\t 1  =", T("apple_count", "%d apple(s)", 1))
+	println("\t 42 =", T("apple_count", "%d apple(s)", 42))
+}
+```
+*/

core/text/i18n/gettext.odin

@@ -0,0 +1,167 @@
+package i18n
+/*
+	A parser for GNU GetText .MO files.
+
+	Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
+	Made available under Odin's BSD-3 license.
+
+	A from-scratch implementation based after the specification found here:
+		https://www.gnu.org/software/gettext/manual/html_node/MO-Files.html
+
+	Options are ignored as they're not applicable to this format.
+	They're part of the signature for consistency with other catalog formats.
+
+	List of contributors:
+		Jeroen van Rijn: Initial implementation.
+*/
+import "core:os"
+import "core:strings"
+import "core:bytes"
+
+parse_mo_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
+	context.allocator = allocator
+	/*
+		An MO file should have at least a 4-byte magic, 2 x 2 byte version info,
+		a 4-byte number of strings value, and 2 x 4-byte offsets.
+	*/
+	if len(data) < 20 {
+		return {}, .MO_File_Invalid
+	}
+
+	/*
+		Check magic. Should be 0x950412de in native Endianness.
+	*/
+	native := true
+	magic  := read_u32(data, native) or_return
+
+	if magic != 0x950412de {
+		native = false
+		magic = read_u32(data, native) or_return
+
+		if magic != 0x950412de { return {}, .MO_File_Invalid_Signature }
+	}
+
+	/*
+		We can ignore version_minor at offset 6.
+	*/
+	version_major := read_u16(data[4:]) or_return
+	if version_major > 1 { return {}, .MO_File_Unsupported_Version }
+
+	count             := read_u32(data[ 8:]) or_return
+	original_offset   := read_u32(data[12:]) or_return
+	translated_offset := read_u32(data[16:]) or_return
+
+	if count == 0 { return {}, .Empty_Translation_Catalog }
+
+	/*
+		Initalize Translation, interner and optional pluralizer.
+	*/
+	translation = new(Translation)
+	translation.pluralize = pluralizer
+	strings.intern_init(&translation.intern, allocator, allocator)
+
+	// Gettext MO files only have one section.
+	translation.k_v[""] = {}
+	section := &translation.k_v[""]
+
+	for n := u32(0); n < count; n += 1 {
+		/*
+			Grab string's original length and offset.
+		*/
+		offset := original_offset + 8 * n
+		if len(data) < int(offset + 8) { return translation, .MO_File_Invalid }
+
+		o_length := read_u32(data[offset    :], native) or_return
+		o_offset := read_u32(data[offset + 4:], native) or_return
+
+		offset = translated_offset + 8 * n
+		if len(data) < int(offset + 8) { return translation, .MO_File_Invalid }
+
+		t_length := read_u32(data[offset    :], native) or_return
+		t_offset := read_u32(data[offset + 4:], native) or_return
+
+		max_offset := int(max(o_offset + o_length + 1, t_offset + t_length + 1))
+		if len(data) < max_offset { return translation, .Premature_EOF }
+
+		key := data[o_offset:][:o_length]
+		val := data[t_offset:][:t_length]
+
+		/*
+			Could be a pluralized string.
+		*/
+		zero := []byte{0}
+
+		keys := bytes.split(key, zero)
+		vals := bytes.split(val, zero)
+	
+		if len(keys) != len(vals) || max(len(keys), len(vals)) > MAX_PLURALS {
+			return translation, .MO_File_Incorrect_Plural_Count
+		}
+
+		for k in keys {
+			interned_key := strings.intern_get(&translation.intern, string(k))
+
+			interned_vals := make([]string, len(keys))
+			last_val: string
+
+			i := 0
+			for v in vals {
+				interned_vals[i] = strings.intern_get(&translation.intern, string(v))
+				last_val = interned_vals[i]
+				i += 1
+			}
+			section[interned_key] = interned_vals
+		}
+		delete(vals)
+		delete(keys)
+	}
+	return
+}
+
+parse_mo_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
+	context.allocator = allocator
+
+	data, data_ok := os.read_entire_file(filename)
+	defer delete(data)
+
+	if !data_ok { return {}, .File_Error }
+
+	return parse_mo_from_slice(data, options, pluralizer, allocator)
+}
+
+parse_mo :: proc { parse_mo_file, parse_mo_from_slice }
+
+/*
+	Helpers.
+*/
+read_u32 :: proc(data: []u8, native_endian := true) -> (res: u32, err: Error) {
+	if len(data) < size_of(u32) { return 0, .Premature_EOF }
+
+	val := (^u32)(raw_data(data))^
+
+	if native_endian {
+		return val, .None
+	} else {
+		when ODIN_ENDIAN == .Little {
+			return u32(transmute(u32be)val), .None
+		} else {
+			return u32(transmute(u32le)val), .None
+		}
+	}
+}
+
+read_u16 :: proc(data: []u8, native_endian := true) -> (res: u16, err: Error) {
+	if len(data) < size_of(u16) { return 0, .Premature_EOF }
+
+	val := (^u16)(raw_data(data))^
+
+	if native_endian {
+		return val, .None
+	} else {
+		when ODIN_ENDIAN == .Little {
+			return u16(transmute(u16be)val), .None
+		} else {
+			return u16(transmute(u16le)val), .None
+		}
+	}
+}

core/text/i18n/i18n.odin

@@ -0,0 +1,178 @@
+package i18n
+/*
+	Internationalization helpers.
+
+	Copyright 2021-2022 Jeroen van Rijn <nom@duclavier.com>.
+	Made available under Odin's BSD-3 license.
+
+	List of contributors:
+		Jeroen van Rijn: Initial implementation.
+*/
+import "core:strings"
+
+/*
+	TODO:
+	- Support for more translation catalog file formats.
+*/
+
+/*
+	Currently active catalog.
+*/
+ACTIVE: ^Translation
+
+// Allow between 1 and 255 plural forms. Default: 10.
+MAX_PLURALS :: min(max(#config(ODIN_i18N_MAX_PLURAL_FORMS, 10), 1), 255)
+
+/*
+	The main data structure. This can be generated from various different file formats, as long as we have a parser for them.
+*/
+
+Section :: map[string][]string
+
+Translation :: struct {
+	k_v:    map[string]Section, // k_v[section][key][plural_form] = ...
+	intern: strings.Intern,
+
+	pluralize: proc(number: int) -> int,
+}
+
+Error :: enum {
+	/*
+		General return values.
+	*/
+	None = 0,
+	Empty_Translation_Catalog,
+	Duplicate_Key,
+
+	/*
+		Couldn't find, open or read file.
+	*/
+	File_Error,
+
+	/*
+		File too short.
+	*/
+	Premature_EOF,
+
+	/*
+		GNU Gettext *.MO file errors.
+	*/
+	MO_File_Invalid_Signature,
+	MO_File_Unsupported_Version,
+	MO_File_Invalid,
+	MO_File_Incorrect_Plural_Count,
+
+	/*
+		Qt Linguist *.TS file errors.
+	*/
+	TS_File_Parse_Error,
+	TS_File_Expected_Context,
+	TS_File_Expected_Context_Name,
+	TS_File_Expected_Source,
+	TS_File_Expected_Translation,
+	TS_File_Expected_NumerusForm,
+
+}
+
+Parse_Options :: struct {
+	merge_sections: bool,
+}
+
+DEFAULT_PARSE_OPTIONS :: Parse_Options{
+	merge_sections = false,
+}
+
+/*
+	Several ways to use:
+	- get(key), which defaults to the singular form and i18n.ACTIVE catalog, or
+	- get(key, number), which returns the appropriate plural from the active catalog, or
+	- get(key, number, catalog) to grab text from a specific one.
+*/
+get_single_section :: proc(key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) {
+	/*
+		A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule.
+	*/
+	plural := 1 if number != 1 else 0
+
+	if catalog.pluralize != nil {
+		plural = catalog.pluralize(number)
+	}
+	return get_by_slot(key, plural, catalog)
+}
+
+/*
+	Several ways to use:
+	- get(section, key), which defaults to the singular form and i18n.ACTIVE catalog, or
+	- get(section, key, number), which returns the appropriate plural from the active catalog, or
+	- get(section, key, number, catalog) to grab text from a specific one.
+*/
+get_by_section :: proc(section, key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) {
+	/*
+		A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule.
+	*/
+	plural := 1 if number != 1 else 0
+
+	if catalog.pluralize != nil {
+		plural = catalog.pluralize(number)
+	}
+	return get_by_slot(section, key, plural, catalog)
+}
+get :: proc{get_single_section, get_by_section}
+
+/*
+	Several ways to use:
+	- get_by_slot(key), which defaults to the singular form and i18n.ACTIVE catalog, or
+	- get_by_slot(key, slot), which returns the requested plural from the active catalog, or
+	- get_by_slot(key, slot, catalog) to grab text from a specific one.
+
+	If a file format parser doesn't (yet) support plural slots, each of the slots will point at the same string.
+*/
+get_by_slot_single_section :: proc(key: string, slot := 0, catalog: ^Translation = ACTIVE) -> (value: string) {
+	return get_by_slot_by_section("", key, slot, catalog)
+}
+
+/*
+	Several ways to use:
+	- get_by_slot(key), which defaults to the singular form and i18n.ACTIVE catalog, or
+	- get_by_slot(key, slot), which returns the requested plural from the active catalog, or
+	- get_by_slot(key, slot, catalog) to grab text from a specific one.
+
+	If a file format parser doesn't (yet) support plural slots, each of the slots will point at the same string.
+*/
+get_by_slot_by_section :: proc(section, key: string, slot := 0, catalog: ^Translation = ACTIVE) -> (value: string) {
+	if catalog == nil || section not_in catalog.k_v {
+		/*
+			Return the key if the catalog catalog hasn't been initialized yet, or the section is not present.
+		*/
+		return key
+	}
+
+	/*
+		Return the translation from the requested slot if this key is known, else return the key.
+	*/
+	if translations, ok := catalog.k_v[section][key]; ok {
+		plural := min(max(0, slot), len(catalog.k_v[section][key]) - 1)
+		return translations[plural]
+	}
+	return key
+}
+get_by_slot :: proc{get_by_slot_single_section, get_by_slot_by_section}
+
+/*
+	Same for destroy:
+	- destroy(), to clean up the currently active catalog catalog i18n.ACTIVE
+	- destroy(catalog), to clean up a specific catalog.
+*/
+destroy :: proc(catalog: ^Translation = ACTIVE) {
+	if catalog != nil {
+		strings.intern_destroy(&catalog.intern)
+		for section in &catalog.k_v {
+			for key in &catalog.k_v[section] {
+				delete(catalog.k_v[section][key])
+			}
+			delete(catalog.k_v[section])
+		}
+		delete(catalog.k_v)
+		free(catalog)
+	}
+}

core/text/i18n/qt_linguist.odin

@@ -0,0 +1,153 @@
+package i18n
+/*
+	A parser for Qt Linguist TS files.
+
+	Copyright 2022 Jeroen van Rijn <nom@duclavier.com>.
+	Made available under Odin's BSD-3 license.
+
+	A from-scratch implementation based after the specification found here:
+		https://doc.qt.io/qt-5/linguist-ts-file-format.html
+
+	List of contributors:
+		Jeroen van Rijn: Initial implementation.
+*/
+import "core:os"
+import "core:encoding/xml"
+import "core:strings"
+
+TS_XML_Options := xml.Options{
+	flags = {
+		.Input_May_Be_Modified,
+		.Must_Have_Prolog,
+		.Must_Have_DocType,
+		.Ignore_Unsupported,
+		.Unbox_CDATA,
+		.Decode_SGML_Entities,
+	},
+	expected_doctype = "TS",
+}
+
+parse_qt_linguist_from_slice :: proc(data: []u8, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
+	context.allocator = allocator
+
+	ts, xml_err := xml.parse(data, TS_XML_Options)
+	defer xml.destroy(ts)
+
+	if xml_err != .None || ts.element_count < 1 || ts.elements[0].ident != "TS" || len(ts.elements[0].children) == 0 {
+		return nil, .TS_File_Parse_Error
+	}
+
+	/*
+		Initalize Translation, interner and optional pluralizer.
+	*/
+	translation = new(Translation)
+	translation.pluralize = pluralizer
+	strings.intern_init(&translation.intern, allocator, allocator)
+
+	section: ^Section
+
+	for child_id in ts.elements[0].children {
+		// These should be <context>s.
+		child := ts.elements[child_id]
+		if child.ident != "context" {
+			return translation, .TS_File_Expected_Context
+		}
+
+		// Find section name.
+		section_name_id, section_name_found := xml.find_child_by_ident(ts, child_id, "name")
+		if !section_name_found {
+			return translation, .TS_File_Expected_Context_Name,
+		}
+
+		section_name := "" if options.merge_sections else ts.elements[section_name_id].value
+
+		if section_name not_in translation.k_v {
+			translation.k_v[section_name] = {}
+		}
+		section = &translation.k_v[section_name]
+
+		// Find messages in section.
+		nth: int
+		for {
+			message_id, message_found := xml.find_child_by_ident(ts, child_id, "message", nth)
+			if !message_found {
+				break
+			}
+
+			numerus_tag, _ := xml.find_attribute_val_by_key(ts, message_id, "numerus")
+			has_plurals := numerus_tag == "yes"
+
+			// We must have a <source> = key
+			source_id, source_found := xml.find_child_by_ident(ts, message_id, "source")
+			if !source_found {
+				return translation, .TS_File_Expected_Source
+			}
+
+			// We must have a <translation>
+			translation_id, translation_found := xml.find_child_by_ident(ts, message_id, "translation")
+			if !translation_found {
+				return translation, .TS_File_Expected_Translation
+			}
+
+			source := ts.elements[source_id]
+			xlat   := ts.elements[translation_id]
+
+			if source.value in section {
+				return translation, .Duplicate_Key
+			}
+
+			if has_plurals {
+				if xlat.value != "" {
+					return translation, .TS_File_Expected_NumerusForm
+				}
+
+				num_plurals: int
+				for {
+					numerus_id, numerus_found := xml.find_child_by_ident(ts, translation_id, "numerusform", num_plurals)
+					if !numerus_found {
+						break
+					}
+					num_plurals += 1
+				}
+
+				if num_plurals < 2 {
+					return translation, .TS_File_Expected_NumerusForm
+				}
+				section[source.value] = make([]string, num_plurals)
+
+				num_plurals = 0
+				for {
+					numerus_id, numerus_found := xml.find_child_by_ident(ts, translation_id, "numerusform", num_plurals)
+					if !numerus_found {
+						break
+					}
+					numerus := ts.elements[numerus_id]
+					section[source.value][num_plurals] = strings.intern_get(&translation.intern, numerus.value)
+
+					num_plurals += 1
+				}
+			} else {
+				// Single translation
+				section[source.value] = make([]string, 1)
+				section[source.value][0] = strings.intern_get(&translation.intern, xlat.value)
+			}
+
+			nth += 1
+		}
+	}
+
+	return
+}
+
+parse_qt_linguist_file :: proc(filename: string, options := DEFAULT_PARSE_OPTIONS, pluralizer: proc(int) -> int = nil, allocator := context.allocator) -> (translation: ^Translation, err: Error) {
+	context.allocator = allocator
+
+	data, data_ok := os.read_entire_file(filename)
+	defer delete(data)
+
+	if !data_ok { return {}, .File_Error }
+
+	return parse_qt_linguist_from_slice(data, options, pluralizer, allocator)
+}
+
+parse_qt :: proc { parse_qt_linguist_file, parse_qt_linguist_from_slice }