Add decode_grapheme_clusters to core:unicode/utf8

core/unicode/letter.odin

@@ -5,6 +5,9 @@ REPLACEMENT_CHAR :: '\ufffd'     // Represented an invalid code point
 MAX_ASCII        :: '\u007f'     // Maximum ASCII value
 MAX_LATIN1       :: '\u00ff'     // Maximum Latin-1 value
 
+ZERO_WIDTH_NON_JOINER :: '\u200C'
+ZERO_WIDTH_JOINER     :: '\u200D'
+
 binary_search :: proc(c: i32, table: []i32, length, stride: int) -> int {
 	n := length
 	t := 0
@@ -193,3 +196,222 @@ is_symbol :: proc(r: rune) -> bool {
 	}
 	return false
 }
+
+//
+// The procedures below are accurate as of Unicode 15.1.0.
+//
+
+// Emoji_Modifier
+is_emoji_modifier :: proc(r: rune) -> bool {
+	return 0x1F3FB <= r && r <= 0x1F3FF
+}
+
+// Regional_Indicator
+is_regional_indicator :: proc(r: rune) -> bool {
+	return 0x1F1E6 <= r && r <= 0x1F1FF
+}
+
+// General_Category=Enclosing_Mark
+is_enclosing_mark :: proc(r: rune) -> bool {
+	switch r {
+	case 0x0488,
+	     0x0489,
+	     0x1ABE,
+	     0x20DD ..= 0x20E0,
+	     0x20E2 ..= 0x20E4,
+	     0xA670 ..= 0xA672: return true
+	}
+
+	return false
+}
+
+// Prepended_Concatenation_Mark
+is_prepended_concatenation_mark :: proc(r: rune) -> bool {
+	switch r {
+	case 0x00600 ..= 0x00605,
+	     0x006DD,
+	     0x0070F,
+	     0x00890 ..= 0x00891,
+	     0x008E2,
+	     0x110BD,
+	     0x110CD:
+		return true
+	case:
+		return false
+	}
+}
+
+// General_Category=Spacing_Mark
+is_spacing_mark :: proc(r: rune) -> bool {
+	c := i32(r)
+	p := binary_search(c, spacing_mark_ranges[:], len(spacing_mark_ranges)/2, 2)
+	if p >= 0 && spacing_mark_ranges[p] <= c && c <= spacing_mark_ranges[p+1] {
+		return true
+	}
+	return false
+}
+
+// General_Category=Nonspacing_Mark
+is_nonspacing_mark :: proc(r: rune) -> bool {
+	c := i32(r)
+	p := binary_search(c, nonspacing_mark_ranges[:], len(nonspacing_mark_ranges)/2, 2)
+	if p >= 0 && nonspacing_mark_ranges[p] <= c && c <= nonspacing_mark_ranges[p+1] {
+		return true
+	}
+	return false
+}
+
+// Extended_Pictographic
+is_emoji_extended_pictographic :: proc(r: rune) -> bool {
+	c := i32(r)
+	p := binary_search(c, emoji_extended_pictographic_ranges[:], len(emoji_extended_pictographic_ranges)/2, 2)
+	if p >= 0 && emoji_extended_pictographic_ranges[p] <= c && c <= emoji_extended_pictographic_ranges[p+1] {
+		return true
+	}
+	return false
+}
+
+// Grapheme_Extend
+is_grapheme_extend :: proc(r: rune) -> bool {
+	c := i32(r)
+	p := binary_search(c, grapheme_extend_ranges[:], len(grapheme_extend_ranges)/2, 2)
+	if p >= 0 && grapheme_extend_ranges[p] <= c && c <= grapheme_extend_ranges[p+1] {
+		return true
+	}
+	return false
+}
+
+
+// Hangul_Syllable_Type=Leading_Jamo
+is_hangul_syllable_leading :: proc(r: rune) -> bool {
+	return 0x1100 <= r && r <= 0x115F || 0xA960 <= r && r <= 0xA97C
+}
+
+// Hangul_Syllable_Type=Vowel_Jamo
+is_hangul_syllable_vowel :: proc(r: rune) -> bool {
+	return 0x1160 <= r && r <= 0x11A7 || 0xD7B0 <= r && r <= 0xD7C6
+}
+
+// Hangul_Syllable_Type=Trailing_Jamo
+is_hangul_syllable_trailing :: proc(r: rune) -> bool {
+	return 0x11A8 <= r && r <= 0x11FF || 0xD7CB <= r && r <= 0xD7FB
+}
+
+// Hangul_Syllable_Type=LV_Syllable
+is_hangul_syllable_lv :: proc(r: rune) -> bool {
+	c := i32(r)
+	p := binary_search(c, hangul_syllable_lv_singlets[:], len(hangul_syllable_lv_singlets), 1)
+	if p >= 0 && c == hangul_syllable_lv_singlets[p] {
+		return true
+	}
+	return false
+}
+
+// Hangul_Syllable_Type=LVT_Syllable
+is_hangul_syllable_lvt :: proc(r: rune) -> bool {
+	c := i32(r)
+	p := binary_search(c, hangul_syllable_lvt_ranges[:], len(hangul_syllable_lvt_ranges)/2, 2)
+	if p >= 0 && hangul_syllable_lvt_ranges[p] <= c && c <= hangul_syllable_lvt_ranges[p+1] {
+		return true
+	}
+	return false
+}
+
+
+// Indic_Syllabic_Category=Consonant_Preceding_Repha
+is_indic_consonant_preceding_repha :: proc(r: rune) -> bool {
+	switch r {
+	case 0x00D4E,
+	     0x11941,
+	     0x11D46,
+	     0x11F02:
+		return true
+	case:
+		return false
+	}
+}
+
+// Indic_Syllabic_Category=Consonant_Prefixed
+is_indic_consonant_prefixed :: proc(r: rune) -> bool {
+	switch r {
+	case 0x111C2 ..= 0x111C3,
+	     0x1193F,
+	     0x11A3A,
+	     0x11A84 ..= 0x11A89:
+		return true
+	case:
+		return false
+	}
+}
+
+// Indic_Conjunct_Break=Linker
+is_indic_conjunct_break_linker :: proc(r: rune) -> bool {
+	switch r {
+	case 0x094D,
+	     0x09CD,
+	     0x0ACD,
+	     0x0B4D,
+	     0x0C4D,
+	     0x0D4D:
+		return true
+	case:
+		return false
+	}
+}
+
+// Indic_Conjunct_Break=Consonant
+is_indic_conjunct_break_consonant :: proc(r: rune) -> bool {
+	c := i32(r)
+	p := binary_search(c, indic_conjunct_break_consonant_ranges[:], len(indic_conjunct_break_consonant_ranges)/2, 2)
+	if p >= 0 && indic_conjunct_break_consonant_ranges[p] <= c && c <= indic_conjunct_break_consonant_ranges[p+1] {
+		return true
+	}
+	return false
+}
+
+// Indic_Conjunct_Break=Extend
+is_indic_conjunct_break_extend :: proc(r: rune) -> bool {
+	c := i32(r)
+	p := binary_search(c, indic_conjunct_break_extend_ranges[:], len(indic_conjunct_break_extend_ranges)/2, 2)
+	if p >= 0 && indic_conjunct_break_extend_ranges[p] <= c && c <= indic_conjunct_break_extend_ranges[p+1] {
+		return true
+	}
+	return false
+}
+
+
+/*
+For grapheme text segmentation, from Unicode TR 29 Rev 43:
+
+```
+Indic_Syllabic_Category = Consonant_Preceding_Repha, or
+Indic_Syllabic_Category = Consonant_Prefixed, or
+Prepended_Concatenation_Mark = Yes
+```
+*/
+is_gcb_prepend_class :: proc(r: rune) -> bool {
+	return is_indic_consonant_preceding_repha(r) || is_indic_consonant_prefixed(r) || is_prepended_concatenation_mark(r)
+}
+
+/*
+For grapheme text segmentation, from Unicode TR 29 Rev 43:
+
+```
+Grapheme_Extend = Yes, or
+Emoji_Modifier = Yes
+
+This includes:
+General_Category = Nonspacing_Mark
+General_Category = Enclosing_Mark
+U+200C ZERO WIDTH NON-JOINER
+
+plus a few General_Category = Spacing_Mark needed for canonical equivalence.
+```
+*/
+is_gcb_extend_class :: proc(r: rune) -> bool {
+	return is_grapheme_extend(r) || is_emoji_modifier(r)
+}
+
+//
+// End of Unicode 15.1.0 block.
+//