mirror of
https://github.com/Ed94/Odin.git
synced 2026-08-03 14:18:47 +00:00
Add decode_grapheme_clusters to core:unicode/utf8
This commit is contained in:
Feoramund
@@ -0,0 +1,387 @@
|
||||
package utf8
|
||||
|
||||
import "core:unicode"
|
||||
|
||||
ZERO_WIDTH_JOINER :: unicode.ZERO_WIDTH_JOINER
|
||||
is_control :: unicode.is_control
|
||||
is_hangul_syllable_leading :: unicode.is_hangul_syllable_leading
|
||||
is_hangul_syllable_vowel :: unicode.is_hangul_syllable_vowel
|
||||
is_hangul_syllable_trailing :: unicode.is_hangul_syllable_trailing
|
||||
is_hangul_syllable_lv :: unicode.is_hangul_syllable_lv
|
||||
is_hangul_syllable_lvt :: unicode.is_hangul_syllable_lvt
|
||||
is_indic_conjunct_break_extend :: unicode.is_indic_conjunct_break_extend
|
||||
is_indic_conjunct_break_linker :: unicode.is_indic_conjunct_break_linker
|
||||
is_indic_conjunct_break_consonant :: unicode.is_indic_conjunct_break_consonant
|
||||
is_gcb_extend_class :: unicode.is_gcb_extend_class
|
||||
is_spacing_mark :: unicode.is_spacing_mark
|
||||
is_gcb_prepend_class :: unicode.is_gcb_prepend_class
|
||||
is_emoji_extended_pictographic :: unicode.is_emoji_extended_pictographic
|
||||
is_regional_indicator :: unicode.is_regional_indicator
|
||||
|
||||
|
||||
Grapheme :: struct {
|
||||
byte_index: int,
|
||||
rune_index: int,
|
||||
}
|
||||
|
||||
/*
|
||||
Count the individual graphemes in a UTF-8 string.
|
||||
|
||||
Inputs:
|
||||
- str: The input string.
|
||||
|
||||
Returns:
|
||||
- graphemes: The number of graphemes in the string.
|
||||
- runes: The number of runes in the string.
|
||||
*/
|
||||
@(require_results)
|
||||
grapheme_count :: proc(str: string) -> (graphemes, runes: int) {
|
||||
_, graphemes, runes = decode_grapheme_clusters(str, false)
|
||||
return
|
||||
}
|
||||
|
||||
/*
|
||||
Decode the individual graphemes in a UTF-8 string.
|
||||
|
||||
*Allocates Using Provided Allocator*
|
||||
|
||||
Inputs:
|
||||
- str: The input string.
|
||||
- track_graphemes: Whether or not to allocate and return `graphemes` with extra data about each grapheme.
|
||||
- allocator: (default: context.allocator)
|
||||
|
||||
Returns:
|
||||
- graphemes: Extra data about each grapheme.
|
||||
- grapheme_count: The number of graphemes in the string.
|
||||
- rune_count: The number of runes in the string.
|
||||
*/
|
||||
@(require_results)
|
||||
decode_grapheme_clusters :: proc(
|
||||
str: string,
|
||||
track_graphemes := true,
|
||||
allocator := context.allocator,
|
||||
) -> (
|
||||
graphemes: [dynamic]Grapheme,
|
||||
grapheme_count: int,
|
||||
rune_count: int,
|
||||
) {
|
||||
// The following procedure implements text segmentation by breaking on
|
||||
// Grapheme Cluster Boundaries[1], using the values[2] and rules[3] from
|
||||
// the Unicode® Standard Annex #29, entitled:
|
||||
//
|
||||
// UNICODE TEXT SEGMENTATION
|
||||
//
|
||||
// Version: Unicode 15.1.0
|
||||
// Date: 2023-08-16
|
||||
// Revision: 43
|
||||
//
|
||||
// This procedure is conformant[4] to UAX29-C1-1, otherwise known as the
|
||||
// extended, non-legacy ruleset.
|
||||
//
|
||||
// Please see the references below for more information.
|
||||
//
|
||||
//
|
||||
// NOTE(Feoramund): This procedure has not been highly optimized.
|
||||
// A couple opportunities were taken to bypass repeated checking when a
|
||||
// rune is outside of certain codepoint ranges, but little else has been
|
||||
// done. Standard switches, conditionals, and binary search are used to
|
||||
// see if a rune fits into a certain category.
|
||||
//
|
||||
// I did find that only one prior rune of state was necessary to build an
|
||||
// algorithm that successfully passes all 4,835 test cases provided with
|
||||
// this implementation from the Unicode organization's website.
|
||||
//
|
||||
// My initial implementation tracked explicit breaks and counted them once
|
||||
// the string iteration had terminated. I've found this current
|
||||
// implementation to be far simpler and need no allocations (unless the
|
||||
// caller wants position data).
|
||||
//
|
||||
// Most rules work backwards instead of forwards which has helped keep this
|
||||
// simple, despite its length and verbosity.
|
||||
//
|
||||
//
|
||||
// The implementation has been left verbose and in the order described by
|
||||
// the specification, to enable better readability and future upkeep.
|
||||
//
|
||||
// Some possible optimizations might include:
|
||||
//
|
||||
// - saving the type of `last_rune` instead of the exact rune.
|
||||
// - reordering rules.
|
||||
// - combining tables.
|
||||
//
|
||||
//
|
||||
// [1]: https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
|
||||
// [2]: https://www.unicode.org/reports/tr29/#Default_Grapheme_Cluster_Table
|
||||
// [3]: https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules
|
||||
// [4]: https://www.unicode.org/reports/tr29/#Conformance
|
||||
|
||||
Grapheme_Cluster_Sequence :: enum {
|
||||
None,
|
||||
Indic,
|
||||
Emoji,
|
||||
Regional,
|
||||
}
|
||||
|
||||
context.allocator = allocator
|
||||
|
||||
last_rune: rune
|
||||
last_rune_breaks_forward: bool
|
||||
|
||||
last_grapheme_count: int
|
||||
|
||||
bypass_next_rune: bool
|
||||
|
||||
regional_indicator_counter: int
|
||||
|
||||
current_sequence: Grapheme_Cluster_Sequence
|
||||
continue_sequence: bool
|
||||
|
||||
for this_rune, byte_index in str {
|
||||
defer {
|
||||
// "Break at the start and end of text, unless the text is empty."
|
||||
//
|
||||
// GB1: sot ÷ Any
|
||||
// GB2: Any ÷ eot
|
||||
if rune_count == 0 && grapheme_count == 0 {
|
||||
grapheme_count += 1
|
||||
}
|
||||
if track_graphemes && grapheme_count > last_grapheme_count {
|
||||
append(&graphemes, Grapheme{ byte_index, rune_count })
|
||||
}
|
||||
last_grapheme_count = grapheme_count
|
||||
|
||||
last_rune = this_rune
|
||||
rune_count += 1
|
||||
|
||||
if !continue_sequence {
|
||||
current_sequence = .None
|
||||
regional_indicator_counter = 0
|
||||
}
|
||||
continue_sequence = false
|
||||
}
|
||||
|
||||
// "Do not break between a CR and LF. Otherwise, break before and after controls."
|
||||
//
|
||||
// GB3: CR × LF
|
||||
// GB4: (Control | CR | LF) ÷
|
||||
// GB5: ÷ (Control | CR | LF)
|
||||
if this_rune == '\n' && last_rune == '\r' {
|
||||
last_rune_breaks_forward = false
|
||||
bypass_next_rune = false
|
||||
continue
|
||||
}
|
||||
|
||||
if is_control(this_rune) {
|
||||
grapheme_count += 1
|
||||
last_rune_breaks_forward = true
|
||||
bypass_next_rune = true
|
||||
continue
|
||||
}
|
||||
|
||||
// (This check is for rules that work forwards, instead of backwards.)
|
||||
if bypass_next_rune {
|
||||
if last_rune_breaks_forward {
|
||||
grapheme_count += 1
|
||||
last_rune_breaks_forward = false
|
||||
}
|
||||
|
||||
bypass_next_rune = false
|
||||
continue
|
||||
}
|
||||
|
||||
// (Optimization 1: Prevent low runes from proceeding further.)
|
||||
//
|
||||
// * 0xA9 and 0xAE are in the Extended_Pictographic range,
|
||||
// which is checked later in GB11.
|
||||
if this_rune != 0xA9 && this_rune != 0xAE && this_rune <= 0x2FF {
|
||||
grapheme_count += 1
|
||||
continue
|
||||
}
|
||||
|
||||
// (Optimization 2: Check if the rune is in the Hangul space before getting specific.)
|
||||
if 0x1100 <= this_rune && this_rune <= 0xD7FB {
|
||||
// "Do not break Hangul syllable sequences."
|
||||
//
|
||||
// GB6: L × (L | V | LV | LVT)
|
||||
// GB7: (LV | V) × (V | T)
|
||||
// GB8: (LVT | T) × T
|
||||
if is_hangul_syllable_leading(this_rune) ||
|
||||
is_hangul_syllable_lv(this_rune) ||
|
||||
is_hangul_syllable_lvt(this_rune)
|
||||
{
|
||||
if !is_hangul_syllable_leading(last_rune) {
|
||||
grapheme_count += 1
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if is_hangul_syllable_vowel(this_rune) {
|
||||
if is_hangul_syllable_leading(last_rune) ||
|
||||
is_hangul_syllable_vowel(last_rune) ||
|
||||
is_hangul_syllable_lv(last_rune)
|
||||
{
|
||||
continue
|
||||
}
|
||||
grapheme_count += 1
|
||||
continue
|
||||
}
|
||||
|
||||
if is_hangul_syllable_trailing(this_rune) {
|
||||
if is_hangul_syllable_trailing(last_rune) ||
|
||||
is_hangul_syllable_lvt(last_rune) ||
|
||||
is_hangul_syllable_lv(last_rune) ||
|
||||
is_hangul_syllable_vowel(last_rune)
|
||||
{
|
||||
continue
|
||||
}
|
||||
grapheme_count += 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// "Do not break before extending characters or ZWJ."
|
||||
//
|
||||
// GB9: × (Extend | ZWJ)
|
||||
if this_rune == ZERO_WIDTH_JOINER {
|
||||
continue_sequence = true
|
||||
continue
|
||||
}
|
||||
|
||||
if is_gcb_extend_class(this_rune) {
|
||||
// (Support for GB9c.)
|
||||
if current_sequence == .Indic {
|
||||
if is_indic_conjunct_break_extend(this_rune) && (
|
||||
is_indic_conjunct_break_linker(last_rune) ||
|
||||
is_indic_conjunct_break_consonant(last_rune) )
|
||||
{
|
||||
continue_sequence = true
|
||||
continue
|
||||
}
|
||||
|
||||
if is_indic_conjunct_break_linker(this_rune) && (
|
||||
is_indic_conjunct_break_linker(last_rune) ||
|
||||
is_indic_conjunct_break_extend(last_rune) ||
|
||||
is_indic_conjunct_break_consonant(last_rune) )
|
||||
{
|
||||
continue_sequence = true
|
||||
continue
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// (Support for GB11.)
|
||||
if current_sequence == .Emoji && (
|
||||
is_gcb_extend_class(last_rune) ||
|
||||
is_emoji_extended_pictographic(last_rune) )
|
||||
{
|
||||
continue_sequence = true
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// _The GB9a and GB9b rules only apply to extended grapheme clusters:_
|
||||
// "Do not break before SpacingMarks, or after Prepend characters."
|
||||
//
|
||||
// GB9a: × SpacingMark
|
||||
// GB9b: Prepend ×
|
||||
if is_spacing_mark(this_rune) {
|
||||
continue
|
||||
}
|
||||
|
||||
if is_gcb_prepend_class(this_rune) {
|
||||
grapheme_count += 1
|
||||
bypass_next_rune = true
|
||||
continue
|
||||
}
|
||||
|
||||
// _The GB9c rule only applies to extended grapheme clusters:_
|
||||
// "Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker."
|
||||
//
|
||||
// GB9c: \p{InCB=Consonant} [ \p{InCB=Extend} \p{InCB=Linker} ]* \p{InCB=Linker} [ \p{InCB=Extend} \p{InCB=Linker} ]* × \p{InCB=Consonant}
|
||||
if is_indic_conjunct_break_consonant(this_rune) {
|
||||
if current_sequence == .Indic {
|
||||
if last_rune == ZERO_WIDTH_JOINER ||
|
||||
is_indic_conjunct_break_linker(last_rune)
|
||||
{
|
||||
continue_sequence = true
|
||||
} else {
|
||||
grapheme_count += 1
|
||||
}
|
||||
} else {
|
||||
grapheme_count += 1
|
||||
current_sequence = .Indic
|
||||
continue_sequence = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if is_indic_conjunct_break_extend(this_rune) {
|
||||
if current_sequence == .Indic {
|
||||
if is_indic_conjunct_break_consonant(last_rune) ||
|
||||
is_indic_conjunct_break_linker(last_rune)
|
||||
{
|
||||
continue_sequence = true
|
||||
} else {
|
||||
grapheme_count += 1
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if is_indic_conjunct_break_linker(this_rune) {
|
||||
if current_sequence == .Indic {
|
||||
if is_indic_conjunct_break_extend(last_rune) ||
|
||||
is_indic_conjunct_break_linker(last_rune)
|
||||
{
|
||||
continue_sequence = true
|
||||
} else {
|
||||
grapheme_count += 1
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
//
|
||||
// (Curiously, there is no GB10.)
|
||||
//
|
||||
|
||||
// "Do not break within emoji modifier sequences or emoji zwj sequences."
|
||||
//
|
||||
// GB11: \p{Extended_Pictographic} Extend* ZWJ × \p{Extended_Pictographic}
|
||||
if is_emoji_extended_pictographic(this_rune) {
|
||||
if current_sequence != .Emoji || last_rune != ZERO_WIDTH_JOINER {
|
||||
grapheme_count += 1
|
||||
}
|
||||
current_sequence = .Emoji
|
||||
continue_sequence = true
|
||||
continue
|
||||
}
|
||||
|
||||
// "Do not break within emoji flag sequences.
|
||||
// That is, do not break between regional indicator (RI) symbols
|
||||
// if there is an odd number of RI characters before the break point."
|
||||
//
|
||||
// GB12: sot (RI RI)* RI × RI
|
||||
// GB13: [^RI] (RI RI)* RI × RI
|
||||
if is_regional_indicator(this_rune) {
|
||||
if regional_indicator_counter & 1 == 0 {
|
||||
grapheme_count += 1
|
||||
}
|
||||
|
||||
current_sequence = .Regional
|
||||
continue_sequence = true
|
||||
regional_indicator_counter += 1
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
// "Otherwise, break everywhere."
|
||||
//
|
||||
// GB999: Any ÷ Any
|
||||
grapheme_count += 1
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user