unicode_normalization_alignments/
normalize.rs

1// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
2// file at the top-level directory of this distribution and at
3// http://rust-lang.org/COPYRIGHT.
4//
5// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8// option. This file may not be copied, modified, or distributed
9// except according to those terms.
10
11//! Functions for computing canonical and compatible decompositions for Unicode characters.
12use std::char;
13use std::ops::FnMut;
14use lookups::{canonical_fully_decomposed, composition_table, compatibility_fully_decomposed};
15
16/// Compute canonical Unicode decomposition for character.
17/// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
18/// for more information.
19#[inline]
20pub fn decompose_canonical<F>(c: char, emit_char: F) where F: FnMut(char) {
21    decompose(c, canonical_fully_decomposed, emit_char)
22}
23
24/// Compute canonical or compatible Unicode decomposition for character.
25/// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
26/// for more information.
27#[inline]
28pub fn decompose_compatible<F: FnMut(char)>(c: char, emit_char: F) {
29    let decompose_char = |c| compatibility_fully_decomposed(c)
30        .or_else(|| canonical_fully_decomposed(c));
31    decompose(c, decompose_char, emit_char)
32}
33
34#[inline]
35fn decompose<D, F>(c: char, decompose_char: D, mut emit_char: F)
36    where D: Fn(char) -> Option<&'static [char]>, F: FnMut(char)
37{
38    // 7-bit ASCII never decomposes
39    if c <= '\x7f' {
40        emit_char(c);
41        return;
42    }
43
44    // Perform decomposition for Hangul
45    if is_hangul_syllable(c) {
46        decompose_hangul(c, emit_char);
47        return;
48    }
49
50    if let Some(decomposed) = decompose_char(c) {
51        for &d in decomposed {
52            emit_char(d);
53        }
54        return;
55    }
56
57    // Finally bottom out.
58    emit_char(c);
59}
60
61/// Compose two characters into a single character, if possible.
62/// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
63/// for more information.
64pub fn compose(a: char, b: char) -> Option<char> {
65    compose_hangul(a, b).or_else(|| composition_table(a, b))
66}
67
68// Constants from Unicode 9.0.0 Section 3.12 Conjoining Jamo Behavior
69// http://www.unicode.org/versions/Unicode9.0.0/ch03.pdf#M9.32468.Heading.310.Combining.Jamo.Behavior
70const S_BASE: u32 = 0xAC00;
71const L_BASE: u32 = 0x1100;
72const V_BASE: u32 = 0x1161;
73const T_BASE: u32 = 0x11A7;
74const L_COUNT: u32 = 19;
75const V_COUNT: u32 = 21;
76const T_COUNT: u32 = 28;
77const N_COUNT: u32 = (V_COUNT * T_COUNT);
78const S_COUNT: u32 = (L_COUNT * N_COUNT);
79
80const S_LAST: u32 = S_BASE + S_COUNT - 1;
81const L_LAST: u32 = L_BASE + L_COUNT - 1;
82const V_LAST: u32 = V_BASE + V_COUNT - 1;
83const T_LAST: u32 = T_BASE + T_COUNT - 1;
84
85// Composition only occurs for `TPart`s in `U+11A8 ... U+11C2`,
86// i.e. `T_BASE + 1 ... T_LAST`.
87const T_FIRST: u32 = T_BASE + 1;
88
89pub(crate) fn is_hangul_syllable(c: char) -> bool {
90    (c as u32) >= S_BASE && (c as u32) < (S_BASE + S_COUNT)
91}
92
93// Decompose a precomposed Hangul syllable
94#[allow(unsafe_code)]
95#[inline(always)]
96fn decompose_hangul<F>(s: char, mut emit_char: F) where F: FnMut(char) {
97    let s_index = s as u32 - S_BASE;
98    let l_index = s_index / N_COUNT;
99    unsafe {
100        emit_char(char::from_u32_unchecked(L_BASE + l_index));
101
102        let v_index = (s_index % N_COUNT) / T_COUNT;
103        emit_char(char::from_u32_unchecked(V_BASE + v_index));
104
105        let t_index = s_index % T_COUNT;
106        if t_index > 0 {
107            emit_char(char::from_u32_unchecked(T_BASE + t_index));
108        }
109    }
110}
111
112#[inline]
113pub(crate) fn hangul_decomposition_length(s: char) -> usize {
114    let si = s as u32 - S_BASE;
115    let ti = si % T_COUNT;
116    if ti > 0 { 3 } else { 2 }
117}
118
119// Compose a pair of Hangul Jamo
120#[allow(unsafe_code)]
121#[inline(always)]
122#[allow(ellipsis_inclusive_range_patterns)]
123fn compose_hangul(a: char, b: char) -> Option<char> {
124    let (a, b) = (a as u32, b as u32);
125    match (a, b) {
126        // Compose a leading consonant and a vowel together into an LV_Syllable
127        (L_BASE ... L_LAST, V_BASE ... V_LAST) => {
128            let l_index = a - L_BASE;
129            let v_index = b - V_BASE;
130            let lv_index = l_index * N_COUNT + v_index * T_COUNT;
131            let s = S_BASE + lv_index;
132            Some(unsafe {char::from_u32_unchecked(s)})
133        },
134        // Compose an LV_Syllable and a trailing consonant into an LVT_Syllable
135        (S_BASE ... S_LAST, T_FIRST ... T_LAST) if (a - S_BASE) % T_COUNT == 0 => {
136            Some(unsafe {char::from_u32_unchecked(a + (b - T_BASE))})
137        },
138        _ => None,
139    }
140}
141
142#[cfg(test)]
143mod tests {
144    use super::compose_hangul;
145
146    // Regression test from a bugfix where we were composing an LV_Syllable with
147    // T_BASE directly. (We should only compose an LV_Syllable with a character
148    // in the range `T_BASE + 1 ... T_LAST`.)
149    #[test]
150    fn test_hangul_composition() {
151        assert_eq!(compose_hangul('\u{c8e0}', '\u{11a7}'), None);
152    }
153}
unicode_normalization_alignments/normalize.rs

unicode_normalization_alignments/
normalize.rs
