1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

// Copyright 2023 The Servo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use std::os::raw::c_void;

use crate::array::CFMutableArrayRef;
use crate::base::{CFAllocatorRef, CFIndex, CFOptionFlags, CFRange, CFTypeID, CFTypeRef};
use crate::locale::CFLocaleRef;
use crate::string::CFStringRef;

#[repr(C)]
pub struct __CFStringTokenizer(c_void);
pub type CFStringTokenizerRef = *mut __CFStringTokenizer;

pub type CFStringTokenizerTokenType = CFOptionFlags;

pub const kCFStringTokenizerTokenNone: CFStringTokenizerTokenType = 0;
pub const kCFStringTokenizerTokenNormal: CFStringTokenizerTokenType = 1 << 0;
pub const kCFStringTokenizerTokenHasSubTokensMask: CFStringTokenizerTokenType = 1 << 1;
pub const kCFStringTokenizerTokenHasDerivedSubTokensMask: CFStringTokenizerTokenType = 1 << 2;
pub const kCFStringTokenizerTokenHasHasNumbersMask: CFStringTokenizerTokenType = 1 << 3;
pub const kCFStringTokenizerTokenHasNonLettersMask: CFStringTokenizerTokenType = 1 << 4;
pub const kCFStringTokenizerTokenIsCJWordMask: CFStringTokenizerTokenType = 1 << 5;

/* Tokenization Modifiers */
pub const kCFStringTokenizerUnitWord: CFOptionFlags = 0;
pub const kCFStringTokenizerUnitSentence: CFOptionFlags = 1;
pub const kCFStringTokenizerUnitParagraph: CFOptionFlags = 2;
pub const kCFStringTokenizerUnitLineBreak: CFOptionFlags = 3;
pub const kCFStringTokenizerUnitWordBoundary: CFOptionFlags = 4;
pub const kCFStringTokenizerAttributeLatinTranscription: CFOptionFlags = 1 << 16;
pub const kCFStringTokenizerAttributeLanguage: CFOptionFlags = 1 << 17;

extern "C" {
    /*
     * CFStringTokenizer.h
     */

    /* Creating a Tokenizer */
    pub fn CFStringTokenizerCreate(
        alloc: CFAllocatorRef,
        string: CFStringRef,
        range: CFRange,
        options: CFOptionFlags,
        locale: CFLocaleRef,
    ) -> CFStringTokenizerRef;

    /* Setting the String */
    pub fn CFStringTokenizerSetString(
        tokenizer: CFStringTokenizerRef,
        string: CFStringRef,
        range: CFRange,
    );

    /* Changing the Location */
    pub fn CFStringTokenizerAdvanceToNextToken(
        tokenizer: CFStringTokenizerRef,
    ) -> CFStringTokenizerTokenType;
    pub fn CFStringTokenizerGoToTokenAtIndex(
        tokenizer: CFStringTokenizerRef,
        index: CFIndex,
    ) -> CFStringTokenizerTokenType;

    /* Getting Information About the Current Token */
    pub fn CFStringTokenizerCopyCurrentTokenAttribute(
        tokenizer: CFStringTokenizerRef,
        attribute: CFOptionFlags,
    ) -> CFTypeRef;
    pub fn CFStringTokenizerGetCurrentTokenRange(tokenizer: CFStringTokenizerRef) -> CFRange;
    pub fn CFStringTokenizerGetCurrentSubTokens(
        tokenizer: CFStringTokenizerRef,
        ranges: *mut CFRange,
        maxRangeLength: CFIndex,
        derivedSubTokens: CFMutableArrayRef,
    ) -> CFIndex;

    /* Identifying a Language */
    pub fn CFStringTokenizerCopyBestStringLanguage(
        string: CFStringRef,
        range: CFRange,
    ) -> CFStringRef;

    /* Getting the CFStringTokenizer Type ID */
    pub fn CFStringTokenizerGetTypeID() -> CFTypeID;
}