Function collect_tokens

Source
pub fn collect_tokens(
    text: &str,
    tokenizer: &mut TextAnalyzer,
    inclusive: Option<&HashSet<String>>,
) -> Vec<String>