pub trait Language {
// Required methods
fn name(&self) -> String;
fn code(&self) -> String;
fn tokenize(&self, text: &str) -> Vec<String>;
fn make_pipeline(&self) -> Pipeline;
}
Required Methods§
Sourcefn tokenize(&self, text: &str) -> Vec<String>
fn tokenize(&self, text: &str) -> Vec<String>
Separates the input text into individual tokens. In most languages a token is a word, separated by whitespace.
Sourcefn make_pipeline(&self) -> Pipeline
fn make_pipeline(&self) -> Pipeline
Returns the Pipeline
to process the tokens with