Struct Tokenizer

pub struct Tokenizer<'a> { /* private fields */ }

Implementations

impl<'a> Tokenizer<'a>

pub fn new(input: &'a str, comment_style: ParserLanguage) -> Tokenizer<'a>

pub fn loc(&self) -> Loc

pub fn lookahead_loc(&mut self) -> Loc

pub fn lookahead_some(&mut self) -> TokenizerResult<&Token>

pub fn next_some(&mut self) -> TokenizerResult<Token>

pub fn advance(&mut self) -> TokenizerResult<Token>

Can be called only after lookahead, otherwise it’s error

pub fn syntax_eof(&mut self) -> TokenizerResult<bool>

No more tokens

pub fn next_token_if_map<P, R>(&mut self, p: P) -> TokenizerResult<Option<R>>

where P: FnOnce(&Token) -> Option<R>,

pub fn next_token_check_map<P, R, E>(&mut self, p: P) -> Result<R, E>

where P: FnOnce(&Token) -> Result<R, E>, E: From<TokenizerError>,

pub fn next_ident_if_in( &mut self, idents: &[&str], ) -> TokenizerResult<Option<String>>

pub fn next_ident_if_eq(&mut self, word: &str) -> TokenizerResult<bool>

pub fn next_ident_expect_eq(&mut self, word: &str) -> TokenizerResult<()>

pub fn next_ident_if_eq_error( &mut self, word: &'static str, ) -> TokenizerResult<()>

pub fn next_symbol_if_in(&mut self, symbols: &[char]) -> TokenizerResult<bool>

pub fn next_symbol_if_eq(&mut self, symbol: char) -> TokenizerResult<bool>

pub fn next_symbol_expect_eq( &mut self, symbol: char, desc: &'static str, ) -> TokenizerResult<()>

pub fn next_symbol_expect_eq_oneof( &mut self, symbols: &[char], ) -> TokenizerResult<char>

pub fn lookahead_is_str_lit(&mut self) -> TokenizerResult<bool>

pub fn lookahead_is_int_lit(&mut self) -> TokenizerResult<bool>

pub fn lookahead_is_json_number(&mut self) -> TokenizerResult<bool>

pub fn lookahead_if_symbol(&mut self) -> TokenizerResult<Option<char>>

pub fn lookahead_is_symbol(&mut self, symbol: char) -> TokenizerResult<bool>

pub fn lookahead_is_ident(&mut self, ident: &str) -> TokenizerResult<bool>

pub fn next_ident(&mut self) -> TokenizerResult<String>

pub fn next_str_lit(&mut self) -> TokenizerResult<StrLit>

pub fn next_int_lit(&mut self) -> TokenizerResult<u64>

pub fn next_float_lit(&mut self) -> TokenizerResult<f64>

Trait Implementations

impl<'a> Clone for Tokenizer<'a>

fn clone(&self) -> Tokenizer<'a>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.