Struct Tokenizer

pub struct Tokenizer<'a> { /* private fields */ }

Implementations

impl<'a> Tokenizer<'a>

pub fn new(input: &'a str) -> Tokenizer<'a>

pub fn step(&mut self) -> Result<Option<(Span, Token<'a>)>, Error>

pub fn peek(&mut self) -> Result<Option<(Span, Token<'a>)>, Error>

pub fn eat(&mut self, expected: Token<'a>) -> Result<bool, Error>

pub fn eat_spanned( &mut self, expected: Token<'a>, ) -> Result<Option<Span>, Error>

Eat a value, returning it’s span if it was consumed.

pub fn expect(&mut self, expected: Token<'a>) -> Result<(), Error>

pub fn expect_spanned(&mut self, expected: Token<'a>) -> Result<Span, Error>

Expect the given token returning its span.

pub fn table_key(&mut self) -> Result<Key<'a>, Error>

pub fn eat_whitespace(&mut self)

pub fn eat_comment(&mut self) -> Result<bool, Error>

pub fn eat_newline_or_eof(&mut self) -> Result<(), Error>

pub fn skip_to_newline(&mut self)

pub fn current(&mut self) -> usize

pub fn substr_offset(&self, s: &'a str) -> usize

pub fn one(&mut self) -> Option<(usize, char)>

Take one char.

Trait Implementations

impl<'a> Clone for Tokenizer<'a>

fn clone(&self) -> Tokenizer<'a>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.