protobuf_support/lexer/
token.rs

1use crate::lexer::json_number_lit::JsonNumberLit;
2use crate::lexer::lexer_impl::LexerError;
3use crate::lexer::lexer_impl::LexerResult;
4use crate::lexer::loc::Loc;
5use crate::lexer::num_lit::NumLit;
6use crate::lexer::str_lit::StrLit;
7
8#[derive(Clone, Debug, PartialEq)]
9pub enum Token {
10    Ident(String),
11    Symbol(char),
12    // Protobuf tokenizer has separate tokens for int and float.
13    // Tokens do not include sign.
14    IntLit(u64),
15    FloatLit(f64),
16    JsonNumber(JsonNumberLit),
17    // including quotes
18    StrLit(StrLit),
19}
20
21impl Token {
22    /// Back to original
23    pub fn format(&self) -> String {
24        match self {
25            &Token::Ident(ref s) => s.clone(),
26            &Token::Symbol(c) => c.to_string(),
27            &Token::IntLit(ref i) => i.to_string(),
28            &Token::StrLit(ref s) => s.quoted(),
29            &Token::FloatLit(ref f) => f.to_string(),
30            &Token::JsonNumber(ref f) => f.to_string(),
31        }
32    }
33
34    pub fn to_num_lit(&self) -> LexerResult<NumLit> {
35        match self {
36            &Token::IntLit(i) => Ok(NumLit::U64(i)),
37            &Token::FloatLit(f) => Ok(NumLit::F64(f)),
38            _ => Err(LexerError::IncorrectInput),
39        }
40    }
41}
42
43#[derive(Clone)]
44pub struct TokenWithLocation {
45    pub token: Token,
46    pub loc: Loc,
47}
protobuf_support/lexer/token.rs

protobuf_support/lexer/
token.rs
