dora_parser/lexer/
token.rs

1use std::fmt;
2use std::result::Result;
3
4use crate::lexer::position::{Position, Span};
5
6#[derive(PartialEq, Eq, Debug, Clone)]
7pub enum TokenKind {
8    StringTail(String),
9    StringExpr(String),
10    LitChar(char),
11    LitInt(String, IntBase, IntSuffix),
12    LitFloat(String, FloatSuffix),
13    Identifier(String),
14    End,
15
16    // Keywords
17    Class,
18    This,
19    CapitalThis,
20    Super,
21    Fun,
22    Let,
23    Var,
24    While,
25    If,
26    Else,
27    Loop,
28    For,
29    In,
30    Break,
31    Continue,
32    Return,
33    True,
34    False,
35    Nil,
36    Throws,
37    Throw,
38    Try,
39    TryForce,
40    TryOpt,
41    Do,
42    Catch,
43    Finally,
44    At,
45
46    Enum,
47    Type,
48    Alias,
49    Struct,
50    Trait,
51    Impl,
52    Module,
53    Const,
54
55    Underscore,
56    Defer,
57
58    // Operators
59    Add,
60    AddEq,
61    Sub,
62    Mul,
63    Div,
64    Mod,
65    Not,
66    LParen,
67    RParen,
68    LBracket,
69    RBracket,
70    LBrace,
71    RBrace,
72    Comma,
73    Semicolon,
74    Dot,
75    Colon,
76    Sep, // ::
77    Arrow,
78    Tilde,
79    BitOr,
80    BitAnd,
81    Caret,
82    And,
83    Or,
84
85    Eq,
86    EqEq,
87    Ne,
88    Lt,
89    Le,
90    Gt,
91    Ge,
92    EqEqEq,
93    NeEqEq,
94    Is,
95    As,
96
97    GtGt,
98    GtGtGt,
99    LtLt,
100}
101
102impl TokenKind {
103    pub fn name(&self) -> &str {
104        match *self {
105            TokenKind::StringTail(_) => "string tail",
106            TokenKind::StringExpr(_) => "string epxr",
107            TokenKind::LitInt(_, _, suffix) => match suffix {
108                IntSuffix::Byte => "byte number",
109                IntSuffix::Int => "int number",
110                IntSuffix::Long => "long number",
111            },
112
113            TokenKind::LitChar(_) => "char",
114
115            TokenKind::LitFloat(_, suffix) => match suffix {
116                FloatSuffix::Float => "float number",
117                FloatSuffix::Double => "double number",
118            },
119
120            TokenKind::Identifier(_) => "identifier",
121            TokenKind::End => "<<EOF>>",
122
123            // Keywords
124            TokenKind::Class => "class",
125            TokenKind::This => "self",
126            TokenKind::CapitalThis => "Self",
127            TokenKind::Super => "super",
128            TokenKind::Fun => "fun",
129            TokenKind::Let => "let",
130            TokenKind::Var => "var",
131            TokenKind::While => "while",
132            TokenKind::If => "if",
133            TokenKind::Else => "else",
134            TokenKind::Loop => "loop",
135            TokenKind::For => "for",
136            TokenKind::In => "in",
137            TokenKind::Break => "break",
138            TokenKind::Continue => "continue",
139            TokenKind::Return => "return",
140            TokenKind::True => "true",
141            TokenKind::False => "false",
142            TokenKind::Nil => "nil",
143            TokenKind::Throws => "throws",
144            TokenKind::Throw => "throw",
145            TokenKind::Try => "try",
146            TokenKind::TryForce => "try!",
147            TokenKind::TryOpt => "try?",
148            TokenKind::Do => "do",
149            TokenKind::Catch => "catch",
150            TokenKind::Finally => "finally",
151            TokenKind::At => "@",
152
153            TokenKind::Enum => "enum",
154            TokenKind::Type => "type",
155            TokenKind::Alias => "alias",
156            TokenKind::Struct => "struct",
157            TokenKind::Trait => "trait",
158            TokenKind::Impl => "impl",
159            TokenKind::Module => "module",
160            TokenKind::Const => "const",
161
162            TokenKind::Underscore => "_",
163            TokenKind::Defer => "defer",
164
165            // Operators
166            TokenKind::Add => "+",
167            TokenKind::AddEq => "+=",
168            TokenKind::Sub => "-",
169            TokenKind::Mul => "*",
170            TokenKind::Div => "/",
171            TokenKind::Mod => "%",
172            TokenKind::Not => "!",
173            TokenKind::LParen => "(",
174            TokenKind::RParen => ")",
175            TokenKind::LBracket => "[",
176            TokenKind::RBracket => "]",
177            TokenKind::LBrace => "{",
178            TokenKind::RBrace => "}",
179            TokenKind::Comma => ",",
180            TokenKind::Semicolon => ";",
181            TokenKind::Dot => ".",
182            TokenKind::Colon => ":",
183            TokenKind::Sep => "::",
184            TokenKind::Arrow => "=>",
185            TokenKind::Tilde => "~",
186            TokenKind::BitOr => "|",
187            TokenKind::BitAnd => "&",
188            TokenKind::Caret => "^",
189            TokenKind::And => "&&",
190            TokenKind::Or => "||",
191
192            TokenKind::Eq => "=",
193            TokenKind::EqEq => "==",
194            TokenKind::Ne => "!=",
195            TokenKind::Lt => "<",
196            TokenKind::Le => "<=",
197            TokenKind::Gt => ">",
198            TokenKind::Ge => ">=",
199
200            TokenKind::GtGt => ">>",
201            TokenKind::GtGtGt => ">>>",
202            TokenKind::LtLt => "<<",
203
204            TokenKind::EqEqEq => "===",
205            TokenKind::NeEqEq => "!==",
206            TokenKind::Is => "is",
207            TokenKind::As => "as",
208        }
209    }
210}
211
212#[derive(Copy, Clone, PartialEq, Eq, Debug)]
213pub enum IntSuffix {
214    Int,
215    Long,
216    Byte,
217}
218
219#[derive(Copy, Clone, PartialEq, Eq, Debug)]
220pub enum FloatSuffix {
221    Float,
222    Double,
223}
224
225#[derive(Debug)]
226pub struct Token {
227    pub kind: TokenKind,
228    pub position: Position,
229    pub span: Span,
230}
231
232impl Token {
233    pub fn new(tok: TokenKind, pos: Position, span: Span) -> Token {
234        Token {
235            kind: tok,
236            position: pos,
237            span,
238        }
239    }
240
241    pub fn is_eof(&self) -> bool {
242        self.kind == TokenKind::End
243    }
244
245    pub fn is(&self, kind: TokenKind) -> bool {
246        self.kind == kind
247    }
248
249    pub fn name(&self) -> String {
250        match self.kind {
251            TokenKind::LitInt(ref val, _, suffix) => {
252                let suffix = match suffix {
253                    IntSuffix::Byte => "B",
254                    IntSuffix::Int => "",
255                    IntSuffix::Long => "L",
256                };
257
258                format!("{}{}", val, suffix)
259            }
260
261            TokenKind::StringTail(ref val) => format!("\"{}\" tail", &val),
262            TokenKind::StringExpr(ref val) => format!("\"{}\" expr", &val),
263
264            TokenKind::Identifier(ref val) => val.clone(),
265
266            _ => self.kind.name().into(),
267        }
268    }
269}
270
271impl fmt::Display for Token {
272    fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
273        write!(f, "{}", self.name())
274    }
275}
276
277#[derive(Copy, Clone, Debug, PartialEq, Eq)]
278pub enum IntBase {
279    Bin,
280    Dec,
281    Hex,
282}
283
284impl IntBase {
285    pub fn num(self) -> u32 {
286        match self {
287            IntBase::Bin => 2,
288            IntBase::Dec => 10,
289            IntBase::Hex => 16,
290        }
291    }
292}
dora_parser/lexer/token.rs

dora_parser/lexer/
token.rs
