pub use BinOpToken::*;
pub use DelimToken::*;
pub use LitKind::*;
pub use Nonterminal::*;
pub use TokenKind::*;
use crate::ast;
use crate::ptr::P;
use crate::tokenstream::TokenTree;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_data_structures::sync::Lrc;
use rustc_macros::HashStable_Generic;
use rustc_span::symbol::kw;
use rustc_span::symbol::Symbol;
use rustc_span::{self, Span, DUMMY_SP};
use std::fmt;
use std::mem;
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
#[derive(HashStable_Generic)]
pub enum BinOpToken {
Plus,
Minus,
Star,
Slash,
Percent,
Caret,
And,
Or,
Shl,
Shr,
}
#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
#[derive(HashStable_Generic)]
pub enum DelimToken {
Paren,
Bracket,
Brace,
NoDelim,
}
impl DelimToken {
pub fn len(self) -> usize {
if self == NoDelim { 0 } else { 1 }
}
pub fn is_empty(self) -> bool {
self == NoDelim
}
}
#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
pub enum LitKind {
Bool, Byte,
Char,
Integer,
Float,
Str,
StrRaw(u16), ByteStr,
ByteStrRaw(u16), Err,
}
#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
pub struct Lit {
pub kind: LitKind,
pub symbol: Symbol,
pub suffix: Option<Symbol>,
}
impl fmt::Display for Lit {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let Lit { kind, symbol, suffix } = *self;
match kind {
Byte => write!(f, "b'{}'", symbol)?,
Char => write!(f, "'{}'", symbol)?,
Str => write!(f, "\"{}\"", symbol)?,
StrRaw(n) => write!(
f,
"r{delim}\"{string}\"{delim}",
delim = "#".repeat(n as usize),
string = symbol
)?,
ByteStr => write!(f, "b\"{}\"", symbol)?,
ByteStrRaw(n) => write!(
f,
"br{delim}\"{string}\"{delim}",
delim = "#".repeat(n as usize),
string = symbol
)?,
Integer | Float | Bool | Err => write!(f, "{}", symbol)?,
}
if let Some(suffix) = suffix {
write!(f, "{}", suffix)?;
}
Ok(())
}
}
impl LitKind {
pub fn article(self) -> &'static str {
match self {
Integer | Err => "an",
_ => "a",
}
}
pub fn descr(self) -> &'static str {
match self {
Bool => panic!("literal token contains `Lit::Bool`"),
Byte => "byte",
Char => "char",
Integer => "integer",
Float => "float",
Str | StrRaw(..) => "string",
ByteStr | ByteStrRaw(..) => "byte string",
Err => "error",
}
}
crate fn may_have_suffix(self) -> bool {
match self {
Integer | Float | Err => true,
_ => false,
}
}
}
impl Lit {
pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
Lit { kind, symbol, suffix }
}
}
pub fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool {
let ident_token = Token::new(Ident(name, is_raw), span);
token_can_begin_expr(&ident_token)
}
pub fn token_can_begin_expr(ident_token: &Token) -> bool {
!ident_token.is_reserved_ident()
|| ident_token.is_path_segment_keyword()
|| match ident_token.kind {
TokenKind::Ident(ident, _) => [
kw::Async,
kw::Do,
kw::Box,
kw::Break,
kw::Continue,
kw::False,
kw::For,
kw::If,
kw::Let,
kw::Loop,
kw::Match,
kw::Move,
kw::Return,
kw::True,
kw::Unsafe,
kw::While,
kw::Yield,
kw::Static,
]
.contains(&ident),
_ => false,
}
}
fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool {
let ident_token = Token::new(Ident(name, is_raw), span);
!ident_token.is_reserved_ident()
|| ident_token.is_path_segment_keyword()
|| [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn]
.contains(&name)
}
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
pub enum TokenKind {
Eq,
Lt,
Le,
EqEq,
Ne,
Ge,
Gt,
AndAnd,
OrOr,
Not,
Tilde,
BinOp(BinOpToken),
BinOpEq(BinOpToken),
At,
Dot,
DotDot,
DotDotDot,
DotDotEq,
Comma,
Semi,
Colon,
ModSep,
RArrow,
LArrow,
FatArrow,
Pound,
Dollar,
Question,
SingleQuote,
OpenDelim(DelimToken),
CloseDelim(DelimToken),
Literal(Lit),
Ident(ast::Name, bool),
Lifetime(ast::Name),
Interpolated(Lrc<Nonterminal>),
DocComment(ast::Name),
Whitespace,
Comment,
Shebang(ast::Name),
Unknown(ast::Name),
Eof,
}
#[cfg(target_arch = "x86_64")]
rustc_data_structures::static_assert_size!(TokenKind, 16);
#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
}
impl TokenKind {
pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind {
Literal(Lit::new(kind, symbol, suffix))
}
pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> {
Some(match *self {
Le => (Lt, Eq),
EqEq => (Eq, Eq),
Ne => (Not, Eq),
Ge => (Gt, Eq),
AndAnd => (BinOp(And), BinOp(And)),
OrOr => (BinOp(Or), BinOp(Or)),
BinOp(Shl) => (Lt, Lt),
BinOp(Shr) => (Gt, Gt),
BinOpEq(Plus) => (BinOp(Plus), Eq),
BinOpEq(Minus) => (BinOp(Minus), Eq),
BinOpEq(Star) => (BinOp(Star), Eq),
BinOpEq(Slash) => (BinOp(Slash), Eq),
BinOpEq(Percent) => (BinOp(Percent), Eq),
BinOpEq(Caret) => (BinOp(Caret), Eq),
BinOpEq(And) => (BinOp(And), Eq),
BinOpEq(Or) => (BinOp(Or), Eq),
BinOpEq(Shl) => (Lt, Le),
BinOpEq(Shr) => (Gt, Ge),
DotDot => (Dot, Dot),
DotDotDot => (Dot, DotDot),
ModSep => (Colon, Colon),
RArrow => (BinOp(Minus), Gt),
LArrow => (Lt, BinOp(Minus)),
FatArrow => (Eq, Gt),
_ => return None,
})
}
pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
match *self {
Comma => Some(vec![Dot, Lt, Semi]),
Semi => Some(vec![Colon, Comma]),
_ => None,
}
}
}
impl Token {
pub fn new(kind: TokenKind, span: Span) -> Self {
Token { kind, span }
}
pub fn dummy() -> Self {
Token::new(TokenKind::Whitespace, DUMMY_SP)
}
pub fn from_ast_ident(ident: ast::Ident) -> Self {
Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span)
}
pub fn take(&mut self) -> Self {
mem::replace(self, Token::dummy())
}
pub fn is_op(&self) -> bool {
match self.kind {
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
| Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false,
_ => true,
}
}
pub fn is_like_plus(&self) -> bool {
match self.kind {
BinOp(Plus) | BinOpEq(Plus) => true,
_ => false,
}
}
pub fn can_begin_expr(&self) -> bool {
match self.kind {
Ident(name, is_raw) =>
ident_can_begin_expr(name, self.span, is_raw), OpenDelim(..) | Literal(..) | Not | BinOp(Minus) | BinOp(Star) | BinOp(Or) | OrOr | BinOp(And) | AndAnd | DotDot | DotDotDot | DotDotEq | Lt | BinOp(Shl) | ModSep | Lifetime(..) | Pound => true, Interpolated(ref nt) => match **nt {
NtLiteral(..) |
NtIdent(..) |
NtExpr(..) |
NtBlock(..) |
NtPath(..) |
NtLifetime(..) => true,
_ => false,
},
_ => false,
}
}
pub fn can_begin_type(&self) -> bool {
match self.kind {
Ident(name, is_raw) =>
ident_can_begin_type(name, self.span, is_raw), OpenDelim(Paren) | OpenDelim(Bracket) | Not | BinOp(Star) | BinOp(And) | AndAnd | Question | Lifetime(..) | Lt | BinOp(Shl) | ModSep => true, Interpolated(ref nt) => match **nt {
NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true,
_ => false,
},
_ => false,
}
}
pub fn can_begin_const_arg(&self) -> bool {
match self.kind {
OpenDelim(Brace) => true,
Interpolated(ref nt) => match **nt {
NtExpr(..) | NtBlock(..) | NtLiteral(..) => true,
_ => false,
},
_ => self.can_begin_literal_or_bool(),
}
}
pub fn can_begin_bound(&self) -> bool {
self.is_path_start()
|| self.is_lifetime()
|| self.is_keyword(kw::For)
|| self == &Question
|| self == &OpenDelim(Paren)
}
pub fn is_lit(&self) -> bool {
match self.kind {
Literal(..) => true,
_ => false,
}
}
pub fn can_begin_literal_or_bool(&self) -> bool {
match self.kind {
Literal(..) | BinOp(Minus) => true,
Ident(name, false) if name.is_bool_lit() => true,
Interpolated(ref nt) => match &**nt {
NtExpr(e) | NtLiteral(e) => matches!(e.kind, ast::ExprKind::Lit(_)),
_ => false,
},
_ => false,
}
}
pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> {
match self.kind {
Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)),
Interpolated(ref nt) => match **nt {
NtIdent(ident, is_raw) => Some((ident, is_raw)),
_ => None,
},
_ => None,
}
}
pub fn lifetime(&self) -> Option<ast::Ident> {
match self.kind {
Lifetime(name) => Some(ast::Ident::new(name, self.span)),
Interpolated(ref nt) => match **nt {
NtLifetime(ident) => Some(ident),
_ => None,
},
_ => None,
}
}
pub fn is_ident(&self) -> bool {
self.ident().is_some()
}
pub fn is_lifetime(&self) -> bool {
self.lifetime().is_some()
}
pub fn is_ident_named(&self, name: Symbol) -> bool {
self.ident().map_or(false, |(ident, _)| ident.name == name)
}
fn is_path(&self) -> bool {
if let Interpolated(ref nt) = self.kind {
if let NtPath(..) = **nt {
return true;
}
}
false
}
pub fn is_whole_expr(&self) -> bool {
if let Interpolated(ref nt) = self.kind {
if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt {
return true;
}
}
false
}
pub fn is_mutability(&self) -> bool {
self.is_keyword(kw::Mut) || self.is_keyword(kw::Const)
}
pub fn is_qpath_start(&self) -> bool {
self == &Lt || self == &BinOp(Shl)
}
pub fn is_path_start(&self) -> bool {
self == &ModSep
|| self.is_qpath_start()
|| self.is_path()
|| self.is_path_segment_keyword()
|| self.is_ident() && !self.is_reserved_ident()
}
pub fn is_keyword(&self, kw: Symbol) -> bool {
self.is_non_raw_ident_where(|id| id.name == kw)
}
pub fn is_path_segment_keyword(&self) -> bool {
self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword)
}
pub fn is_special_ident(&self) -> bool {
self.is_non_raw_ident_where(ast::Ident::is_special)
}
pub fn is_used_keyword(&self) -> bool {
self.is_non_raw_ident_where(ast::Ident::is_used_keyword)
}
pub fn is_unused_keyword(&self) -> bool {
self.is_non_raw_ident_where(ast::Ident::is_unused_keyword)
}
pub fn is_reserved_ident(&self) -> bool {
self.is_non_raw_ident_where(ast::Ident::is_reserved)
}
pub fn is_bool_lit(&self) -> bool {
self.is_non_raw_ident_where(|id| id.name.is_bool_lit())
}
pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool {
match self.ident() {
Some((id, false)) => pred(id),
_ => false,
}
}
pub fn glue(&self, joint: &Token) -> Option<Token> {
let kind = match self.kind {
Eq => match joint.kind {
Eq => EqEq,
Gt => FatArrow,
_ => return None,
},
Lt => match joint.kind {
Eq => Le,
Lt => BinOp(Shl),
Le => BinOpEq(Shl),
BinOp(Minus) => LArrow,
_ => return None,
},
Gt => match joint.kind {
Eq => Ge,
Gt => BinOp(Shr),
Ge => BinOpEq(Shr),
_ => return None,
},
Not => match joint.kind {
Eq => Ne,
_ => return None,
},
BinOp(op) => match joint.kind {
Eq => BinOpEq(op),
BinOp(And) if op == And => AndAnd,
BinOp(Or) if op == Or => OrOr,
Gt if op == Minus => RArrow,
_ => return None,
},
Dot => match joint.kind {
Dot => DotDot,
DotDot => DotDotDot,
_ => return None,
},
DotDot => match joint.kind {
Dot => DotDotDot,
Eq => DotDotEq,
_ => return None,
},
Colon => match joint.kind {
Colon => ModSep,
_ => return None,
},
SingleQuote => match joint.kind {
Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))),
_ => return None,
},
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
| DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
| Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment
| Shebang(..) | Unknown(..) | Eof => return None,
};
Some(Token::new(kind, self.span.to(joint.span)))
}
crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool {
if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) {
return false;
}
match (&self.kind, &other.kind) {
(&Eq, &Eq)
| (&Lt, &Lt)
| (&Le, &Le)
| (&EqEq, &EqEq)
| (&Ne, &Ne)
| (&Ge, &Ge)
| (&Gt, &Gt)
| (&AndAnd, &AndAnd)
| (&OrOr, &OrOr)
| (&Not, &Not)
| (&Tilde, &Tilde)
| (&At, &At)
| (&Dot, &Dot)
| (&DotDot, &DotDot)
| (&DotDotDot, &DotDotDot)
| (&DotDotEq, &DotDotEq)
| (&Comma, &Comma)
| (&Semi, &Semi)
| (&Colon, &Colon)
| (&ModSep, &ModSep)
| (&RArrow, &RArrow)
| (&LArrow, &LArrow)
| (&FatArrow, &FatArrow)
| (&Pound, &Pound)
| (&Dollar, &Dollar)
| (&Question, &Question)
| (&Whitespace, &Whitespace)
| (&Comment, &Comment)
| (&Eof, &Eof) => true,
(&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
(&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b,
(&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b,
(&Literal(a), &Literal(b)) => a == b,
(&Lifetime(a), &Lifetime(b)) => a == b,
(&Ident(a, b), &Ident(c, d)) => {
b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate)
}
(&Interpolated(_), &Interpolated(_)) => false,
_ => panic!("forgot to add a token?"),
}
}
}
impl PartialEq<TokenKind> for Token {
fn eq(&self, rhs: &TokenKind) -> bool {
self.kind == *rhs
}
}
#[derive(Clone, RustcEncodable, RustcDecodable)]
pub enum Nonterminal {
NtItem(P<ast::Item>),
NtBlock(P<ast::Block>),
NtStmt(ast::Stmt),
NtPat(P<ast::Pat>),
NtExpr(P<ast::Expr>),
NtTy(P<ast::Ty>),
NtIdent(ast::Ident, bool),
NtLifetime(ast::Ident),
NtLiteral(P<ast::Expr>),
NtMeta(P<ast::AttrItem>),
NtPath(ast::Path),
NtVis(ast::Visibility),
NtTT(TokenTree),
NtTraitItem(P<ast::AssocItem>),
NtImplItem(P<ast::AssocItem>),
NtForeignItem(P<ast::ForeignItem>),
}
#[cfg(target_arch = "x86_64")]
rustc_data_structures::static_assert_size!(Nonterminal, 40);
impl PartialEq for Nonterminal {
fn eq(&self, rhs: &Self) -> bool {
match (self, rhs) {
(NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => {
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs
}
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs,
(NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs,
_ => false,
}
}
}
impl fmt::Debug for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
NtItem(..) => f.pad("NtItem(..)"),
NtBlock(..) => f.pad("NtBlock(..)"),
NtStmt(..) => f.pad("NtStmt(..)"),
NtPat(..) => f.pad("NtPat(..)"),
NtExpr(..) => f.pad("NtExpr(..)"),
NtTy(..) => f.pad("NtTy(..)"),
NtIdent(..) => f.pad("NtIdent(..)"),
NtLiteral(..) => f.pad("NtLiteral(..)"),
NtMeta(..) => f.pad("NtMeta(..)"),
NtPath(..) => f.pad("NtPath(..)"),
NtTT(..) => f.pad("NtTT(..)"),
NtImplItem(..) => f.pad("NtImplItem(..)"),
NtTraitItem(..) => f.pad("NtTraitItem(..)"),
NtForeignItem(..) => f.pad("NtForeignItem(..)"),
NtVis(..) => f.pad("NtVis(..)"),
NtLifetime(..) => f.pad("NtLifetime(..)"),
}
}
}
impl<CTX> HashStable<CTX> for Nonterminal
where
CTX: crate::HashStableContext,
{
fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
panic!("interpolated tokens should not be present in the HIR")
}
}