dprint_swc_ext/common/
tokens.rs

1use rustc_hash::FxHashMap;
2
3use super::pos::*;
4use crate::swc::parser::token::TokenAndSpan;
5
6pub struct TokenContainer<'a> {
7  pub tokens: &'a [TokenAndSpan],
8  // Uses an FxHashMap because it has faster lookups for u32 keys than the default hasher.
9  start_to_index: FxHashMap<SourcePos, usize>,
10  end_to_index: FxHashMap<SourcePos, usize>,
11}
12
13impl<'a> TokenContainer<'a> {
14  pub fn new(tokens: &'a [TokenAndSpan]) -> Self {
15    TokenContainer {
16      tokens,
17      start_to_index: tokens.iter().enumerate().map(|(i, token)| (token.start(), i)).collect(),
18      end_to_index: tokens.iter().enumerate().map(|(i, token)| (token.end(), i)).collect(),
19    }
20  }
21
22  pub fn get_token_index_at_start(&self, start: SourcePos) -> Option<usize> {
23    self.start_to_index.get(&start).copied()
24  }
25
26  pub fn get_token_index_at_end(&self, end: SourcePos) -> Option<usize> {
27    self.end_to_index.get(&end).copied()
28  }
29
30  pub fn get_token_at_index(&self, index: usize) -> Option<&TokenAndSpan> {
31    self.tokens.get(index)
32  }
33
34  pub fn get_tokens_in_range(&self, start: SourcePos, end: SourcePos) -> &'a [TokenAndSpan] {
35    let start_index = self.get_leftmost_token_index(start);
36    let end_index = self.get_rightmost_token_index(end);
37
38    let start_index = start_index.unwrap_or_else(|| end_index.unwrap_or(0));
39    let end_index = end_index.map(|i| i + 1).unwrap_or(start_index);
40
41    &self.tokens[start_index..end_index]
42  }
43
44  fn get_leftmost_token_index(&self, start: SourcePos) -> Option<usize> {
45    if let Some(&start_index) = self.start_to_index.get(&start) {
46      Some(start_index)
47    // fallback
48    } else if let Some(&start_index) = self.end_to_index.get(&start) {
49      Some(start_index + 1)
50    } else {
51      // todo: binary search leftmost
52      for (i, token) in self.tokens.iter().enumerate() {
53        if token.start() >= start {
54          return Some(i);
55        }
56      }
57
58      None
59    }
60  }
61
62  fn get_rightmost_token_index(&self, end: SourcePos) -> Option<usize> {
63    if let Some(&end_index) = self.end_to_index.get(&end) {
64      Some(end_index)
65    // fallback
66    } else if let Some(&end_index) = self.start_to_index.get(&end) {
67      if end_index > 0 {
68        Some(end_index - 1)
69      } else {
70        None
71      }
72    } else {
73      // todo: binary search rightmost
74      for (i, token) in self.tokens.iter().enumerate().rev() {
75        if token.end() <= end {
76          return Some(i);
77        }
78      }
79
80      None
81    }
82  }
83
84  pub fn get_previous_token(&self, start: SourcePos) -> Option<&TokenAndSpan> {
85    let index = self.start_to_index.get(&start);
86    if let Some(&index) = index {
87      if index == 0 {
88        None
89      } else {
90        Some(&self.tokens[index - 1])
91      }
92    } else {
93      // todo: binary search leftmost
94      let mut last_token = None;
95      for token in self.tokens {
96        if token.end() > start {
97          return last_token;
98        } else {
99          last_token = Some(token);
100        }
101      }
102
103      None
104    }
105  }
106
107  pub fn get_next_token(&self, end: SourcePos) -> Option<&TokenAndSpan> {
108    if let Some(index) = self.end_to_index.get(&end) {
109      self.tokens.get(index + 1)
110    } else {
111      // todo: binary search rightmost
112      for token in self.tokens {
113        if token.start() > end {
114          return Some(token);
115        }
116      }
117
118      None
119    }
120  }
121}
122
123#[cfg(test)]
124mod test {
125  use std::path::PathBuf;
126
127  use super::super::pos::SourcePos;
128  use super::TokenContainer;
129  use crate::common::SourceRangedForSpanned;
130  use crate::test_helpers::*;
131
132  #[test]
133  fn get_next_token() {
134    let (_, tokens, _, _) = get_swc_module(&PathBuf::from("path.js"), r#"let /* a */ a = 5;"#);
135    let token_container = TokenContainer::new(&tokens);
136    // low token of previous token
137    assert_eq!(token_container.get_next_token(SourcePos::new(0)).unwrap().start(), SourcePos::new(12));
138    // hi of previous token
139    assert_eq!(token_container.get_next_token(SourcePos::new(3)).unwrap().start(), SourcePos::new(12));
140    // in comment before token
141    assert_eq!(token_container.get_next_token(SourcePos::new(5)).unwrap().start(), SourcePos::new(12));
142    // in whitespace before token
143    assert_eq!(token_container.get_next_token(SourcePos::new(11)).unwrap().start(), SourcePos::new(12));
144    // at hi of last token
145    assert_eq!(token_container.get_next_token(SourcePos::new(18)), None);
146  }
147}
dprint_swc_ext/common/tokens.rs

dprint_swc_ext/common/
tokens.rs
