pub_just/
lexer.rs

1use {super::*, CompileErrorKind::*, TokenKind::*};
2
3/// Just language lexer
4///
5/// The lexer proceeds character-by-character, as opposed to using regular
6/// expressions to lex tokens or semi-tokens at a time. As a result, it is
7/// verbose and straightforward. Just used to have a regex-based lexer, which
8/// was slower and generally godawful.  However, this should not be taken as a
9/// slight against regular expressions, the lexer was just idiosyncratically
10/// bad.
11pub struct Lexer<'src> {
12  /// Char iterator
13  chars: Chars<'src>,
14  /// Indentation stack
15  indentation: Vec<&'src str>,
16  /// Interpolation token start stack
17  interpolation_stack: Vec<Token<'src>>,
18  /// Next character to be lexed
19  next: Option<char>,
20  /// Current open delimiters
21  open_delimiters: Vec<(Delimiter, usize)>,
22  /// Path to source file
23  path: &'src Path,
24  /// Inside recipe body
25  recipe_body: bool,
26  /// Next indent will start a recipe body
27  recipe_body_pending: bool,
28  /// Source text
29  src: &'src str,
30  /// Tokens
31  tokens: Vec<Token<'src>>,
32  /// Current token end
33  token_end: Position,
34  /// Current token start
35  token_start: Position,
36}
37
38impl<'src> Lexer<'src> {
39  /// Lex `src`
40  pub fn lex(path: &'src Path, src: &'src str) -> CompileResult<'src, Vec<Token<'src>>> {
41    Self::new(path, src).tokenize()
42  }
43
44  #[cfg(test)]
45  pub fn test_lex(src: &'src str) -> CompileResult<'src, Vec<Token<'src>>> {
46    Self::new("justfile".as_ref(), src).tokenize()
47  }
48
49  /// Create a new Lexer to lex `src`
50  fn new(path: &'src Path, src: &'src str) -> Self {
51    let mut chars = src.chars();
52    let next = chars.next();
53
54    let start = Position {
55      offset: 0,
56      column: 0,
57      line: 0,
58    };
59
60    Self {
61      indentation: vec![""],
62      tokens: Vec::new(),
63      token_start: start,
64      token_end: start,
65      recipe_body_pending: false,
66      recipe_body: false,
67      interpolation_stack: Vec::new(),
68      open_delimiters: Vec::new(),
69      chars,
70      next,
71      src,
72      path,
73    }
74  }
75
76  /// Advance over the character in `self.next`, updating `self.token_end`
77  /// accordingly.
78  fn advance(&mut self) -> CompileResult<'src> {
79    match self.next {
80      Some(c) => {
81        let len_utf8 = c.len_utf8();
82
83        self.token_end.offset += len_utf8;
84        self.token_end.column += len_utf8;
85
86        if c == '\n' {
87          self.token_end.column = 0;
88          self.token_end.line += 1;
89        }
90
91        self.next = self.chars.next();
92
93        Ok(())
94      }
95      None => Err(self.internal_error("Lexer advanced past end of text")),
96    }
97  }
98
99  /// Advance over N characters.
100  fn skip(&mut self, n: usize) -> CompileResult<'src> {
101    for _ in 0..n {
102      self.advance()?;
103    }
104
105    Ok(())
106  }
107
108  /// Lexeme of in-progress token
109  fn lexeme(&self) -> &'src str {
110    &self.src[self.token_start.offset..self.token_end.offset]
111  }
112
113  /// Length of current token
114  fn current_token_length(&self) -> usize {
115    self.token_end.offset - self.token_start.offset
116  }
117
118  fn accepted(&mut self, c: char) -> CompileResult<'src, bool> {
119    if self.next_is(c) {
120      self.advance()?;
121      Ok(true)
122    } else {
123      Ok(false)
124    }
125  }
126
127  fn presume(&mut self, c: char) -> CompileResult<'src> {
128    if !self.next_is(c) {
129      return Err(self.internal_error(format!("Lexer presumed character `{c}`")));
130    }
131
132    self.advance()?;
133
134    Ok(())
135  }
136
137  fn presume_str(&mut self, s: &str) -> CompileResult<'src> {
138    for c in s.chars() {
139      self.presume(c)?;
140    }
141
142    Ok(())
143  }
144
145  /// Is next character c?
146  fn next_is(&self, c: char) -> bool {
147    self.next == Some(c)
148  }
149
150  /// Is next character ' ' or '\t'?
151  fn next_is_whitespace(&self) -> bool {
152    self.next_is(' ') || self.next_is('\t')
153  }
154
155  /// Un-lexed text
156  fn rest(&self) -> &'src str {
157    &self.src[self.token_end.offset..]
158  }
159
160  /// Check if unlexed text begins with prefix
161  fn rest_starts_with(&self, prefix: &str) -> bool {
162    self.rest().starts_with(prefix)
163  }
164
165  /// Does rest start with "\n" or "\r\n"?
166  fn at_eol(&self) -> bool {
167    self.next_is('\n') || self.rest_starts_with("\r\n")
168  }
169
170  /// Are we at end-of-file?
171  fn at_eof(&self) -> bool {
172    self.rest().is_empty()
173  }
174
175  /// Are we at end-of-line or end-of-file?
176  fn at_eol_or_eof(&self) -> bool {
177    self.at_eol() || self.at_eof()
178  }
179
180  /// Get current indentation
181  fn indentation(&self) -> &'src str {
182    self.indentation.last().unwrap()
183  }
184
185  /// Are we currently indented
186  fn indented(&self) -> bool {
187    !self.indentation().is_empty()
188  }
189
190  /// Create a new token with `kind` whose lexeme is between `self.token_start`
191  /// and `self.token_end`
192  fn token(&mut self, kind: TokenKind) {
193    self.tokens.push(Token {
194      offset: self.token_start.offset,
195      column: self.token_start.column,
196      line: self.token_start.line,
197      src: self.src,
198      length: self.token_end.offset - self.token_start.offset,
199      kind,
200      path: self.path,
201    });
202
203    // Set `token_start` to point after the lexed token
204    self.token_start = self.token_end;
205  }
206
207  /// Create an internal error with `message`
208  fn internal_error(&self, message: impl Into<String>) -> CompileError<'src> {
209    // Use `self.token_end` as the location of the error
210    let token = Token {
211      src: self.src,
212      offset: self.token_end.offset,
213      line: self.token_end.line,
214      column: self.token_end.column,
215      length: 0,
216      kind: Unspecified,
217      path: self.path,
218    };
219    CompileError::new(
220      token,
221      Internal {
222        message: message.into(),
223      },
224    )
225  }
226
227  /// Create a compilation error with `kind`
228  fn error(&self, kind: CompileErrorKind<'src>) -> CompileError<'src> {
229    // Use the in-progress token span as the location of the error.
230
231    // The width of the error site to highlight depends on the kind of error:
232    let length = match kind {
233      UnterminatedString | UnterminatedBacktick => {
234        let Some(kind) = StringKind::from_token_start(self.lexeme()) else {
235          return self.internal_error("Lexer::error: expected string or backtick token start");
236        };
237        kind.delimiter().len()
238      }
239      // highlight the full token
240      _ => self.lexeme().len(),
241    };
242
243    let token = Token {
244      kind: Unspecified,
245      src: self.src,
246      offset: self.token_start.offset,
247      line: self.token_start.line,
248      column: self.token_start.column,
249      length,
250      path: self.path,
251    };
252
253    CompileError::new(token, kind)
254  }
255
256  fn unterminated_interpolation_error(interpolation_start: Token<'src>) -> CompileError<'src> {
257    CompileError::new(interpolation_start, UnterminatedInterpolation)
258  }
259
260  /// True if `text` could be an identifier
261  pub fn is_identifier(text: &str) -> bool {
262    if !text.chars().next().map_or(false, Self::is_identifier_start) {
263      return false;
264    }
265
266    for c in text.chars().skip(1) {
267      if !Self::is_identifier_continue(c) {
268        return false;
269      }
270    }
271
272    true
273  }
274
275  /// True if `c` can be the first character of an identifier
276  pub fn is_identifier_start(c: char) -> bool {
277    matches!(c, 'a'..='z' | 'A'..='Z' | '_')
278  }
279
280  /// True if `c` can be a continuation character of an identifier
281  pub fn is_identifier_continue(c: char) -> bool {
282    Self::is_identifier_start(c) || matches!(c, '0'..='9' | '-')
283  }
284
285  /// Consume the text and produce a series of tokens
286  fn tokenize(mut self) -> CompileResult<'src, Vec<Token<'src>>> {
287    loop {
288      if self.token_start.column == 0 {
289        self.lex_line_start()?;
290      }
291
292      match self.next {
293        Some(first) => {
294          if let Some(&interpolation_start) = self.interpolation_stack.last() {
295            self.lex_interpolation(interpolation_start, first)?;
296          } else if self.recipe_body {
297            self.lex_body()?;
298          } else {
299            self.lex_normal(first)?;
300          };
301        }
302        None => break,
303      }
304    }
305
306    if let Some(&interpolation_start) = self.interpolation_stack.last() {
307      return Err(Self::unterminated_interpolation_error(interpolation_start));
308    }
309
310    while self.indented() {
311      self.lex_dedent();
312    }
313
314    self.token(Eof);
315
316    assert_eq!(self.token_start.offset, self.token_end.offset);
317    assert_eq!(self.token_start.offset, self.src.len());
318    assert_eq!(self.indentation.len(), 1);
319
320    Ok(self.tokens)
321  }
322
323  /// Handle blank lines and indentation
324  fn lex_line_start(&mut self) -> CompileResult<'src> {
325    enum Indentation<'src> {
326      // Line only contains whitespace
327      Blank,
328      // Indentation continues
329      Continue,
330      // Indentation decreases
331      Decrease,
332      // Indentation isn't consistent
333      Inconsistent,
334      // Indentation increases
335      Increase,
336      // Indentation mixes spaces and tabs
337      Mixed { whitespace: &'src str },
338    }
339
340    use Indentation::*;
341
342    let nonblank_index = self
343      .rest()
344      .char_indices()
345      .skip_while(|&(_, c)| c == ' ' || c == '\t')
346      .map(|(i, _)| i)
347      .next()
348      .unwrap_or_else(|| self.rest().len());
349
350    let rest = &self.rest()[nonblank_index..];
351
352    let whitespace = &self.rest()[..nonblank_index];
353
354    let body_whitespace = &whitespace[..whitespace
355      .char_indices()
356      .take(self.indentation().chars().count())
357      .map(|(i, _c)| i)
358      .next()
359      .unwrap_or(0)];
360
361    let spaces = whitespace.chars().any(|c| c == ' ');
362    let tabs = whitespace.chars().any(|c| c == '\t');
363
364    let body_spaces = body_whitespace.chars().any(|c| c == ' ');
365    let body_tabs = body_whitespace.chars().any(|c| c == '\t');
366
367    #[allow(clippy::if_same_then_else)]
368    let indentation = if rest.starts_with('\n') || rest.starts_with("\r\n") || rest.is_empty() {
369      Blank
370    } else if whitespace == self.indentation() {
371      Continue
372    } else if self.indentation.contains(&whitespace) {
373      Decrease
374    } else if self.recipe_body && whitespace.starts_with(self.indentation()) {
375      Continue
376    } else if self.recipe_body && body_spaces && body_tabs {
377      Mixed {
378        whitespace: body_whitespace,
379      }
380    } else if !self.recipe_body && spaces && tabs {
381      Mixed { whitespace }
382    } else if whitespace.len() < self.indentation().len() {
383      Inconsistent
384    } else if self.recipe_body
385      && body_whitespace.len() >= self.indentation().len()
386      && !body_whitespace.starts_with(self.indentation())
387    {
388      Inconsistent
389    } else if whitespace.len() >= self.indentation().len()
390      && !whitespace.starts_with(self.indentation())
391    {
392      Inconsistent
393    } else {
394      Increase
395    };
396
397    match indentation {
398      Blank => {
399        if !whitespace.is_empty() {
400          while self.next_is_whitespace() {
401            self.advance()?;
402          }
403
404          self.token(Whitespace);
405        };
406
407        Ok(())
408      }
409      Continue => {
410        if !self.indentation().is_empty() {
411          for _ in self.indentation().chars() {
412            self.advance()?;
413          }
414
415          self.token(Whitespace);
416        }
417
418        Ok(())
419      }
420      Decrease => {
421        while self.indentation() != whitespace {
422          self.lex_dedent();
423        }
424
425        if !whitespace.is_empty() {
426          while self.next_is_whitespace() {
427            self.advance()?;
428          }
429
430          self.token(Whitespace);
431        }
432
433        Ok(())
434      }
435      Mixed { whitespace } => {
436        for _ in whitespace.chars() {
437          self.advance()?;
438        }
439
440        Err(self.error(MixedLeadingWhitespace { whitespace }))
441      }
442      Inconsistent => {
443        for _ in whitespace.chars() {
444          self.advance()?;
445        }
446
447        Err(self.error(InconsistentLeadingWhitespace {
448          expected: self.indentation(),
449          found: whitespace,
450        }))
451      }
452      Increase => {
453        while self.next_is_whitespace() {
454          self.advance()?;
455        }
456
457        if self.open_delimiters() {
458          self.token(Whitespace);
459        } else {
460          let indentation = self.lexeme();
461          self.indentation.push(indentation);
462          self.token(Indent);
463          if self.recipe_body_pending {
464            self.recipe_body = true;
465          }
466        }
467
468        Ok(())
469      }
470    }
471  }
472
473  /// Lex token beginning with `start` outside of a recipe body
474  fn lex_normal(&mut self, start: char) -> CompileResult<'src> {
475    match start {
476      ' ' | '\t' => self.lex_whitespace(),
477      '!' if self.rest().starts_with("!include") => Err(self.error(Include)),
478      '!' => self.lex_digraph('!', '=', BangEquals),
479      '#' => self.lex_comment(),
480      '$' => self.lex_single(Dollar),
481      '&' => self.lex_digraph('&', '&', AmpersandAmpersand),
482      '(' => self.lex_delimiter(ParenL),
483      ')' => self.lex_delimiter(ParenR),
484      '*' => self.lex_single(Asterisk),
485      '+' => self.lex_single(Plus),
486      ',' => self.lex_single(Comma),
487      '/' => self.lex_single(Slash),
488      ':' => self.lex_colon(),
489      '=' => self.lex_choices('=', &[('=', EqualsEquals), ('~', EqualsTilde)], Equals),
490      '?' => self.lex_single(QuestionMark),
491      '@' => self.lex_single(At),
492      '[' => self.lex_delimiter(BracketL),
493      '\\' => self.lex_escape(),
494      '\n' | '\r' => self.lex_eol(),
495      '\u{feff}' => self.lex_single(ByteOrderMark),
496      ']' => self.lex_delimiter(BracketR),
497      '`' | '"' | '\'' => self.lex_string(),
498      '{' => self.lex_delimiter(BraceL),
499      '|' => self.lex_digraph('|', '|', BarBar),
500      '}' => self.lex_delimiter(BraceR),
501      _ if Self::is_identifier_start(start) => self.lex_identifier(),
502      _ => {
503        self.advance()?;
504        Err(self.error(UnknownStartOfToken))
505      }
506    }
507  }
508
509  /// Lex token beginning with `start` inside an interpolation
510  fn lex_interpolation(
511    &mut self,
512    interpolation_start: Token<'src>,
513    start: char,
514  ) -> CompileResult<'src> {
515    if self.rest_starts_with("}}") {
516      // end current interpolation
517      if self.interpolation_stack.pop().is_none() {
518        self.advance()?;
519        self.advance()?;
520        return Err(self.internal_error(
521          "Lexer::lex_interpolation found `}}` but was called with empty interpolation stack.",
522        ));
523      }
524      // Emit interpolation end token
525      self.lex_double(InterpolationEnd)
526    } else if self.at_eol_or_eof() {
527      // Return unterminated interpolation error that highlights the opening
528      // {{
529      Err(Self::unterminated_interpolation_error(interpolation_start))
530    } else {
531      // Otherwise lex as per normal
532      self.lex_normal(start)
533    }
534  }
535
536  /// Lex token while in recipe body
537  fn lex_body(&mut self) -> CompileResult<'src> {
538    enum Terminator {
539      Newline,
540      NewlineCarriageReturn,
541      Interpolation,
542      EndOfFile,
543    }
544
545    use Terminator::*;
546
547    let terminator = loop {
548      if self.rest_starts_with("{{{{") {
549        self.skip(4)?;
550        continue;
551      }
552
553      if self.rest_starts_with("\n") {
554        break Newline;
555      }
556
557      if self.rest_starts_with("\r\n") {
558        break NewlineCarriageReturn;
559      }
560
561      if self.rest_starts_with("{{") {
562        break Interpolation;
563      }
564
565      if self.at_eof() {
566        break EndOfFile;
567      }
568
569      self.advance()?;
570    };
571
572    // emit text token containing text so far
573    if self.current_token_length() > 0 {
574      self.token(Text);
575    }
576
577    match terminator {
578      Newline => self.lex_single(Eol),
579      NewlineCarriageReturn => self.lex_double(Eol),
580      Interpolation => {
581        self.lex_double(InterpolationStart)?;
582        self
583          .interpolation_stack
584          .push(self.tokens[self.tokens.len() - 1]);
585        Ok(())
586      }
587      EndOfFile => Ok(()),
588    }
589  }
590
591  fn lex_dedent(&mut self) {
592    assert_eq!(self.current_token_length(), 0);
593    self.token(Dedent);
594    self.indentation.pop();
595    self.recipe_body_pending = false;
596    self.recipe_body = false;
597  }
598
599  /// Lex a single-character token
600  fn lex_single(&mut self, kind: TokenKind) -> CompileResult<'src> {
601    self.advance()?;
602    self.token(kind);
603    Ok(())
604  }
605
606  /// Lex a double-character token
607  fn lex_double(&mut self, kind: TokenKind) -> CompileResult<'src> {
608    self.advance()?;
609    self.advance()?;
610    self.token(kind);
611    Ok(())
612  }
613
614  /// Lex a double-character token of kind `then` if the second character of
615  /// that token would be `second`, otherwise lex a single-character token of
616  /// kind `otherwise`
617  fn lex_choices(
618    &mut self,
619    first: char,
620    choices: &[(char, TokenKind)],
621    otherwise: TokenKind,
622  ) -> CompileResult<'src> {
623    self.presume(first)?;
624
625    for (second, then) in choices {
626      if self.accepted(*second)? {
627        self.token(*then);
628        return Ok(());
629      }
630    }
631
632    self.token(otherwise);
633
634    Ok(())
635  }
636
637  /// Lex an opening or closing delimiter
638  fn lex_delimiter(&mut self, kind: TokenKind) -> CompileResult<'src> {
639    use Delimiter::*;
640
641    match kind {
642      BraceL => self.open_delimiter(Brace),
643      BraceR => self.close_delimiter(Brace)?,
644      BracketL => self.open_delimiter(Bracket),
645      BracketR => self.close_delimiter(Bracket)?,
646      ParenL => self.open_delimiter(Paren),
647      ParenR => self.close_delimiter(Paren)?,
648      _ => {
649        return Err(self.internal_error(format!(
650          "Lexer::lex_delimiter called with non-delimiter token: `{kind}`",
651        )))
652      }
653    }
654
655    // Emit the delimiter token
656    self.lex_single(kind)
657  }
658
659  /// Push a delimiter onto the open delimiter stack
660  fn open_delimiter(&mut self, delimiter: Delimiter) {
661    self
662      .open_delimiters
663      .push((delimiter, self.token_start.line));
664  }
665
666  /// Pop a delimiter from the open delimiter stack and error if incorrect type
667  fn close_delimiter(&mut self, close: Delimiter) -> CompileResult<'src> {
668    match self.open_delimiters.pop() {
669      Some((open, _)) if open == close => Ok(()),
670      Some((open, open_line)) => Err(self.error(MismatchedClosingDelimiter {
671        open,
672        close,
673        open_line,
674      })),
675      None => Err(self.error(UnexpectedClosingDelimiter { close })),
676    }
677  }
678
679  /// Return true if there are any unclosed delimiters
680  fn open_delimiters(&self) -> bool {
681    !self.open_delimiters.is_empty()
682  }
683
684  /// Lex a two-character digraph
685  fn lex_digraph(&mut self, left: char, right: char, token: TokenKind) -> CompileResult<'src> {
686    self.presume(left)?;
687
688    if self.accepted(right)? {
689      self.token(token);
690      Ok(())
691    } else {
692      // Emit an unspecified token to consume the current character,
693      self.token(Unspecified);
694
695      if self.at_eof() {
696        return Err(self.error(UnexpectedEndOfToken { expected: right }));
697      }
698
699      // …and advance past another character,
700      self.advance()?;
701
702      // …so that the error we produce highlights the unexpected character.
703      Err(self.error(UnexpectedCharacter { expected: right }))
704    }
705  }
706
707  /// Lex a token starting with ':'
708  fn lex_colon(&mut self) -> CompileResult<'src> {
709    self.presume(':')?;
710
711    if self.accepted('=')? {
712      self.token(ColonEquals);
713    } else {
714      self.token(Colon);
715      self.recipe_body_pending = true;
716    }
717
718    Ok(())
719  }
720
721  /// Lex an token starting with '\' escape
722  fn lex_escape(&mut self) -> CompileResult<'src> {
723    self.presume('\\')?;
724
725    // Treat newline escaped with \ as whitespace
726    if self.accepted('\n')? {
727      while self.next_is_whitespace() {
728        self.advance()?;
729      }
730      self.token(Whitespace);
731    } else if self.accepted('\r')? {
732      if !self.accepted('\n')? {
733        return Err(self.error(UnpairedCarriageReturn));
734      }
735      while self.next_is_whitespace() {
736        self.advance()?;
737      }
738      self.token(Whitespace);
739    } else if let Some(character) = self.next {
740      return Err(self.error(InvalidEscapeSequence { character }));
741    }
742
743    Ok(())
744  }
745
746  /// Lex a carriage return and line feed
747  fn lex_eol(&mut self) -> CompileResult<'src> {
748    if self.accepted('\r')? {
749      if !self.accepted('\n')? {
750        return Err(self.error(UnpairedCarriageReturn));
751      }
752    } else {
753      self.presume('\n')?;
754    }
755
756    // Emit an eol if there are no open delimiters, otherwise emit a whitespace
757    // token.
758    if self.open_delimiters() {
759      self.token(Whitespace);
760    } else {
761      self.token(Eol);
762    }
763
764    Ok(())
765  }
766
767  /// Lex name: [a-zA-Z_][a-zA-Z0-9_]*
768  fn lex_identifier(&mut self) -> CompileResult<'src> {
769    self.advance()?;
770
771    while let Some(c) = self.next {
772      if !Self::is_identifier_continue(c) {
773        break;
774      }
775
776      self.advance()?;
777    }
778
779    self.token(Identifier);
780
781    Ok(())
782  }
783
784  /// Lex comment: #[^\r\n]
785  fn lex_comment(&mut self) -> CompileResult<'src> {
786    self.presume('#')?;
787
788    while !self.at_eol_or_eof() {
789      self.advance()?;
790    }
791
792    self.token(Comment);
793
794    Ok(())
795  }
796
797  /// Lex whitespace: [ \t]+
798  fn lex_whitespace(&mut self) -> CompileResult<'src> {
799    while self.next_is_whitespace() {
800      self.advance()?;
801    }
802
803    self.token(Whitespace);
804
805    Ok(())
806  }
807
808  /// Lex a backtick, cooked string, or raw string.
809  ///
810  /// Backtick:      ``[^`]*``
811  /// Cooked string: "[^"]*" # also processes escape sequences
812  /// Raw string:    '[^']*'
813  fn lex_string(&mut self) -> CompileResult<'src> {
814    let Some(kind) = StringKind::from_token_start(self.rest()) else {
815      self.advance()?;
816      return Err(self.internal_error("Lexer::lex_string: invalid string start"));
817    };
818
819    self.presume_str(kind.delimiter())?;
820
821    let mut escape = false;
822
823    loop {
824      if self.next.is_none() {
825        return Err(self.error(kind.unterminated_error_kind()));
826      } else if kind.processes_escape_sequences() && self.next_is('\\') && !escape {
827        escape = true;
828      } else if self.rest_starts_with(kind.delimiter()) && !escape {
829        break;
830      } else {
831        escape = false;
832      }
833
834      self.advance()?;
835    }
836
837    self.presume_str(kind.delimiter())?;
838    self.token(kind.token_kind());
839
840    Ok(())
841  }
842}
843
844#[cfg(test)]
845mod tests {
846  use super::*;
847
848  use pretty_assertions::assert_eq;
849
850  macro_rules! test {
851    {
852      name:     $name:ident,
853      text:     $text:expr,
854      tokens:   ($($kind:ident $(: $lexeme:literal)?),* $(,)?)$(,)?
855    } => {
856      #[test]
857      fn $name() {
858        let kinds: &[TokenKind] = &[$($kind,)* Eof];
859
860        let lexemes: &[&str] = &[$(lexeme!($kind $(, $lexeme)?),)* ""];
861
862        test($text, true, kinds, lexemes);
863      }
864    };
865    {
866      name:     $name:ident,
867      text:     $text:expr,
868      tokens:   ($($kind:ident $(: $lexeme:literal)?),* $(,)?)$(,)?
869      unindent: $unindent:expr,
870    } => {
871      #[test]
872      fn $name() {
873        let kinds: &[TokenKind] = &[$($kind,)* Eof];
874
875        let lexemes: &[&str] = &[$(lexeme!($kind $(, $lexeme)?),)* ""];
876
877        test($text, $unindent, kinds, lexemes);
878      }
879    }
880  }
881
882  macro_rules! lexeme {
883    {
884      $kind:ident, $lexeme:literal
885    } => {
886      $lexeme
887    };
888    {
889      $kind:ident
890    } => {
891      default_lexeme($kind)
892    }
893  }
894
895  fn test(text: &str, unindent_text: bool, want_kinds: &[TokenKind], want_lexemes: &[&str]) {
896    let text = if unindent_text {
897      unindent(text)
898    } else {
899      text.to_owned()
900    };
901
902    let have = Lexer::test_lex(&text).unwrap();
903
904    let have_kinds = have
905      .iter()
906      .map(|token| token.kind)
907      .collect::<Vec<TokenKind>>();
908
909    let have_lexemes = have.iter().map(Token::lexeme).collect::<Vec<&str>>();
910
911    assert_eq!(have_kinds, want_kinds, "Token kind mismatch");
912    assert_eq!(have_lexemes, want_lexemes, "Token lexeme mismatch");
913
914    let mut roundtrip = String::new();
915
916    for lexeme in have_lexemes {
917      roundtrip.push_str(lexeme);
918    }
919
920    assert_eq!(roundtrip, text, "Roundtrip mismatch");
921
922    let mut offset = 0;
923    let mut line = 0;
924    let mut column = 0;
925
926    for token in have {
927      assert_eq!(token.offset, offset);
928      assert_eq!(token.line, line);
929      assert_eq!(token.lexeme().len(), token.length);
930      assert_eq!(token.column, column);
931
932      for c in token.lexeme().chars() {
933        if c == '\n' {
934          line += 1;
935          column = 0;
936        } else {
937          column += c.len_utf8();
938        }
939      }
940
941      offset += token.length;
942    }
943  }
944
945  fn default_lexeme(kind: TokenKind) -> &'static str {
946    match kind {
947      // Fixed lexemes
948      AmpersandAmpersand => "&&",
949      Asterisk => "*",
950      At => "@",
951      BangEquals => "!=",
952      BarBar => "||",
953      BraceL => "{",
954      BraceR => "}",
955      BracketL => "[",
956      BracketR => "]",
957      ByteOrderMark => "\u{feff}",
958      Colon => ":",
959      ColonEquals => ":=",
960      Comma => ",",
961      Dollar => "$",
962      Eol => "\n",
963      Equals => "=",
964      EqualsEquals => "==",
965      EqualsTilde => "=~",
966      Indent => "  ",
967      InterpolationEnd => "}}",
968      InterpolationStart => "{{",
969      ParenL => "(",
970      ParenR => ")",
971      Plus => "+",
972      QuestionMark => "?",
973      Slash => "/",
974      Whitespace => " ",
975
976      // Empty lexemes
977      Dedent | Eof => "",
978
979      // Variable lexemes
980      Text | StringToken | Backtick | Identifier | Comment | Unspecified => {
981        panic!("Token {kind:?} has no default lexeme")
982      }
983    }
984  }
985
986  macro_rules! error {
987    (
988      name:   $name:ident,
989      input:  $input:expr,
990      offset: $offset:expr,
991      line:   $line:expr,
992      column: $column:expr,
993      width:  $width:expr,
994      kind:   $kind:expr,
995    ) => {
996      #[test]
997      fn $name() {
998        error($input, $offset, $line, $column, $width, $kind);
999      }
1000    };
1001  }
1002
1003  fn error(
1004    src: &str,
1005    offset: usize,
1006    line: usize,
1007    column: usize,
1008    length: usize,
1009    kind: CompileErrorKind,
1010  ) {
1011    match Lexer::test_lex(src) {
1012      Ok(_) => panic!("Lexing succeeded but expected"),
1013      Err(have) => {
1014        let want = CompileError {
1015          token: Token {
1016            kind: have.token.kind,
1017            src,
1018            offset,
1019            line,
1020            column,
1021            length,
1022            path: "justfile".as_ref(),
1023          },
1024          kind: kind.into(),
1025        };
1026        assert_eq!(have, want);
1027      }
1028    }
1029  }
1030
1031  test! {
1032    name:   name_new,
1033    text:   "foo",
1034    tokens: (Identifier:"foo"),
1035  }
1036
1037  test! {
1038    name:   comment,
1039    text:   "# hello",
1040    tokens: (Comment:"# hello"),
1041  }
1042
1043  test! {
1044    name:   backtick,
1045    text:   "`echo`",
1046    tokens: (Backtick:"`echo`"),
1047  }
1048
1049  test! {
1050    name:   backtick_multi_line,
1051    text:   "`echo\necho`",
1052    tokens: (Backtick:"`echo\necho`"),
1053  }
1054
1055  test! {
1056    name:   raw_string,
1057    text:   "'hello'",
1058    tokens: (StringToken:"'hello'"),
1059  }
1060
1061  test! {
1062    name:   raw_string_multi_line,
1063    text:   "'hello\ngoodbye'",
1064    tokens: (StringToken:"'hello\ngoodbye'"),
1065  }
1066
1067  test! {
1068    name:   cooked_string,
1069    text:   "\"hello\"",
1070    tokens: (StringToken:"\"hello\""),
1071  }
1072
1073  test! {
1074    name:   cooked_string_multi_line,
1075    text:   "\"hello\ngoodbye\"",
1076    tokens: (StringToken:"\"hello\ngoodbye\""),
1077  }
1078
1079  test! {
1080    name:   cooked_multiline_string,
1081    text:   "\"\"\"hello\ngoodbye\"\"\"",
1082    tokens: (StringToken:"\"\"\"hello\ngoodbye\"\"\""),
1083  }
1084
1085  test! {
1086    name:   ampersand_ampersand,
1087    text:   "&&",
1088    tokens: (AmpersandAmpersand),
1089  }
1090
1091  test! {
1092    name:   equals,
1093    text:   "=",
1094    tokens: (Equals),
1095  }
1096
1097  test! {
1098    name:   equals_equals,
1099    text:   "==",
1100    tokens: (EqualsEquals),
1101  }
1102
1103  test! {
1104    name:   bang_equals,
1105    text:   "!=",
1106    tokens: (BangEquals),
1107  }
1108
1109  test! {
1110    name:   brace_l,
1111    text:   "{",
1112    tokens: (BraceL),
1113  }
1114
1115  test! {
1116    name:   brace_r,
1117    text:   "{}",
1118    tokens: (BraceL, BraceR),
1119  }
1120
1121  test! {
1122    name:   brace_lll,
1123    text:   "{{{",
1124    tokens: (BraceL, BraceL, BraceL),
1125  }
1126
1127  test! {
1128    name:   brace_rrr,
1129    text:   "{{{}}}",
1130    tokens: (BraceL, BraceL, BraceL, BraceR, BraceR, BraceR),
1131  }
1132
1133  test! {
1134    name:   dollar,
1135    text:   "$",
1136    tokens: (Dollar),
1137  }
1138
1139  test! {
1140    name:   export_concatenation,
1141    text:   "export foo = 'foo' + 'bar'",
1142    tokens: (
1143      Identifier:"export",
1144      Whitespace,
1145      Identifier:"foo",
1146      Whitespace,
1147      Equals,
1148      Whitespace,
1149      StringToken:"'foo'",
1150      Whitespace,
1151      Plus,
1152      Whitespace,
1153      StringToken:"'bar'",
1154    )
1155  }
1156
1157  test! {
1158    name: export_complex,
1159    text: "export foo = ('foo' + 'bar') + `baz`",
1160    tokens: (
1161      Identifier:"export",
1162      Whitespace,
1163      Identifier:"foo",
1164      Whitespace,
1165      Equals,
1166      Whitespace,
1167      ParenL,
1168      StringToken:"'foo'",
1169      Whitespace,
1170      Plus,
1171      Whitespace,
1172      StringToken:"'bar'",
1173      ParenR,
1174      Whitespace,
1175      Plus,
1176      Whitespace,
1177      Backtick:"`baz`",
1178    ),
1179  }
1180
1181  test! {
1182    name:     eol_linefeed,
1183    text:     "\n",
1184    tokens:   (Eol),
1185    unindent: false,
1186  }
1187
1188  test! {
1189    name:     eol_carriage_return_linefeed,
1190    text:     "\r\n",
1191    tokens:   (Eol:"\r\n"),
1192    unindent: false,
1193  }
1194
1195  test! {
1196    name:   indented_line,
1197    text:   "foo:\n a",
1198    tokens: (Identifier:"foo", Colon, Eol, Indent:" ", Text:"a", Dedent),
1199  }
1200
1201  test! {
1202    name:   indented_normal,
1203    text:   "
1204      a
1205        b
1206        c
1207    ",
1208    tokens: (
1209      Identifier:"a",
1210      Eol,
1211      Indent:"  ",
1212      Identifier:"b",
1213      Eol,
1214      Whitespace:"  ",
1215      Identifier:"c",
1216      Eol,
1217      Dedent,
1218    ),
1219  }
1220
1221  test! {
1222    name:   indented_normal_nonempty_blank,
1223    text:   "a\n  b\n\t\t\n  c\n",
1224    tokens: (
1225      Identifier:"a",
1226      Eol,
1227      Indent:"  ",
1228      Identifier:"b",
1229      Eol,
1230      Whitespace:"\t\t",
1231      Eol,
1232      Whitespace:"  ",
1233      Identifier:"c",
1234      Eol,
1235      Dedent,
1236    ),
1237    unindent: false,
1238  }
1239
1240  test! {
1241    name:   indented_normal_multiple,
1242    text:   "
1243      a
1244        b
1245          c
1246    ",
1247    tokens: (
1248      Identifier:"a",
1249      Eol,
1250      Indent:"  ",
1251      Identifier:"b",
1252      Eol,
1253      Indent:"    ",
1254      Identifier:"c",
1255      Eol,
1256      Dedent,
1257      Dedent,
1258    ),
1259  }
1260
1261  test! {
1262    name:   indent_indent_dedent_indent,
1263    text:   "
1264      a
1265        b
1266          c
1267        d
1268          e
1269    ",
1270    tokens: (
1271      Identifier:"a",
1272      Eol,
1273      Indent:"  ",
1274        Identifier:"b",
1275        Eol,
1276        Indent:"    ",
1277          Identifier:"c",
1278          Eol,
1279        Dedent,
1280        Whitespace:"  ",
1281        Identifier:"d",
1282        Eol,
1283        Indent:"    ",
1284          Identifier:"e",
1285          Eol,
1286        Dedent,
1287      Dedent,
1288    ),
1289  }
1290
1291  test! {
1292    name:   indent_recipe_dedent_indent,
1293    text:   "
1294      a
1295        b:
1296          c
1297        d
1298          e
1299    ",
1300    tokens: (
1301      Identifier:"a",
1302      Eol,
1303      Indent:"  ",
1304        Identifier:"b",
1305        Colon,
1306        Eol,
1307        Indent:"    ",
1308          Text:"c",
1309          Eol,
1310        Dedent,
1311        Whitespace:"  ",
1312        Identifier:"d",
1313        Eol,
1314        Indent:"    ",
1315          Identifier:"e",
1316          Eol,
1317        Dedent,
1318      Dedent,
1319    ),
1320  }
1321
1322  test! {
1323    name: indented_block,
1324    text: "
1325      foo:
1326        a
1327        b
1328        c
1329    ",
1330    tokens: (
1331      Identifier:"foo",
1332      Colon,
1333      Eol,
1334      Indent,
1335      Text:"a",
1336      Eol,
1337      Whitespace:"  ",
1338      Text:"b",
1339      Eol,
1340      Whitespace:"  ",
1341      Text:"c",
1342      Eol,
1343      Dedent,
1344    )
1345  }
1346
1347  test! {
1348    name: brace_escape,
1349    text: "
1350      foo:
1351        {{{{
1352    ",
1353    tokens: (
1354      Identifier:"foo",
1355      Colon,
1356      Eol,
1357      Indent,
1358      Text:"{{{{",
1359      Eol,
1360      Dedent,
1361    )
1362  }
1363
1364  test! {
1365    name: indented_block_followed_by_item,
1366    text: "
1367      foo:
1368        a
1369      b:
1370    ",
1371    tokens: (
1372      Identifier:"foo",
1373      Colon,
1374      Eol,
1375      Indent,
1376      Text:"a",
1377      Eol,
1378      Dedent,
1379      Identifier:"b",
1380      Colon,
1381      Eol,
1382    )
1383  }
1384
1385  test! {
1386    name: indented_block_followed_by_blank,
1387    text: "
1388      foo:
1389          a
1390
1391      b:
1392    ",
1393    tokens: (
1394      Identifier:"foo",
1395      Colon,
1396      Eol,
1397      Indent:"    ",
1398      Text:"a",
1399      Eol,
1400      Eol,
1401      Dedent,
1402      Identifier:"b",
1403      Colon,
1404      Eol,
1405    ),
1406  }
1407
1408  test! {
1409    name: indented_line_containing_unpaired_carriage_return,
1410    text: "foo:\n \r \n",
1411    tokens: (
1412      Identifier:"foo",
1413      Colon,
1414      Eol,
1415      Indent:" ",
1416      Text:"\r ",
1417      Eol,
1418      Dedent,
1419    ),
1420    unindent: false,
1421  }
1422
1423  test! {
1424    name: indented_blocks,
1425    text: "
1426      b: a
1427        @mv a b
1428
1429      a:
1430        @touch F
1431        @touch a
1432
1433      d: c
1434        @rm c
1435
1436      c: b
1437        @mv b c
1438    ",
1439    tokens: (
1440      Identifier:"b",
1441      Colon,
1442      Whitespace,
1443      Identifier:"a",
1444      Eol,
1445      Indent,
1446      Text:"@mv a b",
1447      Eol,
1448      Eol,
1449      Dedent,
1450      Identifier:"a",
1451      Colon,
1452      Eol,
1453      Indent,
1454      Text:"@touch F",
1455      Eol,
1456      Whitespace:"  ",
1457      Text:"@touch a",
1458      Eol,
1459      Eol,
1460      Dedent,
1461      Identifier:"d",
1462      Colon,
1463      Whitespace,
1464      Identifier:"c",
1465      Eol,
1466      Indent,
1467      Text:"@rm c",
1468      Eol,
1469      Eol,
1470      Dedent,
1471      Identifier:"c",
1472      Colon,
1473      Whitespace,
1474      Identifier:"b",
1475      Eol,
1476      Indent,
1477      Text:"@mv b c",
1478      Eol,
1479      Dedent
1480    ),
1481  }
1482
1483  test! {
1484    name: interpolation_empty,
1485    text: "hello:\n echo {{}}",
1486    tokens: (
1487      Identifier:"hello",
1488      Colon,
1489      Eol,
1490      Indent:" ",
1491      Text:"echo ",
1492      InterpolationStart,
1493      InterpolationEnd,
1494      Dedent,
1495    ),
1496  }
1497
1498  test! {
1499    name: interpolation_expression,
1500    text: "hello:\n echo {{`echo hello` + `echo goodbye`}}",
1501    tokens: (
1502      Identifier:"hello",
1503      Colon,
1504      Eol,
1505      Indent:" ",
1506      Text:"echo ",
1507      InterpolationStart,
1508      Backtick:"`echo hello`",
1509      Whitespace,
1510      Plus,
1511      Whitespace,
1512      Backtick:"`echo goodbye`",
1513      InterpolationEnd,
1514      Dedent,
1515    ),
1516  }
1517
1518  test! {
1519    name: interpolation_raw_multiline_string,
1520    text: "hello:\n echo {{'\n'}}",
1521    tokens: (
1522      Identifier:"hello",
1523      Colon,
1524      Eol,
1525      Indent:" ",
1526      Text:"echo ",
1527      InterpolationStart,
1528      StringToken:"'\n'",
1529      InterpolationEnd,
1530      Dedent,
1531    ),
1532  }
1533
1534  test! {
1535    name: tokenize_names,
1536    text: "
1537      foo
1538      bar-bob
1539      b-bob_asdfAAAA
1540      test123
1541    ",
1542    tokens: (
1543      Identifier:"foo",
1544      Eol,
1545      Identifier:"bar-bob",
1546      Eol,
1547      Identifier:"b-bob_asdfAAAA",
1548      Eol,
1549      Identifier:"test123",
1550      Eol,
1551    ),
1552  }
1553
1554  test! {
1555    name: tokenize_indented_line,
1556    text: "foo:\n a",
1557    tokens: (
1558      Identifier:"foo",
1559      Colon,
1560      Eol,
1561      Indent:" ",
1562      Text:"a",
1563      Dedent,
1564    ),
1565  }
1566
1567  test! {
1568    name: tokenize_indented_block,
1569    text: "
1570      foo:
1571        a
1572        b
1573        c
1574    ",
1575    tokens: (
1576      Identifier:"foo",
1577      Colon,
1578      Eol,
1579      Indent,
1580      Text:"a",
1581      Eol,
1582      Whitespace:"  ",
1583      Text:"b",
1584      Eol,
1585      Whitespace:"  ",
1586      Text:"c",
1587      Eol,
1588      Dedent,
1589    ),
1590  }
1591
1592  test! {
1593    name: tokenize_strings,
1594    text: r#"a = "'a'" + '"b"' + "'c'" + '"d"'#echo hello"#,
1595    tokens: (
1596      Identifier:"a",
1597      Whitespace,
1598      Equals,
1599      Whitespace,
1600      StringToken:"\"'a'\"",
1601      Whitespace,
1602      Plus,
1603      Whitespace,
1604      StringToken:"'\"b\"'",
1605      Whitespace,
1606      Plus,
1607      Whitespace,
1608      StringToken:"\"'c'\"",
1609      Whitespace,
1610      Plus,
1611      Whitespace,
1612      StringToken:"'\"d\"'",
1613      Comment:"#echo hello",
1614    )
1615  }
1616
1617  test! {
1618    name: tokenize_recipe_interpolation_eol,
1619    text: "
1620      foo: # some comment
1621       {{hello}}
1622    ",
1623    tokens: (
1624      Identifier:"foo",
1625      Colon,
1626      Whitespace,
1627      Comment:"# some comment",
1628      Eol,
1629      Indent:" ",
1630      InterpolationStart,
1631      Identifier:"hello",
1632      InterpolationEnd,
1633      Eol,
1634      Dedent
1635    ),
1636  }
1637
1638  test! {
1639    name: tokenize_recipe_interpolation_eof,
1640    text: "foo: # more comments
1641 {{hello}}
1642# another comment
1643",
1644    tokens: (
1645      Identifier:"foo",
1646      Colon,
1647      Whitespace,
1648      Comment:"# more comments",
1649      Eol,
1650      Indent:" ",
1651      InterpolationStart,
1652      Identifier:"hello",
1653      InterpolationEnd,
1654      Eol,
1655      Dedent,
1656      Comment:"# another comment",
1657      Eol,
1658    ),
1659  }
1660
1661  test! {
1662    name: tokenize_recipe_complex_interpolation_expression,
1663    text: "foo: #lol\n {{a + b + \"z\" + blarg}}",
1664    tokens: (
1665      Identifier:"foo",
1666      Colon,
1667      Whitespace:" ",
1668      Comment:"#lol",
1669      Eol,
1670      Indent:" ",
1671      InterpolationStart,
1672      Identifier:"a",
1673      Whitespace,
1674      Plus,
1675      Whitespace,
1676      Identifier:"b",
1677      Whitespace,
1678      Plus,
1679      Whitespace,
1680      StringToken:"\"z\"",
1681      Whitespace,
1682      Plus,
1683      Whitespace,
1684      Identifier:"blarg",
1685      InterpolationEnd,
1686      Dedent,
1687    ),
1688  }
1689
1690  test! {
1691    name: tokenize_recipe_multiple_interpolations,
1692    text: "foo:,#ok\n {{a}}0{{b}}1{{c}}",
1693    tokens: (
1694      Identifier:"foo",
1695      Colon,
1696      Comma,
1697      Comment:"#ok",
1698      Eol,
1699      Indent:" ",
1700      InterpolationStart,
1701      Identifier:"a",
1702      InterpolationEnd,
1703      Text:"0",
1704      InterpolationStart,
1705      Identifier:"b",
1706      InterpolationEnd,
1707      Text:"1",
1708      InterpolationStart,
1709      Identifier:"c",
1710      InterpolationEnd,
1711      Dedent,
1712
1713    ),
1714  }
1715
1716  test! {
1717    name: tokenize_junk,
1718    text: "
1719      bob
1720
1721      hello blah blah blah : a b c #whatever
1722    ",
1723    tokens: (
1724      Identifier:"bob",
1725      Eol,
1726      Eol,
1727      Identifier:"hello",
1728      Whitespace,
1729      Identifier:"blah",
1730      Whitespace,
1731      Identifier:"blah",
1732      Whitespace,
1733      Identifier:"blah",
1734      Whitespace,
1735      Colon,
1736      Whitespace,
1737      Identifier:"a",
1738      Whitespace,
1739      Identifier:"b",
1740      Whitespace,
1741      Identifier:"c",
1742      Whitespace,
1743      Comment:"#whatever",
1744      Eol,
1745    )
1746  }
1747
1748  test! {
1749    name: tokenize_empty_lines,
1750    text: "
1751
1752      # this does something
1753      hello:
1754        asdf
1755        bsdf
1756
1757        csdf
1758
1759        dsdf # whatever
1760
1761      # yolo
1762    ",
1763    tokens: (
1764      Eol,
1765      Comment:"# this does something",
1766      Eol,
1767      Identifier:"hello",
1768      Colon,
1769      Eol,
1770      Indent,
1771      Text:"asdf",
1772      Eol,
1773      Whitespace:"  ",
1774      Text:"bsdf",
1775      Eol,
1776      Eol,
1777      Whitespace:"  ",
1778      Text:"csdf",
1779      Eol,
1780      Eol,
1781      Whitespace:"  ",
1782      Text:"dsdf # whatever",
1783      Eol,
1784      Eol,
1785      Dedent,
1786      Comment:"# yolo",
1787      Eol,
1788    ),
1789  }
1790
1791  test! {
1792    name: tokenize_comment_before_variable,
1793    text: "
1794      #
1795      A='1'
1796      echo:
1797        echo {{A}}
1798    ",
1799    tokens: (
1800      Comment:"#",
1801      Eol,
1802      Identifier:"A",
1803      Equals,
1804      StringToken:"'1'",
1805      Eol,
1806      Identifier:"echo",
1807      Colon,
1808      Eol,
1809      Indent,
1810      Text:"echo ",
1811      InterpolationStart,
1812      Identifier:"A",
1813      InterpolationEnd,
1814      Eol,
1815      Dedent,
1816    ),
1817  }
1818
1819  test! {
1820    name: tokenize_interpolation_backticks,
1821    text: "hello:\n echo {{`echo hello` + `echo goodbye`}}",
1822    tokens: (
1823      Identifier:"hello",
1824      Colon,
1825      Eol,
1826      Indent:" ",
1827      Text:"echo ",
1828      InterpolationStart,
1829      Backtick:"`echo hello`",
1830      Whitespace,
1831      Plus,
1832      Whitespace,
1833      Backtick:"`echo goodbye`",
1834      InterpolationEnd,
1835      Dedent
1836    ),
1837  }
1838
1839  test! {
1840    name: tokenize_empty_interpolation,
1841    text: "hello:\n echo {{}}",
1842    tokens: (
1843      Identifier:"hello",
1844      Colon,
1845      Eol,
1846      Indent:" ",
1847      Text:"echo ",
1848      InterpolationStart,
1849      InterpolationEnd,
1850      Dedent,
1851    ),
1852  }
1853
1854  test! {
1855    name: tokenize_assignment_backticks,
1856    text: "a = `echo hello` + `echo goodbye`",
1857    tokens: (
1858      Identifier:"a",
1859      Whitespace,
1860      Equals,
1861      Whitespace,
1862      Backtick:"`echo hello`",
1863      Whitespace,
1864      Plus,
1865      Whitespace,
1866      Backtick:"`echo goodbye`",
1867    ),
1868  }
1869
1870  test! {
1871    name: tokenize_multiple,
1872    text: "
1873
1874      hello:
1875        a
1876        b
1877
1878        c
1879
1880        d
1881
1882      # hello
1883      bob:
1884        frank
1885       \t
1886    ",
1887    tokens: (
1888      Eol,
1889      Identifier:"hello",
1890      Colon,
1891      Eol,
1892      Indent,
1893      Text:"a",
1894      Eol,
1895      Whitespace:"  ",
1896      Text:"b",
1897      Eol,
1898      Eol,
1899      Whitespace:"  ",
1900      Text:"c",
1901      Eol,
1902      Eol,
1903      Whitespace:"  ",
1904      Text:"d",
1905      Eol,
1906      Eol,
1907      Dedent,
1908      Comment:"# hello",
1909      Eol,
1910      Identifier:"bob",
1911      Colon,
1912      Eol,
1913      Indent:"  ",
1914      Text:"frank",
1915      Eol,
1916      Eol,
1917      Dedent,
1918    ),
1919  }
1920
1921  test! {
1922    name: tokenize_comment,
1923    text: "a:=#",
1924    tokens: (
1925      Identifier:"a",
1926      ColonEquals,
1927      Comment:"#",
1928    ),
1929  }
1930
1931  test! {
1932    name: tokenize_comment_with_bang,
1933    text: "a:=#foo!",
1934    tokens: (
1935      Identifier:"a",
1936      ColonEquals,
1937      Comment:"#foo!",
1938    ),
1939  }
1940
1941  test! {
1942    name: tokenize_order,
1943    text: "
1944      b: a
1945        @mv a b
1946
1947      a:
1948        @touch F
1949        @touch a
1950
1951      d: c
1952        @rm c
1953
1954      c: b
1955        @mv b c
1956    ",
1957    tokens: (
1958      Identifier:"b",
1959      Colon,
1960      Whitespace,
1961      Identifier:"a",
1962      Eol,
1963      Indent,
1964      Text:"@mv a b",
1965      Eol,
1966      Eol,
1967      Dedent,
1968      Identifier:"a",
1969      Colon,
1970      Eol,
1971      Indent,
1972      Text:"@touch F",
1973      Eol,
1974      Whitespace:"  ",
1975      Text:"@touch a",
1976      Eol,
1977      Eol,
1978      Dedent,
1979      Identifier:"d",
1980      Colon,
1981      Whitespace,
1982      Identifier:"c",
1983      Eol,
1984      Indent,
1985      Text:"@rm c",
1986      Eol,
1987      Eol,
1988      Dedent,
1989      Identifier:"c",
1990      Colon,
1991      Whitespace,
1992      Identifier:"b",
1993      Eol,
1994      Indent,
1995      Text:"@mv b c",
1996      Eol,
1997      Dedent,
1998    ),
1999  }
2000
2001  test! {
2002    name: tokenize_parens,
2003    text: "((())) ()abc(+",
2004    tokens: (
2005      ParenL,
2006      ParenL,
2007      ParenL,
2008      ParenR,
2009      ParenR,
2010      ParenR,
2011      Whitespace,
2012      ParenL,
2013      ParenR,
2014      Identifier:"abc",
2015      ParenL,
2016      Plus,
2017    ),
2018  }
2019
2020  test! {
2021    name: crlf_newline,
2022    text: "#\r\n#asdf\r\n",
2023    tokens: (
2024      Comment:"#",
2025      Eol:"\r\n",
2026      Comment:"#asdf",
2027      Eol:"\r\n",
2028    ),
2029  }
2030
2031  test! {
2032    name: multiple_recipes,
2033    text: "a:\n  foo\nb:",
2034    tokens: (
2035      Identifier:"a",
2036      Colon,
2037      Eol,
2038      Indent:"  ",
2039      Text:"foo",
2040      Eol,
2041      Dedent,
2042      Identifier:"b",
2043      Colon,
2044    ),
2045  }
2046
2047  test! {
2048    name:   brackets,
2049    text:   "[][]",
2050    tokens: (BracketL, BracketR, BracketL, BracketR),
2051  }
2052
2053  test! {
2054    name:   open_delimiter_eol,
2055    text:   "[\n](\n){\n}",
2056    tokens: (
2057      BracketL, Whitespace:"\n", BracketR,
2058      ParenL, Whitespace:"\n", ParenR,
2059      BraceL, Whitespace:"\n", BraceR
2060    ),
2061  }
2062
2063  error! {
2064    name:  tokenize_space_then_tab,
2065    input: "a:
2066 0
2067 1
2068\t2
2069",
2070    offset: 9,
2071    line:   3,
2072    column: 0,
2073    width:  1,
2074    kind:   InconsistentLeadingWhitespace{expected: " ", found: "\t"},
2075  }
2076
2077  error! {
2078    name:  tokenize_tabs_then_tab_space,
2079    input: "a:
2080\t\t0
2081\t\t 1
2082\t  2
2083",
2084    offset: 12,
2085    line:   3,
2086    column: 0,
2087    width:  3,
2088    kind:   InconsistentLeadingWhitespace{expected: "\t\t", found: "\t  "},
2089  }
2090
2091  error! {
2092    name:   tokenize_unknown,
2093    input:  "%",
2094    offset: 0,
2095    line:   0,
2096    column: 0,
2097    width:  1,
2098    kind:   UnknownStartOfToken,
2099  }
2100
2101  error! {
2102    name:   unterminated_string_with_escapes,
2103    input:  r#"a = "\n\t\r\"\\"#,
2104    offset: 4,
2105    line:   0,
2106    column: 4,
2107    width:  1,
2108    kind:   UnterminatedString,
2109  }
2110
2111  error! {
2112    name:   unterminated_raw_string,
2113    input:  "r a='asdf",
2114    offset: 4,
2115    line:   0,
2116    column: 4,
2117    width:  1,
2118    kind:   UnterminatedString,
2119  }
2120
2121  error! {
2122    name:   unterminated_interpolation,
2123    input:  "foo:\n echo {{
2124  ",
2125    offset: 11,
2126    line:   1,
2127    column: 6,
2128    width:  2,
2129    kind:   UnterminatedInterpolation,
2130  }
2131
2132  error! {
2133    name:   unterminated_backtick,
2134    input:  "`echo",
2135    offset: 0,
2136    line:   0,
2137    column: 0,
2138    width:  1,
2139    kind:   UnterminatedBacktick,
2140  }
2141
2142  error! {
2143    name:   unpaired_carriage_return,
2144    input:  "foo\rbar",
2145    offset: 3,
2146    line:   0,
2147    column: 3,
2148    width:  1,
2149    kind:   UnpairedCarriageReturn,
2150  }
2151
2152  error! {
2153    name:   invalid_name_start_dash,
2154    input:  "-foo",
2155    offset: 0,
2156    line:   0,
2157    column: 0,
2158    width:  1,
2159    kind:   UnknownStartOfToken,
2160  }
2161
2162  error! {
2163    name:   invalid_name_start_digit,
2164    input:  "0foo",
2165    offset: 0,
2166    line:   0,
2167    column: 0,
2168    width:  1,
2169    kind:   UnknownStartOfToken,
2170  }
2171
2172  error! {
2173    name:   unterminated_string,
2174    input:  r#"a = ""#,
2175    offset: 4,
2176    line:   0,
2177    column: 4,
2178    width:  1,
2179    kind:   UnterminatedString,
2180  }
2181
2182  error! {
2183    name:   mixed_leading_whitespace_recipe,
2184    input:  "a:\n\t echo hello",
2185    offset: 3,
2186    line:   1,
2187    column: 0,
2188    width:  2,
2189    kind:   MixedLeadingWhitespace{whitespace: "\t "},
2190  }
2191
2192  error! {
2193    name:   mixed_leading_whitespace_normal,
2194    input:  "a\n\t echo hello",
2195    offset: 2,
2196    line:   1,
2197    column: 0,
2198    width:  2,
2199    kind:   MixedLeadingWhitespace{whitespace: "\t "},
2200  }
2201
2202  error! {
2203    name:   mixed_leading_whitespace_indent,
2204    input:  "a\n foo\n \tbar",
2205    offset: 7,
2206    line:   2,
2207    column: 0,
2208    width:  2,
2209    kind:   MixedLeadingWhitespace{whitespace: " \t"},
2210  }
2211
2212  error! {
2213    name:   bad_dedent,
2214    input:  "a\n foo\n   bar\n  baz",
2215    offset: 14,
2216    line:   3,
2217    column: 0,
2218    width:  2,
2219    kind:   InconsistentLeadingWhitespace{expected: "   ", found: "  "},
2220  }
2221
2222  error! {
2223    name:   unclosed_interpolation_delimiter,
2224    input:  "a:\n echo {{ foo",
2225    offset: 9,
2226    line:   1,
2227    column: 6,
2228    width:  2,
2229    kind:   UnterminatedInterpolation,
2230  }
2231
2232  error! {
2233    name:   unexpected_character_after_at,
2234    input:  "@%",
2235    offset: 1,
2236    line:   0,
2237    column: 1,
2238    width:  1,
2239    kind:   UnknownStartOfToken,
2240  }
2241
2242  error! {
2243    name:   mismatched_closing_brace,
2244    input:  "(]",
2245    offset: 1,
2246    line:   0,
2247    column: 1,
2248    width:  0,
2249    kind:   MismatchedClosingDelimiter {
2250      open:      Delimiter::Paren,
2251      close:     Delimiter::Bracket,
2252      open_line: 0,
2253    },
2254  }
2255
2256  error! {
2257    name:   ampersand_eof,
2258    input:  "&",
2259    offset: 1,
2260    line:   0,
2261    column: 1,
2262    width:  0,
2263    kind:   UnexpectedEndOfToken {
2264      expected: '&',
2265    },
2266  }
2267  error! {
2268    name:   ampersand_unexpected,
2269    input:  "&%",
2270    offset: 1,
2271    line:   0,
2272    column: 1,
2273    width:  1,
2274    kind:   UnexpectedCharacter {
2275      expected: '&',
2276    },
2277  }
2278
2279  #[test]
2280  fn presume_error() {
2281    let compile_error = Lexer::new("justfile".as_ref(), "!")
2282      .presume('-')
2283      .unwrap_err();
2284    assert_matches!(
2285      compile_error.token,
2286      Token {
2287        offset: 0,
2288        line: 0,
2289        column: 0,
2290        length: 0,
2291        src: "!",
2292        kind: Unspecified,
2293        path: _,
2294      }
2295    );
2296    assert_matches!(&*compile_error.kind,
2297        Internal { ref message }
2298        if message == "Lexer presumed character `-`"
2299    );
2300
2301    assert_eq!(
2302      Error::Compile { compile_error }
2303        .color_display(Color::never())
2304        .to_string(),
2305      "error: Internal error, this may indicate a bug in just: Lexer presumed character `-`
2306consider filing an issue: https://github.com/casey/just/issues/new
2307 ——▶ justfile:1:1
2308  │
23091 │ !
2310  │ ^"
2311    );
2312  }
2313}