1use {super::*, CompileErrorKind::*, TokenKind::*};
2
3pub struct Lexer<'src> {
12 chars: Chars<'src>,
14 indentation: Vec<&'src str>,
16 interpolation_stack: Vec<Token<'src>>,
18 next: Option<char>,
20 open_delimiters: Vec<(Delimiter, usize)>,
22 path: &'src Path,
24 recipe_body: bool,
26 recipe_body_pending: bool,
28 src: &'src str,
30 tokens: Vec<Token<'src>>,
32 token_end: Position,
34 token_start: Position,
36}
37
38impl<'src> Lexer<'src> {
39 pub fn lex(path: &'src Path, src: &'src str) -> CompileResult<'src, Vec<Token<'src>>> {
41 Self::new(path, src).tokenize()
42 }
43
44 #[cfg(test)]
45 pub fn test_lex(src: &'src str) -> CompileResult<'src, Vec<Token<'src>>> {
46 Self::new("justfile".as_ref(), src).tokenize()
47 }
48
49 fn new(path: &'src Path, src: &'src str) -> Self {
51 let mut chars = src.chars();
52 let next = chars.next();
53
54 let start = Position {
55 offset: 0,
56 column: 0,
57 line: 0,
58 };
59
60 Self {
61 indentation: vec![""],
62 tokens: Vec::new(),
63 token_start: start,
64 token_end: start,
65 recipe_body_pending: false,
66 recipe_body: false,
67 interpolation_stack: Vec::new(),
68 open_delimiters: Vec::new(),
69 chars,
70 next,
71 src,
72 path,
73 }
74 }
75
76 fn advance(&mut self) -> CompileResult<'src> {
79 match self.next {
80 Some(c) => {
81 let len_utf8 = c.len_utf8();
82
83 self.token_end.offset += len_utf8;
84 self.token_end.column += len_utf8;
85
86 if c == '\n' {
87 self.token_end.column = 0;
88 self.token_end.line += 1;
89 }
90
91 self.next = self.chars.next();
92
93 Ok(())
94 }
95 None => Err(self.internal_error("Lexer advanced past end of text")),
96 }
97 }
98
99 fn skip(&mut self, n: usize) -> CompileResult<'src> {
101 for _ in 0..n {
102 self.advance()?;
103 }
104
105 Ok(())
106 }
107
108 fn lexeme(&self) -> &'src str {
110 &self.src[self.token_start.offset..self.token_end.offset]
111 }
112
113 fn current_token_length(&self) -> usize {
115 self.token_end.offset - self.token_start.offset
116 }
117
118 fn accepted(&mut self, c: char) -> CompileResult<'src, bool> {
119 if self.next_is(c) {
120 self.advance()?;
121 Ok(true)
122 } else {
123 Ok(false)
124 }
125 }
126
127 fn presume(&mut self, c: char) -> CompileResult<'src> {
128 if !self.next_is(c) {
129 return Err(self.internal_error(format!("Lexer presumed character `{c}`")));
130 }
131
132 self.advance()?;
133
134 Ok(())
135 }
136
137 fn presume_str(&mut self, s: &str) -> CompileResult<'src> {
138 for c in s.chars() {
139 self.presume(c)?;
140 }
141
142 Ok(())
143 }
144
145 fn next_is(&self, c: char) -> bool {
147 self.next == Some(c)
148 }
149
150 fn next_is_whitespace(&self) -> bool {
152 self.next_is(' ') || self.next_is('\t')
153 }
154
155 fn rest(&self) -> &'src str {
157 &self.src[self.token_end.offset..]
158 }
159
160 fn rest_starts_with(&self, prefix: &str) -> bool {
162 self.rest().starts_with(prefix)
163 }
164
165 fn at_eol(&self) -> bool {
167 self.next_is('\n') || self.rest_starts_with("\r\n")
168 }
169
170 fn at_eof(&self) -> bool {
172 self.rest().is_empty()
173 }
174
175 fn at_eol_or_eof(&self) -> bool {
177 self.at_eol() || self.at_eof()
178 }
179
180 fn indentation(&self) -> &'src str {
182 self.indentation.last().unwrap()
183 }
184
185 fn indented(&self) -> bool {
187 !self.indentation().is_empty()
188 }
189
190 fn token(&mut self, kind: TokenKind) {
193 self.tokens.push(Token {
194 offset: self.token_start.offset,
195 column: self.token_start.column,
196 line: self.token_start.line,
197 src: self.src,
198 length: self.token_end.offset - self.token_start.offset,
199 kind,
200 path: self.path,
201 });
202
203 self.token_start = self.token_end;
205 }
206
207 fn internal_error(&self, message: impl Into<String>) -> CompileError<'src> {
209 let token = Token {
211 src: self.src,
212 offset: self.token_end.offset,
213 line: self.token_end.line,
214 column: self.token_end.column,
215 length: 0,
216 kind: Unspecified,
217 path: self.path,
218 };
219 CompileError::new(
220 token,
221 Internal {
222 message: message.into(),
223 },
224 )
225 }
226
227 fn error(&self, kind: CompileErrorKind<'src>) -> CompileError<'src> {
229 let length = match kind {
233 UnterminatedString | UnterminatedBacktick => {
234 let Some(kind) = StringKind::from_token_start(self.lexeme()) else {
235 return self.internal_error("Lexer::error: expected string or backtick token start");
236 };
237 kind.delimiter().len()
238 }
239 _ => self.lexeme().len(),
241 };
242
243 let token = Token {
244 kind: Unspecified,
245 src: self.src,
246 offset: self.token_start.offset,
247 line: self.token_start.line,
248 column: self.token_start.column,
249 length,
250 path: self.path,
251 };
252
253 CompileError::new(token, kind)
254 }
255
256 fn unterminated_interpolation_error(interpolation_start: Token<'src>) -> CompileError<'src> {
257 CompileError::new(interpolation_start, UnterminatedInterpolation)
258 }
259
260 pub fn is_identifier(text: &str) -> bool {
262 if !text.chars().next().map_or(false, Self::is_identifier_start) {
263 return false;
264 }
265
266 for c in text.chars().skip(1) {
267 if !Self::is_identifier_continue(c) {
268 return false;
269 }
270 }
271
272 true
273 }
274
275 pub fn is_identifier_start(c: char) -> bool {
277 matches!(c, 'a'..='z' | 'A'..='Z' | '_')
278 }
279
280 pub fn is_identifier_continue(c: char) -> bool {
282 Self::is_identifier_start(c) || matches!(c, '0'..='9' | '-')
283 }
284
285 fn tokenize(mut self) -> CompileResult<'src, Vec<Token<'src>>> {
287 loop {
288 if self.token_start.column == 0 {
289 self.lex_line_start()?;
290 }
291
292 match self.next {
293 Some(first) => {
294 if let Some(&interpolation_start) = self.interpolation_stack.last() {
295 self.lex_interpolation(interpolation_start, first)?;
296 } else if self.recipe_body {
297 self.lex_body()?;
298 } else {
299 self.lex_normal(first)?;
300 };
301 }
302 None => break,
303 }
304 }
305
306 if let Some(&interpolation_start) = self.interpolation_stack.last() {
307 return Err(Self::unterminated_interpolation_error(interpolation_start));
308 }
309
310 while self.indented() {
311 self.lex_dedent();
312 }
313
314 self.token(Eof);
315
316 assert_eq!(self.token_start.offset, self.token_end.offset);
317 assert_eq!(self.token_start.offset, self.src.len());
318 assert_eq!(self.indentation.len(), 1);
319
320 Ok(self.tokens)
321 }
322
323 fn lex_line_start(&mut self) -> CompileResult<'src> {
325 enum Indentation<'src> {
326 Blank,
328 Continue,
330 Decrease,
332 Inconsistent,
334 Increase,
336 Mixed { whitespace: &'src str },
338 }
339
340 use Indentation::*;
341
342 let nonblank_index = self
343 .rest()
344 .char_indices()
345 .skip_while(|&(_, c)| c == ' ' || c == '\t')
346 .map(|(i, _)| i)
347 .next()
348 .unwrap_or_else(|| self.rest().len());
349
350 let rest = &self.rest()[nonblank_index..];
351
352 let whitespace = &self.rest()[..nonblank_index];
353
354 let body_whitespace = &whitespace[..whitespace
355 .char_indices()
356 .take(self.indentation().chars().count())
357 .map(|(i, _c)| i)
358 .next()
359 .unwrap_or(0)];
360
361 let spaces = whitespace.chars().any(|c| c == ' ');
362 let tabs = whitespace.chars().any(|c| c == '\t');
363
364 let body_spaces = body_whitespace.chars().any(|c| c == ' ');
365 let body_tabs = body_whitespace.chars().any(|c| c == '\t');
366
367 #[allow(clippy::if_same_then_else)]
368 let indentation = if rest.starts_with('\n') || rest.starts_with("\r\n") || rest.is_empty() {
369 Blank
370 } else if whitespace == self.indentation() {
371 Continue
372 } else if self.indentation.contains(&whitespace) {
373 Decrease
374 } else if self.recipe_body && whitespace.starts_with(self.indentation()) {
375 Continue
376 } else if self.recipe_body && body_spaces && body_tabs {
377 Mixed {
378 whitespace: body_whitespace,
379 }
380 } else if !self.recipe_body && spaces && tabs {
381 Mixed { whitespace }
382 } else if whitespace.len() < self.indentation().len() {
383 Inconsistent
384 } else if self.recipe_body
385 && body_whitespace.len() >= self.indentation().len()
386 && !body_whitespace.starts_with(self.indentation())
387 {
388 Inconsistent
389 } else if whitespace.len() >= self.indentation().len()
390 && !whitespace.starts_with(self.indentation())
391 {
392 Inconsistent
393 } else {
394 Increase
395 };
396
397 match indentation {
398 Blank => {
399 if !whitespace.is_empty() {
400 while self.next_is_whitespace() {
401 self.advance()?;
402 }
403
404 self.token(Whitespace);
405 };
406
407 Ok(())
408 }
409 Continue => {
410 if !self.indentation().is_empty() {
411 for _ in self.indentation().chars() {
412 self.advance()?;
413 }
414
415 self.token(Whitespace);
416 }
417
418 Ok(())
419 }
420 Decrease => {
421 while self.indentation() != whitespace {
422 self.lex_dedent();
423 }
424
425 if !whitespace.is_empty() {
426 while self.next_is_whitespace() {
427 self.advance()?;
428 }
429
430 self.token(Whitespace);
431 }
432
433 Ok(())
434 }
435 Mixed { whitespace } => {
436 for _ in whitespace.chars() {
437 self.advance()?;
438 }
439
440 Err(self.error(MixedLeadingWhitespace { whitespace }))
441 }
442 Inconsistent => {
443 for _ in whitespace.chars() {
444 self.advance()?;
445 }
446
447 Err(self.error(InconsistentLeadingWhitespace {
448 expected: self.indentation(),
449 found: whitespace,
450 }))
451 }
452 Increase => {
453 while self.next_is_whitespace() {
454 self.advance()?;
455 }
456
457 if self.open_delimiters() {
458 self.token(Whitespace);
459 } else {
460 let indentation = self.lexeme();
461 self.indentation.push(indentation);
462 self.token(Indent);
463 if self.recipe_body_pending {
464 self.recipe_body = true;
465 }
466 }
467
468 Ok(())
469 }
470 }
471 }
472
473 fn lex_normal(&mut self, start: char) -> CompileResult<'src> {
475 match start {
476 ' ' | '\t' => self.lex_whitespace(),
477 '!' if self.rest().starts_with("!include") => Err(self.error(Include)),
478 '!' => self.lex_digraph('!', '=', BangEquals),
479 '#' => self.lex_comment(),
480 '$' => self.lex_single(Dollar),
481 '&' => self.lex_digraph('&', '&', AmpersandAmpersand),
482 '(' => self.lex_delimiter(ParenL),
483 ')' => self.lex_delimiter(ParenR),
484 '*' => self.lex_single(Asterisk),
485 '+' => self.lex_single(Plus),
486 ',' => self.lex_single(Comma),
487 '/' => self.lex_single(Slash),
488 ':' => self.lex_colon(),
489 '=' => self.lex_choices('=', &[('=', EqualsEquals), ('~', EqualsTilde)], Equals),
490 '?' => self.lex_single(QuestionMark),
491 '@' => self.lex_single(At),
492 '[' => self.lex_delimiter(BracketL),
493 '\\' => self.lex_escape(),
494 '\n' | '\r' => self.lex_eol(),
495 '\u{feff}' => self.lex_single(ByteOrderMark),
496 ']' => self.lex_delimiter(BracketR),
497 '`' | '"' | '\'' => self.lex_string(),
498 '{' => self.lex_delimiter(BraceL),
499 '|' => self.lex_digraph('|', '|', BarBar),
500 '}' => self.lex_delimiter(BraceR),
501 _ if Self::is_identifier_start(start) => self.lex_identifier(),
502 _ => {
503 self.advance()?;
504 Err(self.error(UnknownStartOfToken))
505 }
506 }
507 }
508
509 fn lex_interpolation(
511 &mut self,
512 interpolation_start: Token<'src>,
513 start: char,
514 ) -> CompileResult<'src> {
515 if self.rest_starts_with("}}") {
516 if self.interpolation_stack.pop().is_none() {
518 self.advance()?;
519 self.advance()?;
520 return Err(self.internal_error(
521 "Lexer::lex_interpolation found `}}` but was called with empty interpolation stack.",
522 ));
523 }
524 self.lex_double(InterpolationEnd)
526 } else if self.at_eol_or_eof() {
527 Err(Self::unterminated_interpolation_error(interpolation_start))
530 } else {
531 self.lex_normal(start)
533 }
534 }
535
536 fn lex_body(&mut self) -> CompileResult<'src> {
538 enum Terminator {
539 Newline,
540 NewlineCarriageReturn,
541 Interpolation,
542 EndOfFile,
543 }
544
545 use Terminator::*;
546
547 let terminator = loop {
548 if self.rest_starts_with("{{{{") {
549 self.skip(4)?;
550 continue;
551 }
552
553 if self.rest_starts_with("\n") {
554 break Newline;
555 }
556
557 if self.rest_starts_with("\r\n") {
558 break NewlineCarriageReturn;
559 }
560
561 if self.rest_starts_with("{{") {
562 break Interpolation;
563 }
564
565 if self.at_eof() {
566 break EndOfFile;
567 }
568
569 self.advance()?;
570 };
571
572 if self.current_token_length() > 0 {
574 self.token(Text);
575 }
576
577 match terminator {
578 Newline => self.lex_single(Eol),
579 NewlineCarriageReturn => self.lex_double(Eol),
580 Interpolation => {
581 self.lex_double(InterpolationStart)?;
582 self
583 .interpolation_stack
584 .push(self.tokens[self.tokens.len() - 1]);
585 Ok(())
586 }
587 EndOfFile => Ok(()),
588 }
589 }
590
591 fn lex_dedent(&mut self) {
592 assert_eq!(self.current_token_length(), 0);
593 self.token(Dedent);
594 self.indentation.pop();
595 self.recipe_body_pending = false;
596 self.recipe_body = false;
597 }
598
599 fn lex_single(&mut self, kind: TokenKind) -> CompileResult<'src> {
601 self.advance()?;
602 self.token(kind);
603 Ok(())
604 }
605
606 fn lex_double(&mut self, kind: TokenKind) -> CompileResult<'src> {
608 self.advance()?;
609 self.advance()?;
610 self.token(kind);
611 Ok(())
612 }
613
614 fn lex_choices(
618 &mut self,
619 first: char,
620 choices: &[(char, TokenKind)],
621 otherwise: TokenKind,
622 ) -> CompileResult<'src> {
623 self.presume(first)?;
624
625 for (second, then) in choices {
626 if self.accepted(*second)? {
627 self.token(*then);
628 return Ok(());
629 }
630 }
631
632 self.token(otherwise);
633
634 Ok(())
635 }
636
637 fn lex_delimiter(&mut self, kind: TokenKind) -> CompileResult<'src> {
639 use Delimiter::*;
640
641 match kind {
642 BraceL => self.open_delimiter(Brace),
643 BraceR => self.close_delimiter(Brace)?,
644 BracketL => self.open_delimiter(Bracket),
645 BracketR => self.close_delimiter(Bracket)?,
646 ParenL => self.open_delimiter(Paren),
647 ParenR => self.close_delimiter(Paren)?,
648 _ => {
649 return Err(self.internal_error(format!(
650 "Lexer::lex_delimiter called with non-delimiter token: `{kind}`",
651 )))
652 }
653 }
654
655 self.lex_single(kind)
657 }
658
659 fn open_delimiter(&mut self, delimiter: Delimiter) {
661 self
662 .open_delimiters
663 .push((delimiter, self.token_start.line));
664 }
665
666 fn close_delimiter(&mut self, close: Delimiter) -> CompileResult<'src> {
668 match self.open_delimiters.pop() {
669 Some((open, _)) if open == close => Ok(()),
670 Some((open, open_line)) => Err(self.error(MismatchedClosingDelimiter {
671 open,
672 close,
673 open_line,
674 })),
675 None => Err(self.error(UnexpectedClosingDelimiter { close })),
676 }
677 }
678
679 fn open_delimiters(&self) -> bool {
681 !self.open_delimiters.is_empty()
682 }
683
684 fn lex_digraph(&mut self, left: char, right: char, token: TokenKind) -> CompileResult<'src> {
686 self.presume(left)?;
687
688 if self.accepted(right)? {
689 self.token(token);
690 Ok(())
691 } else {
692 self.token(Unspecified);
694
695 if self.at_eof() {
696 return Err(self.error(UnexpectedEndOfToken { expected: right }));
697 }
698
699 self.advance()?;
701
702 Err(self.error(UnexpectedCharacter { expected: right }))
704 }
705 }
706
707 fn lex_colon(&mut self) -> CompileResult<'src> {
709 self.presume(':')?;
710
711 if self.accepted('=')? {
712 self.token(ColonEquals);
713 } else {
714 self.token(Colon);
715 self.recipe_body_pending = true;
716 }
717
718 Ok(())
719 }
720
721 fn lex_escape(&mut self) -> CompileResult<'src> {
723 self.presume('\\')?;
724
725 if self.accepted('\n')? {
727 while self.next_is_whitespace() {
728 self.advance()?;
729 }
730 self.token(Whitespace);
731 } else if self.accepted('\r')? {
732 if !self.accepted('\n')? {
733 return Err(self.error(UnpairedCarriageReturn));
734 }
735 while self.next_is_whitespace() {
736 self.advance()?;
737 }
738 self.token(Whitespace);
739 } else if let Some(character) = self.next {
740 return Err(self.error(InvalidEscapeSequence { character }));
741 }
742
743 Ok(())
744 }
745
746 fn lex_eol(&mut self) -> CompileResult<'src> {
748 if self.accepted('\r')? {
749 if !self.accepted('\n')? {
750 return Err(self.error(UnpairedCarriageReturn));
751 }
752 } else {
753 self.presume('\n')?;
754 }
755
756 if self.open_delimiters() {
759 self.token(Whitespace);
760 } else {
761 self.token(Eol);
762 }
763
764 Ok(())
765 }
766
767 fn lex_identifier(&mut self) -> CompileResult<'src> {
769 self.advance()?;
770
771 while let Some(c) = self.next {
772 if !Self::is_identifier_continue(c) {
773 break;
774 }
775
776 self.advance()?;
777 }
778
779 self.token(Identifier);
780
781 Ok(())
782 }
783
784 fn lex_comment(&mut self) -> CompileResult<'src> {
786 self.presume('#')?;
787
788 while !self.at_eol_or_eof() {
789 self.advance()?;
790 }
791
792 self.token(Comment);
793
794 Ok(())
795 }
796
797 fn lex_whitespace(&mut self) -> CompileResult<'src> {
799 while self.next_is_whitespace() {
800 self.advance()?;
801 }
802
803 self.token(Whitespace);
804
805 Ok(())
806 }
807
808 fn lex_string(&mut self) -> CompileResult<'src> {
814 let Some(kind) = StringKind::from_token_start(self.rest()) else {
815 self.advance()?;
816 return Err(self.internal_error("Lexer::lex_string: invalid string start"));
817 };
818
819 self.presume_str(kind.delimiter())?;
820
821 let mut escape = false;
822
823 loop {
824 if self.next.is_none() {
825 return Err(self.error(kind.unterminated_error_kind()));
826 } else if kind.processes_escape_sequences() && self.next_is('\\') && !escape {
827 escape = true;
828 } else if self.rest_starts_with(kind.delimiter()) && !escape {
829 break;
830 } else {
831 escape = false;
832 }
833
834 self.advance()?;
835 }
836
837 self.presume_str(kind.delimiter())?;
838 self.token(kind.token_kind());
839
840 Ok(())
841 }
842}
843
844#[cfg(test)]
845mod tests {
846 use super::*;
847
848 use pretty_assertions::assert_eq;
849
850 macro_rules! test {
851 {
852 name: $name:ident,
853 text: $text:expr,
854 tokens: ($($kind:ident $(: $lexeme:literal)?),* $(,)?)$(,)?
855 } => {
856 #[test]
857 fn $name() {
858 let kinds: &[TokenKind] = &[$($kind,)* Eof];
859
860 let lexemes: &[&str] = &[$(lexeme!($kind $(, $lexeme)?),)* ""];
861
862 test($text, true, kinds, lexemes);
863 }
864 };
865 {
866 name: $name:ident,
867 text: $text:expr,
868 tokens: ($($kind:ident $(: $lexeme:literal)?),* $(,)?)$(,)?
869 unindent: $unindent:expr,
870 } => {
871 #[test]
872 fn $name() {
873 let kinds: &[TokenKind] = &[$($kind,)* Eof];
874
875 let lexemes: &[&str] = &[$(lexeme!($kind $(, $lexeme)?),)* ""];
876
877 test($text, $unindent, kinds, lexemes);
878 }
879 }
880 }
881
882 macro_rules! lexeme {
883 {
884 $kind:ident, $lexeme:literal
885 } => {
886 $lexeme
887 };
888 {
889 $kind:ident
890 } => {
891 default_lexeme($kind)
892 }
893 }
894
895 fn test(text: &str, unindent_text: bool, want_kinds: &[TokenKind], want_lexemes: &[&str]) {
896 let text = if unindent_text {
897 unindent(text)
898 } else {
899 text.to_owned()
900 };
901
902 let have = Lexer::test_lex(&text).unwrap();
903
904 let have_kinds = have
905 .iter()
906 .map(|token| token.kind)
907 .collect::<Vec<TokenKind>>();
908
909 let have_lexemes = have.iter().map(Token::lexeme).collect::<Vec<&str>>();
910
911 assert_eq!(have_kinds, want_kinds, "Token kind mismatch");
912 assert_eq!(have_lexemes, want_lexemes, "Token lexeme mismatch");
913
914 let mut roundtrip = String::new();
915
916 for lexeme in have_lexemes {
917 roundtrip.push_str(lexeme);
918 }
919
920 assert_eq!(roundtrip, text, "Roundtrip mismatch");
921
922 let mut offset = 0;
923 let mut line = 0;
924 let mut column = 0;
925
926 for token in have {
927 assert_eq!(token.offset, offset);
928 assert_eq!(token.line, line);
929 assert_eq!(token.lexeme().len(), token.length);
930 assert_eq!(token.column, column);
931
932 for c in token.lexeme().chars() {
933 if c == '\n' {
934 line += 1;
935 column = 0;
936 } else {
937 column += c.len_utf8();
938 }
939 }
940
941 offset += token.length;
942 }
943 }
944
945 fn default_lexeme(kind: TokenKind) -> &'static str {
946 match kind {
947 AmpersandAmpersand => "&&",
949 Asterisk => "*",
950 At => "@",
951 BangEquals => "!=",
952 BarBar => "||",
953 BraceL => "{",
954 BraceR => "}",
955 BracketL => "[",
956 BracketR => "]",
957 ByteOrderMark => "\u{feff}",
958 Colon => ":",
959 ColonEquals => ":=",
960 Comma => ",",
961 Dollar => "$",
962 Eol => "\n",
963 Equals => "=",
964 EqualsEquals => "==",
965 EqualsTilde => "=~",
966 Indent => " ",
967 InterpolationEnd => "}}",
968 InterpolationStart => "{{",
969 ParenL => "(",
970 ParenR => ")",
971 Plus => "+",
972 QuestionMark => "?",
973 Slash => "/",
974 Whitespace => " ",
975
976 Dedent | Eof => "",
978
979 Text | StringToken | Backtick | Identifier | Comment | Unspecified => {
981 panic!("Token {kind:?} has no default lexeme")
982 }
983 }
984 }
985
986 macro_rules! error {
987 (
988 name: $name:ident,
989 input: $input:expr,
990 offset: $offset:expr,
991 line: $line:expr,
992 column: $column:expr,
993 width: $width:expr,
994 kind: $kind:expr,
995 ) => {
996 #[test]
997 fn $name() {
998 error($input, $offset, $line, $column, $width, $kind);
999 }
1000 };
1001 }
1002
1003 fn error(
1004 src: &str,
1005 offset: usize,
1006 line: usize,
1007 column: usize,
1008 length: usize,
1009 kind: CompileErrorKind,
1010 ) {
1011 match Lexer::test_lex(src) {
1012 Ok(_) => panic!("Lexing succeeded but expected"),
1013 Err(have) => {
1014 let want = CompileError {
1015 token: Token {
1016 kind: have.token.kind,
1017 src,
1018 offset,
1019 line,
1020 column,
1021 length,
1022 path: "justfile".as_ref(),
1023 },
1024 kind: kind.into(),
1025 };
1026 assert_eq!(have, want);
1027 }
1028 }
1029 }
1030
1031 test! {
1032 name: name_new,
1033 text: "foo",
1034 tokens: (Identifier:"foo"),
1035 }
1036
1037 test! {
1038 name: comment,
1039 text: "# hello",
1040 tokens: (Comment:"# hello"),
1041 }
1042
1043 test! {
1044 name: backtick,
1045 text: "`echo`",
1046 tokens: (Backtick:"`echo`"),
1047 }
1048
1049 test! {
1050 name: backtick_multi_line,
1051 text: "`echo\necho`",
1052 tokens: (Backtick:"`echo\necho`"),
1053 }
1054
1055 test! {
1056 name: raw_string,
1057 text: "'hello'",
1058 tokens: (StringToken:"'hello'"),
1059 }
1060
1061 test! {
1062 name: raw_string_multi_line,
1063 text: "'hello\ngoodbye'",
1064 tokens: (StringToken:"'hello\ngoodbye'"),
1065 }
1066
1067 test! {
1068 name: cooked_string,
1069 text: "\"hello\"",
1070 tokens: (StringToken:"\"hello\""),
1071 }
1072
1073 test! {
1074 name: cooked_string_multi_line,
1075 text: "\"hello\ngoodbye\"",
1076 tokens: (StringToken:"\"hello\ngoodbye\""),
1077 }
1078
1079 test! {
1080 name: cooked_multiline_string,
1081 text: "\"\"\"hello\ngoodbye\"\"\"",
1082 tokens: (StringToken:"\"\"\"hello\ngoodbye\"\"\""),
1083 }
1084
1085 test! {
1086 name: ampersand_ampersand,
1087 text: "&&",
1088 tokens: (AmpersandAmpersand),
1089 }
1090
1091 test! {
1092 name: equals,
1093 text: "=",
1094 tokens: (Equals),
1095 }
1096
1097 test! {
1098 name: equals_equals,
1099 text: "==",
1100 tokens: (EqualsEquals),
1101 }
1102
1103 test! {
1104 name: bang_equals,
1105 text: "!=",
1106 tokens: (BangEquals),
1107 }
1108
1109 test! {
1110 name: brace_l,
1111 text: "{",
1112 tokens: (BraceL),
1113 }
1114
1115 test! {
1116 name: brace_r,
1117 text: "{}",
1118 tokens: (BraceL, BraceR),
1119 }
1120
1121 test! {
1122 name: brace_lll,
1123 text: "{{{",
1124 tokens: (BraceL, BraceL, BraceL),
1125 }
1126
1127 test! {
1128 name: brace_rrr,
1129 text: "{{{}}}",
1130 tokens: (BraceL, BraceL, BraceL, BraceR, BraceR, BraceR),
1131 }
1132
1133 test! {
1134 name: dollar,
1135 text: "$",
1136 tokens: (Dollar),
1137 }
1138
1139 test! {
1140 name: export_concatenation,
1141 text: "export foo = 'foo' + 'bar'",
1142 tokens: (
1143 Identifier:"export",
1144 Whitespace,
1145 Identifier:"foo",
1146 Whitespace,
1147 Equals,
1148 Whitespace,
1149 StringToken:"'foo'",
1150 Whitespace,
1151 Plus,
1152 Whitespace,
1153 StringToken:"'bar'",
1154 )
1155 }
1156
1157 test! {
1158 name: export_complex,
1159 text: "export foo = ('foo' + 'bar') + `baz`",
1160 tokens: (
1161 Identifier:"export",
1162 Whitespace,
1163 Identifier:"foo",
1164 Whitespace,
1165 Equals,
1166 Whitespace,
1167 ParenL,
1168 StringToken:"'foo'",
1169 Whitespace,
1170 Plus,
1171 Whitespace,
1172 StringToken:"'bar'",
1173 ParenR,
1174 Whitespace,
1175 Plus,
1176 Whitespace,
1177 Backtick:"`baz`",
1178 ),
1179 }
1180
1181 test! {
1182 name: eol_linefeed,
1183 text: "\n",
1184 tokens: (Eol),
1185 unindent: false,
1186 }
1187
1188 test! {
1189 name: eol_carriage_return_linefeed,
1190 text: "\r\n",
1191 tokens: (Eol:"\r\n"),
1192 unindent: false,
1193 }
1194
1195 test! {
1196 name: indented_line,
1197 text: "foo:\n a",
1198 tokens: (Identifier:"foo", Colon, Eol, Indent:" ", Text:"a", Dedent),
1199 }
1200
1201 test! {
1202 name: indented_normal,
1203 text: "
1204 a
1205 b
1206 c
1207 ",
1208 tokens: (
1209 Identifier:"a",
1210 Eol,
1211 Indent:" ",
1212 Identifier:"b",
1213 Eol,
1214 Whitespace:" ",
1215 Identifier:"c",
1216 Eol,
1217 Dedent,
1218 ),
1219 }
1220
1221 test! {
1222 name: indented_normal_nonempty_blank,
1223 text: "a\n b\n\t\t\n c\n",
1224 tokens: (
1225 Identifier:"a",
1226 Eol,
1227 Indent:" ",
1228 Identifier:"b",
1229 Eol,
1230 Whitespace:"\t\t",
1231 Eol,
1232 Whitespace:" ",
1233 Identifier:"c",
1234 Eol,
1235 Dedent,
1236 ),
1237 unindent: false,
1238 }
1239
1240 test! {
1241 name: indented_normal_multiple,
1242 text: "
1243 a
1244 b
1245 c
1246 ",
1247 tokens: (
1248 Identifier:"a",
1249 Eol,
1250 Indent:" ",
1251 Identifier:"b",
1252 Eol,
1253 Indent:" ",
1254 Identifier:"c",
1255 Eol,
1256 Dedent,
1257 Dedent,
1258 ),
1259 }
1260
1261 test! {
1262 name: indent_indent_dedent_indent,
1263 text: "
1264 a
1265 b
1266 c
1267 d
1268 e
1269 ",
1270 tokens: (
1271 Identifier:"a",
1272 Eol,
1273 Indent:" ",
1274 Identifier:"b",
1275 Eol,
1276 Indent:" ",
1277 Identifier:"c",
1278 Eol,
1279 Dedent,
1280 Whitespace:" ",
1281 Identifier:"d",
1282 Eol,
1283 Indent:" ",
1284 Identifier:"e",
1285 Eol,
1286 Dedent,
1287 Dedent,
1288 ),
1289 }
1290
1291 test! {
1292 name: indent_recipe_dedent_indent,
1293 text: "
1294 a
1295 b:
1296 c
1297 d
1298 e
1299 ",
1300 tokens: (
1301 Identifier:"a",
1302 Eol,
1303 Indent:" ",
1304 Identifier:"b",
1305 Colon,
1306 Eol,
1307 Indent:" ",
1308 Text:"c",
1309 Eol,
1310 Dedent,
1311 Whitespace:" ",
1312 Identifier:"d",
1313 Eol,
1314 Indent:" ",
1315 Identifier:"e",
1316 Eol,
1317 Dedent,
1318 Dedent,
1319 ),
1320 }
1321
1322 test! {
1323 name: indented_block,
1324 text: "
1325 foo:
1326 a
1327 b
1328 c
1329 ",
1330 tokens: (
1331 Identifier:"foo",
1332 Colon,
1333 Eol,
1334 Indent,
1335 Text:"a",
1336 Eol,
1337 Whitespace:" ",
1338 Text:"b",
1339 Eol,
1340 Whitespace:" ",
1341 Text:"c",
1342 Eol,
1343 Dedent,
1344 )
1345 }
1346
1347 test! {
1348 name: brace_escape,
1349 text: "
1350 foo:
1351 {{{{
1352 ",
1353 tokens: (
1354 Identifier:"foo",
1355 Colon,
1356 Eol,
1357 Indent,
1358 Text:"{{{{",
1359 Eol,
1360 Dedent,
1361 )
1362 }
1363
1364 test! {
1365 name: indented_block_followed_by_item,
1366 text: "
1367 foo:
1368 a
1369 b:
1370 ",
1371 tokens: (
1372 Identifier:"foo",
1373 Colon,
1374 Eol,
1375 Indent,
1376 Text:"a",
1377 Eol,
1378 Dedent,
1379 Identifier:"b",
1380 Colon,
1381 Eol,
1382 )
1383 }
1384
1385 test! {
1386 name: indented_block_followed_by_blank,
1387 text: "
1388 foo:
1389 a
1390
1391 b:
1392 ",
1393 tokens: (
1394 Identifier:"foo",
1395 Colon,
1396 Eol,
1397 Indent:" ",
1398 Text:"a",
1399 Eol,
1400 Eol,
1401 Dedent,
1402 Identifier:"b",
1403 Colon,
1404 Eol,
1405 ),
1406 }
1407
1408 test! {
1409 name: indented_line_containing_unpaired_carriage_return,
1410 text: "foo:\n \r \n",
1411 tokens: (
1412 Identifier:"foo",
1413 Colon,
1414 Eol,
1415 Indent:" ",
1416 Text:"\r ",
1417 Eol,
1418 Dedent,
1419 ),
1420 unindent: false,
1421 }
1422
1423 test! {
1424 name: indented_blocks,
1425 text: "
1426 b: a
1427 @mv a b
1428
1429 a:
1430 @touch F
1431 @touch a
1432
1433 d: c
1434 @rm c
1435
1436 c: b
1437 @mv b c
1438 ",
1439 tokens: (
1440 Identifier:"b",
1441 Colon,
1442 Whitespace,
1443 Identifier:"a",
1444 Eol,
1445 Indent,
1446 Text:"@mv a b",
1447 Eol,
1448 Eol,
1449 Dedent,
1450 Identifier:"a",
1451 Colon,
1452 Eol,
1453 Indent,
1454 Text:"@touch F",
1455 Eol,
1456 Whitespace:" ",
1457 Text:"@touch a",
1458 Eol,
1459 Eol,
1460 Dedent,
1461 Identifier:"d",
1462 Colon,
1463 Whitespace,
1464 Identifier:"c",
1465 Eol,
1466 Indent,
1467 Text:"@rm c",
1468 Eol,
1469 Eol,
1470 Dedent,
1471 Identifier:"c",
1472 Colon,
1473 Whitespace,
1474 Identifier:"b",
1475 Eol,
1476 Indent,
1477 Text:"@mv b c",
1478 Eol,
1479 Dedent
1480 ),
1481 }
1482
1483 test! {
1484 name: interpolation_empty,
1485 text: "hello:\n echo {{}}",
1486 tokens: (
1487 Identifier:"hello",
1488 Colon,
1489 Eol,
1490 Indent:" ",
1491 Text:"echo ",
1492 InterpolationStart,
1493 InterpolationEnd,
1494 Dedent,
1495 ),
1496 }
1497
1498 test! {
1499 name: interpolation_expression,
1500 text: "hello:\n echo {{`echo hello` + `echo goodbye`}}",
1501 tokens: (
1502 Identifier:"hello",
1503 Colon,
1504 Eol,
1505 Indent:" ",
1506 Text:"echo ",
1507 InterpolationStart,
1508 Backtick:"`echo hello`",
1509 Whitespace,
1510 Plus,
1511 Whitespace,
1512 Backtick:"`echo goodbye`",
1513 InterpolationEnd,
1514 Dedent,
1515 ),
1516 }
1517
1518 test! {
1519 name: interpolation_raw_multiline_string,
1520 text: "hello:\n echo {{'\n'}}",
1521 tokens: (
1522 Identifier:"hello",
1523 Colon,
1524 Eol,
1525 Indent:" ",
1526 Text:"echo ",
1527 InterpolationStart,
1528 StringToken:"'\n'",
1529 InterpolationEnd,
1530 Dedent,
1531 ),
1532 }
1533
1534 test! {
1535 name: tokenize_names,
1536 text: "
1537 foo
1538 bar-bob
1539 b-bob_asdfAAAA
1540 test123
1541 ",
1542 tokens: (
1543 Identifier:"foo",
1544 Eol,
1545 Identifier:"bar-bob",
1546 Eol,
1547 Identifier:"b-bob_asdfAAAA",
1548 Eol,
1549 Identifier:"test123",
1550 Eol,
1551 ),
1552 }
1553
1554 test! {
1555 name: tokenize_indented_line,
1556 text: "foo:\n a",
1557 tokens: (
1558 Identifier:"foo",
1559 Colon,
1560 Eol,
1561 Indent:" ",
1562 Text:"a",
1563 Dedent,
1564 ),
1565 }
1566
1567 test! {
1568 name: tokenize_indented_block,
1569 text: "
1570 foo:
1571 a
1572 b
1573 c
1574 ",
1575 tokens: (
1576 Identifier:"foo",
1577 Colon,
1578 Eol,
1579 Indent,
1580 Text:"a",
1581 Eol,
1582 Whitespace:" ",
1583 Text:"b",
1584 Eol,
1585 Whitespace:" ",
1586 Text:"c",
1587 Eol,
1588 Dedent,
1589 ),
1590 }
1591
1592 test! {
1593 name: tokenize_strings,
1594 text: r#"a = "'a'" + '"b"' + "'c'" + '"d"'#echo hello"#,
1595 tokens: (
1596 Identifier:"a",
1597 Whitespace,
1598 Equals,
1599 Whitespace,
1600 StringToken:"\"'a'\"",
1601 Whitespace,
1602 Plus,
1603 Whitespace,
1604 StringToken:"'\"b\"'",
1605 Whitespace,
1606 Plus,
1607 Whitespace,
1608 StringToken:"\"'c'\"",
1609 Whitespace,
1610 Plus,
1611 Whitespace,
1612 StringToken:"'\"d\"'",
1613 Comment:"#echo hello",
1614 )
1615 }
1616
1617 test! {
1618 name: tokenize_recipe_interpolation_eol,
1619 text: "
1620 foo: # some comment
1621 {{hello}}
1622 ",
1623 tokens: (
1624 Identifier:"foo",
1625 Colon,
1626 Whitespace,
1627 Comment:"# some comment",
1628 Eol,
1629 Indent:" ",
1630 InterpolationStart,
1631 Identifier:"hello",
1632 InterpolationEnd,
1633 Eol,
1634 Dedent
1635 ),
1636 }
1637
1638 test! {
1639 name: tokenize_recipe_interpolation_eof,
1640 text: "foo: # more comments
1641 {{hello}}
1642# another comment
1643",
1644 tokens: (
1645 Identifier:"foo",
1646 Colon,
1647 Whitespace,
1648 Comment:"# more comments",
1649 Eol,
1650 Indent:" ",
1651 InterpolationStart,
1652 Identifier:"hello",
1653 InterpolationEnd,
1654 Eol,
1655 Dedent,
1656 Comment:"# another comment",
1657 Eol,
1658 ),
1659 }
1660
1661 test! {
1662 name: tokenize_recipe_complex_interpolation_expression,
1663 text: "foo: #lol\n {{a + b + \"z\" + blarg}}",
1664 tokens: (
1665 Identifier:"foo",
1666 Colon,
1667 Whitespace:" ",
1668 Comment:"#lol",
1669 Eol,
1670 Indent:" ",
1671 InterpolationStart,
1672 Identifier:"a",
1673 Whitespace,
1674 Plus,
1675 Whitespace,
1676 Identifier:"b",
1677 Whitespace,
1678 Plus,
1679 Whitespace,
1680 StringToken:"\"z\"",
1681 Whitespace,
1682 Plus,
1683 Whitespace,
1684 Identifier:"blarg",
1685 InterpolationEnd,
1686 Dedent,
1687 ),
1688 }
1689
1690 test! {
1691 name: tokenize_recipe_multiple_interpolations,
1692 text: "foo:,#ok\n {{a}}0{{b}}1{{c}}",
1693 tokens: (
1694 Identifier:"foo",
1695 Colon,
1696 Comma,
1697 Comment:"#ok",
1698 Eol,
1699 Indent:" ",
1700 InterpolationStart,
1701 Identifier:"a",
1702 InterpolationEnd,
1703 Text:"0",
1704 InterpolationStart,
1705 Identifier:"b",
1706 InterpolationEnd,
1707 Text:"1",
1708 InterpolationStart,
1709 Identifier:"c",
1710 InterpolationEnd,
1711 Dedent,
1712
1713 ),
1714 }
1715
1716 test! {
1717 name: tokenize_junk,
1718 text: "
1719 bob
1720
1721 hello blah blah blah : a b c #whatever
1722 ",
1723 tokens: (
1724 Identifier:"bob",
1725 Eol,
1726 Eol,
1727 Identifier:"hello",
1728 Whitespace,
1729 Identifier:"blah",
1730 Whitespace,
1731 Identifier:"blah",
1732 Whitespace,
1733 Identifier:"blah",
1734 Whitespace,
1735 Colon,
1736 Whitespace,
1737 Identifier:"a",
1738 Whitespace,
1739 Identifier:"b",
1740 Whitespace,
1741 Identifier:"c",
1742 Whitespace,
1743 Comment:"#whatever",
1744 Eol,
1745 )
1746 }
1747
1748 test! {
1749 name: tokenize_empty_lines,
1750 text: "
1751
1752 # this does something
1753 hello:
1754 asdf
1755 bsdf
1756
1757 csdf
1758
1759 dsdf # whatever
1760
1761 # yolo
1762 ",
1763 tokens: (
1764 Eol,
1765 Comment:"# this does something",
1766 Eol,
1767 Identifier:"hello",
1768 Colon,
1769 Eol,
1770 Indent,
1771 Text:"asdf",
1772 Eol,
1773 Whitespace:" ",
1774 Text:"bsdf",
1775 Eol,
1776 Eol,
1777 Whitespace:" ",
1778 Text:"csdf",
1779 Eol,
1780 Eol,
1781 Whitespace:" ",
1782 Text:"dsdf # whatever",
1783 Eol,
1784 Eol,
1785 Dedent,
1786 Comment:"# yolo",
1787 Eol,
1788 ),
1789 }
1790
1791 test! {
1792 name: tokenize_comment_before_variable,
1793 text: "
1794 #
1795 A='1'
1796 echo:
1797 echo {{A}}
1798 ",
1799 tokens: (
1800 Comment:"#",
1801 Eol,
1802 Identifier:"A",
1803 Equals,
1804 StringToken:"'1'",
1805 Eol,
1806 Identifier:"echo",
1807 Colon,
1808 Eol,
1809 Indent,
1810 Text:"echo ",
1811 InterpolationStart,
1812 Identifier:"A",
1813 InterpolationEnd,
1814 Eol,
1815 Dedent,
1816 ),
1817 }
1818
1819 test! {
1820 name: tokenize_interpolation_backticks,
1821 text: "hello:\n echo {{`echo hello` + `echo goodbye`}}",
1822 tokens: (
1823 Identifier:"hello",
1824 Colon,
1825 Eol,
1826 Indent:" ",
1827 Text:"echo ",
1828 InterpolationStart,
1829 Backtick:"`echo hello`",
1830 Whitespace,
1831 Plus,
1832 Whitespace,
1833 Backtick:"`echo goodbye`",
1834 InterpolationEnd,
1835 Dedent
1836 ),
1837 }
1838
1839 test! {
1840 name: tokenize_empty_interpolation,
1841 text: "hello:\n echo {{}}",
1842 tokens: (
1843 Identifier:"hello",
1844 Colon,
1845 Eol,
1846 Indent:" ",
1847 Text:"echo ",
1848 InterpolationStart,
1849 InterpolationEnd,
1850 Dedent,
1851 ),
1852 }
1853
1854 test! {
1855 name: tokenize_assignment_backticks,
1856 text: "a = `echo hello` + `echo goodbye`",
1857 tokens: (
1858 Identifier:"a",
1859 Whitespace,
1860 Equals,
1861 Whitespace,
1862 Backtick:"`echo hello`",
1863 Whitespace,
1864 Plus,
1865 Whitespace,
1866 Backtick:"`echo goodbye`",
1867 ),
1868 }
1869
1870 test! {
1871 name: tokenize_multiple,
1872 text: "
1873
1874 hello:
1875 a
1876 b
1877
1878 c
1879
1880 d
1881
1882 # hello
1883 bob:
1884 frank
1885 \t
1886 ",
1887 tokens: (
1888 Eol,
1889 Identifier:"hello",
1890 Colon,
1891 Eol,
1892 Indent,
1893 Text:"a",
1894 Eol,
1895 Whitespace:" ",
1896 Text:"b",
1897 Eol,
1898 Eol,
1899 Whitespace:" ",
1900 Text:"c",
1901 Eol,
1902 Eol,
1903 Whitespace:" ",
1904 Text:"d",
1905 Eol,
1906 Eol,
1907 Dedent,
1908 Comment:"# hello",
1909 Eol,
1910 Identifier:"bob",
1911 Colon,
1912 Eol,
1913 Indent:" ",
1914 Text:"frank",
1915 Eol,
1916 Eol,
1917 Dedent,
1918 ),
1919 }
1920
1921 test! {
1922 name: tokenize_comment,
1923 text: "a:=#",
1924 tokens: (
1925 Identifier:"a",
1926 ColonEquals,
1927 Comment:"#",
1928 ),
1929 }
1930
1931 test! {
1932 name: tokenize_comment_with_bang,
1933 text: "a:=#foo!",
1934 tokens: (
1935 Identifier:"a",
1936 ColonEquals,
1937 Comment:"#foo!",
1938 ),
1939 }
1940
1941 test! {
1942 name: tokenize_order,
1943 text: "
1944 b: a
1945 @mv a b
1946
1947 a:
1948 @touch F
1949 @touch a
1950
1951 d: c
1952 @rm c
1953
1954 c: b
1955 @mv b c
1956 ",
1957 tokens: (
1958 Identifier:"b",
1959 Colon,
1960 Whitespace,
1961 Identifier:"a",
1962 Eol,
1963 Indent,
1964 Text:"@mv a b",
1965 Eol,
1966 Eol,
1967 Dedent,
1968 Identifier:"a",
1969 Colon,
1970 Eol,
1971 Indent,
1972 Text:"@touch F",
1973 Eol,
1974 Whitespace:" ",
1975 Text:"@touch a",
1976 Eol,
1977 Eol,
1978 Dedent,
1979 Identifier:"d",
1980 Colon,
1981 Whitespace,
1982 Identifier:"c",
1983 Eol,
1984 Indent,
1985 Text:"@rm c",
1986 Eol,
1987 Eol,
1988 Dedent,
1989 Identifier:"c",
1990 Colon,
1991 Whitespace,
1992 Identifier:"b",
1993 Eol,
1994 Indent,
1995 Text:"@mv b c",
1996 Eol,
1997 Dedent,
1998 ),
1999 }
2000
2001 test! {
2002 name: tokenize_parens,
2003 text: "((())) ()abc(+",
2004 tokens: (
2005 ParenL,
2006 ParenL,
2007 ParenL,
2008 ParenR,
2009 ParenR,
2010 ParenR,
2011 Whitespace,
2012 ParenL,
2013 ParenR,
2014 Identifier:"abc",
2015 ParenL,
2016 Plus,
2017 ),
2018 }
2019
2020 test! {
2021 name: crlf_newline,
2022 text: "#\r\n#asdf\r\n",
2023 tokens: (
2024 Comment:"#",
2025 Eol:"\r\n",
2026 Comment:"#asdf",
2027 Eol:"\r\n",
2028 ),
2029 }
2030
2031 test! {
2032 name: multiple_recipes,
2033 text: "a:\n foo\nb:",
2034 tokens: (
2035 Identifier:"a",
2036 Colon,
2037 Eol,
2038 Indent:" ",
2039 Text:"foo",
2040 Eol,
2041 Dedent,
2042 Identifier:"b",
2043 Colon,
2044 ),
2045 }
2046
2047 test! {
2048 name: brackets,
2049 text: "[][]",
2050 tokens: (BracketL, BracketR, BracketL, BracketR),
2051 }
2052
2053 test! {
2054 name: open_delimiter_eol,
2055 text: "[\n](\n){\n}",
2056 tokens: (
2057 BracketL, Whitespace:"\n", BracketR,
2058 ParenL, Whitespace:"\n", ParenR,
2059 BraceL, Whitespace:"\n", BraceR
2060 ),
2061 }
2062
2063 error! {
2064 name: tokenize_space_then_tab,
2065 input: "a:
2066 0
2067 1
2068\t2
2069",
2070 offset: 9,
2071 line: 3,
2072 column: 0,
2073 width: 1,
2074 kind: InconsistentLeadingWhitespace{expected: " ", found: "\t"},
2075 }
2076
2077 error! {
2078 name: tokenize_tabs_then_tab_space,
2079 input: "a:
2080\t\t0
2081\t\t 1
2082\t 2
2083",
2084 offset: 12,
2085 line: 3,
2086 column: 0,
2087 width: 3,
2088 kind: InconsistentLeadingWhitespace{expected: "\t\t", found: "\t "},
2089 }
2090
2091 error! {
2092 name: tokenize_unknown,
2093 input: "%",
2094 offset: 0,
2095 line: 0,
2096 column: 0,
2097 width: 1,
2098 kind: UnknownStartOfToken,
2099 }
2100
2101 error! {
2102 name: unterminated_string_with_escapes,
2103 input: r#"a = "\n\t\r\"\\"#,
2104 offset: 4,
2105 line: 0,
2106 column: 4,
2107 width: 1,
2108 kind: UnterminatedString,
2109 }
2110
2111 error! {
2112 name: unterminated_raw_string,
2113 input: "r a='asdf",
2114 offset: 4,
2115 line: 0,
2116 column: 4,
2117 width: 1,
2118 kind: UnterminatedString,
2119 }
2120
2121 error! {
2122 name: unterminated_interpolation,
2123 input: "foo:\n echo {{
2124 ",
2125 offset: 11,
2126 line: 1,
2127 column: 6,
2128 width: 2,
2129 kind: UnterminatedInterpolation,
2130 }
2131
2132 error! {
2133 name: unterminated_backtick,
2134 input: "`echo",
2135 offset: 0,
2136 line: 0,
2137 column: 0,
2138 width: 1,
2139 kind: UnterminatedBacktick,
2140 }
2141
2142 error! {
2143 name: unpaired_carriage_return,
2144 input: "foo\rbar",
2145 offset: 3,
2146 line: 0,
2147 column: 3,
2148 width: 1,
2149 kind: UnpairedCarriageReturn,
2150 }
2151
2152 error! {
2153 name: invalid_name_start_dash,
2154 input: "-foo",
2155 offset: 0,
2156 line: 0,
2157 column: 0,
2158 width: 1,
2159 kind: UnknownStartOfToken,
2160 }
2161
2162 error! {
2163 name: invalid_name_start_digit,
2164 input: "0foo",
2165 offset: 0,
2166 line: 0,
2167 column: 0,
2168 width: 1,
2169 kind: UnknownStartOfToken,
2170 }
2171
2172 error! {
2173 name: unterminated_string,
2174 input: r#"a = ""#,
2175 offset: 4,
2176 line: 0,
2177 column: 4,
2178 width: 1,
2179 kind: UnterminatedString,
2180 }
2181
2182 error! {
2183 name: mixed_leading_whitespace_recipe,
2184 input: "a:\n\t echo hello",
2185 offset: 3,
2186 line: 1,
2187 column: 0,
2188 width: 2,
2189 kind: MixedLeadingWhitespace{whitespace: "\t "},
2190 }
2191
2192 error! {
2193 name: mixed_leading_whitespace_normal,
2194 input: "a\n\t echo hello",
2195 offset: 2,
2196 line: 1,
2197 column: 0,
2198 width: 2,
2199 kind: MixedLeadingWhitespace{whitespace: "\t "},
2200 }
2201
2202 error! {
2203 name: mixed_leading_whitespace_indent,
2204 input: "a\n foo\n \tbar",
2205 offset: 7,
2206 line: 2,
2207 column: 0,
2208 width: 2,
2209 kind: MixedLeadingWhitespace{whitespace: " \t"},
2210 }
2211
2212 error! {
2213 name: bad_dedent,
2214 input: "a\n foo\n bar\n baz",
2215 offset: 14,
2216 line: 3,
2217 column: 0,
2218 width: 2,
2219 kind: InconsistentLeadingWhitespace{expected: " ", found: " "},
2220 }
2221
2222 error! {
2223 name: unclosed_interpolation_delimiter,
2224 input: "a:\n echo {{ foo",
2225 offset: 9,
2226 line: 1,
2227 column: 6,
2228 width: 2,
2229 kind: UnterminatedInterpolation,
2230 }
2231
2232 error! {
2233 name: unexpected_character_after_at,
2234 input: "@%",
2235 offset: 1,
2236 line: 0,
2237 column: 1,
2238 width: 1,
2239 kind: UnknownStartOfToken,
2240 }
2241
2242 error! {
2243 name: mismatched_closing_brace,
2244 input: "(]",
2245 offset: 1,
2246 line: 0,
2247 column: 1,
2248 width: 0,
2249 kind: MismatchedClosingDelimiter {
2250 open: Delimiter::Paren,
2251 close: Delimiter::Bracket,
2252 open_line: 0,
2253 },
2254 }
2255
2256 error! {
2257 name: ampersand_eof,
2258 input: "&",
2259 offset: 1,
2260 line: 0,
2261 column: 1,
2262 width: 0,
2263 kind: UnexpectedEndOfToken {
2264 expected: '&',
2265 },
2266 }
2267 error! {
2268 name: ampersand_unexpected,
2269 input: "&%",
2270 offset: 1,
2271 line: 0,
2272 column: 1,
2273 width: 1,
2274 kind: UnexpectedCharacter {
2275 expected: '&',
2276 },
2277 }
2278
2279 #[test]
2280 fn presume_error() {
2281 let compile_error = Lexer::new("justfile".as_ref(), "!")
2282 .presume('-')
2283 .unwrap_err();
2284 assert_matches!(
2285 compile_error.token,
2286 Token {
2287 offset: 0,
2288 line: 0,
2289 column: 0,
2290 length: 0,
2291 src: "!",
2292 kind: Unspecified,
2293 path: _,
2294 }
2295 );
2296 assert_matches!(&*compile_error.kind,
2297 Internal { ref message }
2298 if message == "Lexer presumed character `-`"
2299 );
2300
2301 assert_eq!(
2302 Error::Compile { compile_error }
2303 .color_display(Color::never())
2304 .to_string(),
2305 "error: Internal error, this may indicate a bug in just: Lexer presumed character `-`
2306consider filing an issue: https://github.com/casey/just/issues/new
2307 ——▶ justfile:1:1
2308 │
23091 │ !
2310 │ ^"
2311 );
2312 }
2313}