extras/
extras.rs

1//! Print line and column positions for each word in a file.
2//!
3//! Usage:
4//!     cargo run --example extras <path/to/file>
5//!
6//! Example:
7//!     cargo run --example extras Cargo.toml
8//!
9//! This is a small example on how to use
10//! [`Extras`](https://docs.rs/logos/latest/logos/trait.Logos.html#associatedtype.Extras)
11//! to convey some (mutable) internal state from token to token.
12//!
13//! Here, the extras will be a tuple with the following fields:
14//!
15//! + 0. the line number;
16//! + 1. the char index of the current line.
17//!
18//! From then, one can easily compute the column number of some token by computing:
19//!
20//! ```rust,no_run,no_playground
21//! fn get_column(lex: &Lexer<Token>) -> usize {
22//!     lex.span().start - lex.extras.1
23//! }
24//! ```
25
26/* ANCHOR: all */
27use logos::{Lexer, Logos, Skip};
28use std::env;
29use std::fs;
30
31/* ANCHOR: callbacks */
32/// Update the line count and the char index.
33fn newline_callback(lex: &mut Lexer<Token>) -> Skip {
34    lex.extras.0 += 1;
35    lex.extras.1 = lex.span().end;
36    Skip
37}
38
39/// Compute the line and column position for the current word.
40fn word_callback(lex: &mut Lexer<Token>) -> (usize, usize) {
41    let line = lex.extras.0;
42    let column = lex.span().start - lex.extras.1;
43
44    (line, column)
45}
46/* ANCHOR_END: callbacks */
47
48/* ANCHOR: tokens */
49/// Simple tokens to retrieve words and their location.
50#[derive(Debug, Logos)]
51#[logos(extras = (usize, usize))]
52enum Token {
53    #[regex(r"\n", newline_callback)]
54    Newline,
55
56    #[regex(r"\w+", word_callback)]
57    Word((usize, usize)),
58}
59/* ANCHOR_END: tokens */
60
61fn main() {
62    let src = fs::read_to_string(env::args().nth(1).expect("Expected file argument"))
63        .expect("Failed to read file");
64
65    let mut lex = Token::lexer(src.as_str());
66
67    while let Some(token) = lex.next() {
68        if let Ok(Token::Word((line, column))) = token {
69            println!("Word '{}' found at ({}, {})", lex.slice(), line, column);
70        }
71    }
72}
73/* ANCHOR_END: all */
extras/extras.rs

extras/
extras.rs
