1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73
//! Print line and column positions for each word in a file.
//!
//! Usage:
//! cargo run --example extras <path/to/file>
//!
//! Example:
//! cargo run --example extras Cargo.toml
//!
//! This is a small example on how to use
//! [`Extras`](https://docs.rs/logos/latest/logos/trait.Logos.html#associatedtype.Extras)
//! to convey some (mutable) internal state from token to token.
//!
//! Here, the extras will be a tuple with the following fields:
//!
//! + 0. the line number;
//! + 1. the char index of the current line.
//!
//! From then, one can easily compute the column number of some token by computing:
//!
//! ```rust,no_run,no_playground
//! fn get_column(lex: &Lexer<Token>) -> usize {
//! lex.span().start - lex.extras.1
//! }
//! ```
/* ANCHOR: all */
use logos::{Lexer, Logos, Skip};
use std::env;
use std::fs;
/* ANCHOR: callbacks */
/// Update the line count and the char index.
fn newline_callback(lex: &mut Lexer<Token>) -> Skip {
lex.extras.0 += 1;
lex.extras.1 = lex.span().end;
Skip
}
/// Compute the line and column position for the current word.
fn word_callback(lex: &mut Lexer<Token>) -> (usize, usize) {
let line = lex.extras.0;
let column = lex.span().start - lex.extras.1;
(line, column)
}
/* ANCHOR_END: callbacks */
/* ANCHOR: tokens */
/// Simple tokens to retrieve words and their location.
#[derive(Debug, Logos)]
#[logos(extras = (usize, usize))]
enum Token {
#[regex(r"\n", newline_callback)]
Newline,
#[regex(r"\w+", word_callback)]
Word((usize, usize)),
}
/* ANCHOR_END: tokens */
fn main() {
let src = fs::read_to_string(env::args().nth(1).expect("Expected file argument"))
.expect("Failed to read file");
let mut lex = Token::lexer(src.as_str());
while let Some(token) = lex.next() {
if let Ok(Token::Word((line, column))) = token {
println!("Word '{}' found at ({}, {})", lex.slice(), line, column);
}
}
}
/* ANCHOR_END: all */