Trait TokenSource

pub trait TokenSource {
    type Token: Hash + Eq;
    type Tokenizer: Iterator<Item = Self::Token>;

    // Required methods
    fn tokenize(&self) -> Self::Tokenizer;
    fn estimate_tokens(&self) -> u32;
}

Available on crate feature blob only.

Required Associated Types

type Token: Hash + Eq

type Tokenizer: Iterator<Item = Self::Token>

Required Methods

fn tokenize(&self) -> Self::Tokenizer

fn estimate_tokens(&self) -> u32

Implementations on Foreign Types

impl<'a> TokenSource for &'a str

By default a line diff is produced for a string

type Token = &'a str

type Tokenizer = Lines<'a, false>

fn tokenize(&self) -> <&'a str as TokenSource>::Tokenizer

fn estimate_tokens(&self) -> u32

impl<'a> TokenSource for &'a [u8]

By default a line diff is produced for a bytes

type Token = &'a [u8]

type Tokenizer = ByteLines<'a, false>

fn tokenize(&self) -> <&'a [u8] as TokenSource>::Tokenizer

fn estimate_tokens(&self) -> u32

Implementors

impl<'a, const INCLUDE_LINE_TERMINATOR: bool> TokenSource for ByteLines<'a, INCLUDE_LINE_TERMINATOR>

By default a line diff is produced for a string

type Token = &'a [u8]

type Tokenizer = ByteLines<'a, INCLUDE_LINE_TERMINATOR>

impl<'a, const INCLUDE_LINE_TERMINATOR: bool> TokenSource for Lines<'a, INCLUDE_LINE_TERMINATOR>

By default a line diff is produced for a string

type Token = &'a str

type Tokenizer = Lines<'a, INCLUDE_LINE_TERMINATOR>