gix_diff::blob::intern

Trait TokenSource

Source
pub trait TokenSource {
    type Token: Hash + Eq;
    type Tokenizer: Iterator<Item = Self::Token>;

    // Required methods
    fn tokenize(&self) -> Self::Tokenizer;
    fn estimate_tokens(&self) -> u32;
}
Available on crate feature blob only.

Required Associated Types§

Source

type Token: Hash + Eq

Source

type Tokenizer: Iterator<Item = Self::Token>

Required Methods§

Source

fn tokenize(&self) -> Self::Tokenizer

Source

fn estimate_tokens(&self) -> u32

Implementations on Foreign Types§

Source§

impl<'a> TokenSource for &'a str

By default a line diff is produced for a string

Source§

type Token = &'a str

Source§

type Tokenizer = Lines<'a, false>

Source§

fn tokenize(&self) -> <&'a str as TokenSource>::Tokenizer

Source§

fn estimate_tokens(&self) -> u32

Source§

impl<'a> TokenSource for &'a [u8]

By default a line diff is produced for a bytes

Source§

type Token = &'a [u8]

Source§

type Tokenizer = ByteLines<'a, false>

Source§

fn tokenize(&self) -> <&'a [u8] as TokenSource>::Tokenizer

Source§

fn estimate_tokens(&self) -> u32

Implementors§

Source§

impl<'a, const INCLUDE_LINE_TERMINATOR: bool> TokenSource for ByteLines<'a, INCLUDE_LINE_TERMINATOR>

By default a line diff is produced for a string

Source§

type Token = &'a [u8]

Source§

type Tokenizer = ByteLines<'a, INCLUDE_LINE_TERMINATOR>

Source§

impl<'a, const INCLUDE_LINE_TERMINATOR: bool> TokenSource for Lines<'a, INCLUDE_LINE_TERMINATOR>

By default a line diff is produced for a string

Source§

type Token = &'a str

Source§

type Tokenizer = Lines<'a, INCLUDE_LINE_TERMINATOR>