Struct Vocabulary

pub struct Vocabulary { /* private fields */ }

Vocabulary of large language model.

Examples

Create a vocabulary from a pretrained model.

use outlines_core::prelude::*;

let vocabulary = Vocabulary::from_pretrained("openai-community/gpt2", None);

Create a vocabulary from a pretrained model with some additional parameters.

use outlines_core::prelude::*;

let params = FromPretrainedParameters {
    revision: "607a30d783dfa663caf39e06633721c8d4cfcd7e".to_string(),
    ..Default::default()
};
let vocabulary = Vocabulary::from_pretrained("openai-community/gpt2", Some(params));

Create an empty vocabulary and manually insert some tokens.

use outlines_core::prelude::*;

let eos_token_id = 1;
let mut vocabulary = Vocabulary::new(eos_token_id);

vocabulary.try_insert("token", 0).expect("New token inserted");
assert_eq!(vocabulary.token_ids("token"), Some(&vec![0]));
assert_eq!(vocabulary.tokens().len(), 1);
assert_eq!(vocabulary.eos_token_id(), eos_token_id);

vocabulary.remove("token");
assert_eq!(vocabulary.token_ids("token"), None);

Implementations

impl Vocabulary

pub fn new(eos_token_id: TokenId) -> Self

Creates an empty vocabulary.

pub fn from_pretrained( model: &str, parameters: Option<FromPretrainedParameters>, ) -> Result<Self>

Creates the vocabulary of pre-trained model from Hugging Face Hub.

pub fn tokens(&self) -> &HashMap<Token, Vec<TokenId>>

Returns all tokens with their token ids in vocabulary.

pub fn token_ids(&self, token: impl AsRef<[u8]>) -> Option<&Vec<TokenId>>

Returns all token ids per provided token if available in the vocabulary.

pub fn eos_token_id(&self) -> TokenId

Gets the identifier of the special end of the sentence token.

pub fn try_insert( &mut self, token: impl Into<Token>, id: TokenId, ) -> Result<(), Error>

Inserts a token to the vocabulary with the specified identifier.

pub fn remove(&mut self, token: impl Into<Token>)

Removes a given token from the vocabulary.

Trait Implementations

impl<'__de> BorrowDecode<'__de> for Vocabulary

fn borrow_decode<__D: BorrowDecoder<'__de>>( decoder: &mut __D, ) -> Result<Self, DecodeError>

Attempt to decode this type with the given BorrowDecode.

impl Clone for Vocabulary

fn clone(&self) -> Vocabulary

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Vocabulary

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Decode for Vocabulary

fn decode<__D: Decoder>(decoder: &mut __D) -> Result<Self, DecodeError>

Attempt to decode this type with the given Decode.

impl Default for Vocabulary

fn default() -> Vocabulary

Returns the “default value” for a type.

impl Display for Vocabulary

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Encode for Vocabulary

fn encode<__E: Encoder>(&self, encoder: &mut __E) -> Result<(), EncodeError>

Encode a given type.

impl PartialEq for Vocabulary

fn eq(&self, other: &Vocabulary) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl TryFrom<(u32, HashMap<String, Vec<u32>, FxBuildHasher>)> for Vocabulary

type Error = Error

The type returned in the event of a conversion error.

fn try_from( values: (TokenId, HashMap<String, Vec<TokenId>>), ) -> Result<Self, Self::Error>

Performs the conversion.

impl TryFrom<(u32, HashMap<Vec<u8>, Vec<u32>, FxBuildHasher>)> for Vocabulary

type Error = Error

The type returned in the event of a conversion error.

fn try_from( values: (TokenId, HashMap<Token, Vec<TokenId>>), ) -> Result<Self, Self::Error>

Performs the conversion.

impl StructuralPartialEq for Vocabulary