Struct LlamaModelParams

pub struct LlamaModelParams { /* private fields */ }

A safe wrapper around llama_model_params.

Implementations

impl LlamaModelParams

pub fn kv_overrides(&self) -> KvOverrides<'_>

See KvOverrides

Examples

let params = Box::pin(LlamaModelParams::default());
let kv_overrides = params.kv_overrides();
let count = kv_overrides.into_iter().count();
assert_eq!(count, 0);

pub fn append_kv_override( self: Pin<&mut Self>, key: &CStr, value: ParamOverrideValue, )

Appends a key-value override to the model parameters. It must be pinned as this creates a self-referential struct.

Examples

use std::pin::pin;
let mut params = pin!(LlamaModelParams::default());
let key = CString::new("key").expect("CString::new failed");
params.as_mut().append_kv_override(&key, ParamOverrideValue::Int(50));

let kv_overrides = params.kv_overrides().into_iter().collect::<Vec<_>>();
assert_eq!(kv_overrides.len(), 1);

let (k, v) = &kv_overrides[0];
assert_eq!(v, &ParamOverrideValue::Int(50));

assert_eq!(k.to_bytes(), b"key", "expected key to be 'key', was {:?}", k);

impl LlamaModelParams

pub fn n_gpu_layers(&self) -> i32

Get the number of layers to offload to the GPU.

pub fn main_gpu(&self) -> i32

The GPU that is used for scratch and small tensors

pub fn vocab_only(&self) -> bool

only load the vocabulary, no weights

pub fn use_mmap(&self) -> bool

use mmap if possible

pub fn use_mlock(&self) -> bool

force system to keep model in RAM

pub fn with_n_gpu_layers(self, n_gpu_layers: u32) -> Self

sets the number of gpu layers to offload to the GPU.

let params = LlamaModelParams::default();
let params = params.with_n_gpu_layers(1);
assert_eq!(params.n_gpu_layers(), 1);

pub fn with_main_gpu(self, main_gpu: i32) -> Self

sets the main GPU

pub fn with_vocab_only(self, vocab_only: bool) -> Self

sets vocab_only

pub fn with_use_mlock(self, use_mlock: bool) -> Self

sets use_mlock

Trait Implementations

impl Debug for LlamaModelParams

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for LlamaModelParams

Default parameters for LlamaModel. (as defined in llama.cpp by llama_model_default_params)

let params = LlamaModelParams::default();
#[cfg(not(target_os = "macos"))]
assert_eq!(params.n_gpu_layers(), 0, "n_gpu_layers should be 0");
#[cfg(target_os = "macos")]
assert_eq!(params.n_gpu_layers(), 999, "n_gpu_layers should be 999");
assert_eq!(params.main_gpu(), 0, "main_gpu should be 0");
assert_eq!(params.vocab_only(), false, "vocab_only should be false");
assert_eq!(params.use_mmap(), true, "use_mmap should be true");
assert_eq!(params.use_mlock(), false, "use_mlock should be false");

fn default() -> Self

Returns the “default value” for a type.