llama_cpp_sys_4

Function common_tokenize

Source
pub unsafe extern "C" fn common_tokenize(
    ctx: *const llama_context,
    text: *const std_string,
    add_special: bool,
    parse_special: bool,
) -> __BindgenOpaqueArray<u64, 3usize>