Struct DatasetBuilder

pub struct DatasetBuilder { /* private fields */ }

builder for loading a Dataset.

Implementations

impl DatasetBuilder

pub fn from_uri<T: AsRef<str>>(table_uri: T) -> Self

impl DatasetBuilder

pub fn with_index_cache_size(self, cache_size: usize) -> Self

Set the cache size for indices. Set to zero, to disable the cache.

pub fn with_metadata_cache_size(self, cache_size: usize) -> Self

Set the cache size for the file metadata. Set to zero to disable this cache.

pub fn with_block_size(self, block_size: usize) -> Self

The block size passed to the underlying Object Store reader.

This is used to control the minimal request size. Defaults to 4KB for local files and 64KB for others

pub fn with_version(self, version: u64) -> Self

Sets version for the builder using a version number

pub fn with_tag(self, tag: &str) -> Self

Sets version for the builder using a tag

pub fn with_commit_handler(self, commit_handler: Arc<dyn CommitHandler>) -> Self

pub fn with_s3_credentials_refresh_offset(self, offset: Duration) -> Self

Sets the s3 credentials refresh. This only applies to s3 storage.

pub fn with_aws_credentials_provider( self, credentials: AwsCredentialProvider, ) -> Self

Sets the aws credentials provider. This only applies to aws object store.

pub fn with_object_store( self, object_store: Arc<DynObjectStore>, location: Url, commit_handler: Arc<dyn CommitHandler>, ) -> Self

Directly set the object store to use.

pub fn with_serialized_manifest(self, manifest: &[u8]) -> Result<Self>

Use a serialized manifest instead of loading it from the object store.

This is common when transferring a dataset across IPC boundaries.

pub fn with_storage_options( self, storage_options: HashMap<String, String>, ) -> Self

Set options used to initialize storage backend

Options may be passed in the HashMap or set as environment variables. See documentation of underlying object store implementation for details.

• Azure options
• S3 options
• Google options

pub fn with_storage_option( self, key: impl AsRef<str>, value: impl AsRef<str>, ) -> Self

Set a single option used to initialize storage backend For example, to set the region for S3, you can use:

let builder = DatasetBuilder::from_uri("s3://bucket/path")
    .with_storage_option("region", "us-east-1");

pub fn with_read_params(self, read_params: ReadParams) -> Self

Set options based on ReadParams.

pub fn with_write_params(self, write_params: WriteParams) -> Self

Set options based on WriteParams.

pub fn with_session(self, session: Arc<Session>) -> Self

Re-use an existing session.

The session holds caches for index and metadata.

If this is set, then with_index_cache_size and with_metadata_cache_size are ignored.

pub fn with_object_store_registry( self, registry: Arc<ObjectStoreRegistry>, ) -> Self

pub async fn build_object_store( self, ) -> Result<(ObjectStore, Path, Arc<dyn CommitHandler>)>

Build a lance object store for the given config

pub async fn load(self) -> Result<Dataset>

Trait Implementations

impl Clone for DatasetBuilder

fn clone(&self) -> DatasetBuilder

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for DatasetBuilder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.