lance_table::io::commit::external_manifest

Trait ExternalManifestStore

Source 
pub trait ExternalManifestStore:
    Debug
    + Send
    + Sync {
    // Required methods
    fn get<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
    ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn get_latest_version<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Option<(u64, String)>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait;
    fn put_if_not_exists<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
        path: &'life2 str,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;
    fn put_if_exists<'life0, 'life1, 'life2, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
        version: u64,
        path: &'life2 str,
    ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait,
             'life2: 'async_trait;

    // Provided method
    fn get_latest_manifest_location<'life0, 'life1, 'async_trait>(
        &'life0 self,
        base_uri: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<Option<ManifestLocation>>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
}
Expand description

External manifest store

This trait abstracts an external storage for source of truth for manifests. The storage is expected to remember (uri, version) -> manifest_path and able to run transactions on the manifest_path.

This trait is called an External manifest store because the store is expected to work in tandem with the object store. We are only leveraging the external store for concurrent commit. Any manifest committed thru this trait should ultimately be materialized in the object store. For a visual explanation of the commit loop see https://github.com/lancedb/lance/assets/12615154/b0822312-0826-432a-b554-3965f8d48d04

Required Methods§

Source

fn get<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, ) -> Pin<Box<dyn Future<Output = Result<String>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the manifest path for a given base_uri and version

Source

fn get_latest_version<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Option<(u64, String)>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the latest version of a dataset at the base_uri, and the path to the manifest. The path is provided as an optimization. The path is deterministic based on the version and the store should not customize it.

Source

fn put_if_not_exists<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, path: &'life2 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Put the manifest path for a given base_uri and version, should fail if the version already exists

Source

fn put_if_exists<'life0, 'life1, 'life2, 'async_trait>( &'life0 self, base_uri: &'life1 str, version: u64, path: &'life2 str, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,

Put the manifest path for a given base_uri and version, should fail if the version does not already exist

Provided Methods§

Source

fn get_latest_manifest_location<'life0, 'life1, 'async_trait>( &'life0 self, base_uri: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<Option<ManifestLocation>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Get the latest manifest location for a given base_uri.

By default, this calls get_latest_version. Impls should override this method if they store both the location and size of the latest manifest.

Implementors§