Struct FileWriter

pub struct FileWriter<M: ManifestProvider + Send + Sync> {
    pub object_writer: ObjectWriter,
    /* private fields */
}

FileWriter writes Arrow RecordBatch to one Lance file.

use lance::io::FileWriter;
use futures::stream::Stream;

let mut file_writer = FileWriter::new(object_store, &path, &schema);
while let Ok(batch) = stream.next().await {
    file_writer.write(&batch).unwrap();
}
// Need to close file writer to flush buffer and footer.
file_writer.shutdown();

Fields

object_writer: ObjectWriter

Implementations

impl<M: ManifestProvider + Send + Sync> FileWriter<M>

pub async fn try_new( object_store: &ObjectStore, path: &Path, schema: Schema, options: &FileWriterOptions, ) -> Result<Self>

pub fn with_object_writer( object_writer: ObjectWriter, schema: Schema, options: &FileWriterOptions, ) -> Result<Self>

pub fn schema(&self) -> &Schema

Return the schema of the file writer.

pub async fn write(&mut self, batches: &[RecordBatch]) -> Result<()>

Write a RecordBatch to the open file. All RecordBatch will be treated as one RecordBatch on disk

Returns Err if the schema does not match with the batch.

pub fn add_metadata(&mut self, key: &str, value: &str)

Add schema metadata, as (key, value) pair to the file.

pub async fn finish_with_metadata( &mut self, metadata: &HashMap<String, String>, ) -> Result<usize>

pub async fn finish(&mut self) -> Result<usize>

pub fn len(&self) -> usize

Total records written in this file.

pub async fn tell(&mut self) -> Result<usize>

Total bytes written so far

pub fn next_batch_id(&self) -> i32

Return the id of the next batch to be written.

pub fn is_empty(&self) -> bool