Struct arrow_ipc::reader::FileDecoder

pub struct FileDecoder { /* private fields */ }

A low-level, push-based interface for reading an IPC file

For a higher-level interface see FileReader

// Write an IPC file

let batch = RecordBatch::try_from_iter([
    ("a", Arc::new(Int32Array::from(vec![1, 2, 3])) as _),
    ("b", Arc::new(Int32Array::from(vec![1, 2, 3])) as _),
    ("c", Arc::new(DictionaryArray::<Int32Type>::from_iter(["hello", "hello", "world"])) as _),
]).unwrap();

let schema = batch.schema();

let mut out = Vec::with_capacity(1024);
let mut writer = FileWriter::try_new(&mut out, schema.as_ref()).unwrap();
writer.write(&batch).unwrap();
writer.finish().unwrap();

drop(writer);

// Read IPC file

let buffer = Buffer::from_vec(out);
let trailer_start = buffer.len() - 10;
let footer_len = read_footer_length(buffer[trailer_start..].try_into().unwrap()).unwrap();
let footer = root_as_footer(&buffer[trailer_start - footer_len..trailer_start]).unwrap();

let back = fb_to_schema(footer.schema().unwrap());
assert_eq!(&back, schema.as_ref());

let mut decoder = FileDecoder::new(schema, footer.version());

// Read dictionaries
for block in footer.dictionaries().iter().flatten() {
    let block_len = block.bodyLength() as usize + block.metaDataLength() as usize;
    let data = buffer.slice_with_length(block.offset() as _, block_len);
    decoder.read_dictionary(&block, &data).unwrap();
}

// Read record batch
let batches = footer.recordBatches().unwrap();
assert_eq!(batches.len(), 1); // Only wrote a single batch

let block = batches.get(0);
let block_len = block.bodyLength() as usize + block.metaDataLength() as usize;
let data = buffer.slice_with_length(block.offset() as _, block_len);
let back = decoder.read_record_batch(block, &data).unwrap().unwrap();

assert_eq!(batch, back);

Implementations

impl FileDecoder

pub fn new(schema: SchemaRef, version: MetadataVersion) -> Self

Create a new FileDecoder with the given schema and version

pub fn with_projection(self, projection: Vec<usize>) -> Self

Specify a projection

pub fn read_dictionary( &mut self, block: &Block, buf: &Buffer ) -> Result<(), ArrowError>

Read the dictionary with the given block and data buffer

pub fn read_record_batch( &self, block: &Block, buf: &Buffer ) -> Result<Option<RecordBatch>, ArrowError>

Read the RecordBatch with the given block and data buffer

Trait Implementations

impl Debug for FileDecoder

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.