Struct GenericListViewArray

pub struct GenericListViewArray<OffsetSize: OffsetSizeTrait> { /* private fields */ }

An array of variable length lists, specifically in the list-view layout.

Differs from GenericListArray (which represents the list layout) in that the sizes of the child arrays are explicitly encoded in a separate buffer, instead of being derived from the difference between subsequent offsets in the offset buffer.

This allows the offsets (and subsequently child data) to be out of order. It also allows take / filter operations to be implemented without copying the underlying data.

Representation

Given the same example array from GenericListArray, it would be represented as such via a list-view layout array:

                                        ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
                                                                        ┌ ─ ─ ─ ─ ─ ─ ┐    │
 ┌─────────────┐  ┌───────┐             │     ┌───┐   ┌───┐   ┌───┐       ┌───┐ ┌───┐
 │   [A,B,C]   │  │ (0,3) │                   │ 1 │   │ 0 │   │ 3 │     │ │ 1 │ │ A │ │ 0  │
 ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
 │      []     │  │ (3,0) │                   │ 1 │   │ 3 │   │ 0 │     │ │ 1 │ │ B │ │ 1  │
 ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
 │    NULL     │  │ (?,?) │                   │ 0 │   │ ? │   │ ? │     │ │ 1 │ │ C │ │ 2  │
 ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
 │     [D]     │  │ (4,1) │                   │ 1 │   │ 4 │   │ 1 │     │ │ ? │ │ ? │ │ 3  │
 ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
 │  [NULL, F]  │  │ (5,2) │                   │ 1 │   │ 5 │   │ 2 │     │ │ 1 │ │ D │ │ 4  │
 └─────────────┘  └───────┘             │     └───┘   └───┘   └───┘       ├───┤ ├───┤
                                                                        │ │ 0 │ │ ? │ │ 5  │
    Logical       Logical               │  Validity  Offsets  Sizes       ├───┤ ├───┤
     Values       Offset                   (nulls)                      │ │ 1 │ │ F │ │ 6  │
                  & Size                │                                 └───┘ └───┘
                                                                        │    Values   │    │
                (offsets[i],            │   ListViewArray                   (Array)
                 sizes[i])                                              └ ─ ─ ─ ─ ─ ─ ┘    │
                                        └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─

Another way of representing the same array but taking advantage of the offsets being out of order:

                                        ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─
                                                                        ┌ ─ ─ ─ ─ ─ ─ ┐    │
 ┌─────────────┐  ┌───────┐             │     ┌───┐   ┌───┐   ┌───┐       ┌───┐ ┌───┐
 │   [A,B,C]   │  │ (2,3) │                   │ 1 │   │ 2 │   │ 3 │     │ │ 0 │ │ ? │ │ 0  │
 ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
 │      []     │  │ (0,0) │                   │ 1 │   │ 0 │   │ 0 │     │ │ 1 │ │ F │ │ 1  │
 ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
 │    NULL     │  │ (?,?) │                   │ 0 │   │ ? │   │ ? │     │ │ 1 │ │ A │ │ 2  │
 ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
 │     [D]     │  │ (5,1) │                   │ 1 │   │ 5 │   │ 1 │     │ │ 1 │ │ B │ │ 3  │
 ├─────────────┤  ├───────┤             │     ├───┤   ├───┤   ├───┤       ├───┤ ├───┤
 │  [NULL, F]  │  │ (0,2) │                   │ 1 │   │ 0 │   │ 2 │     │ │ 1 │ │ C │ │ 4  │
 └─────────────┘  └───────┘             │     └───┘   └───┘   └───┘       ├───┤ ├───┤
                                                                        │ │ 1 │ │ D │ │ 5  │
    Logical       Logical               │  Validity  Offsets  Sizes       └───┘ └───┘
     Values       Offset                   (nulls)                      │    Values   │    │
                  & Size                │                                   (Array)  
                                                                        └ ─ ─ ─ ─ ─ ─ ┘    │
                (offsets[i],            │   ListViewArray                          
                 sizes[i])                                                                 │
                                        └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─

Implementations

impl<OffsetSize: OffsetSizeTrait> GenericListViewArray<OffsetSize>

pub const DATA_TYPE_CONSTRUCTOR: fn(_: FieldRef) -> DataType

The data type constructor of listview array. The input is the schema of the child array and the output is the DataType, ListView or LargeListView.

pub fn try_new( field: FieldRef, offsets: ScalarBuffer<OffsetSize>, sizes: ScalarBuffer<OffsetSize>, values: ArrayRef, nulls: Option<NullBuffer>, ) -> Result<Self, ArrowError>

Create a new GenericListViewArray from the provided parts

Errors

Errors if

• offsets.len() != sizes.len()
• offsets.len() != nulls.len()
• offsets[i] > values.len()
• !field.is_nullable() && values.is_nullable()
• field.data_type() != values.data_type()
• 0 <= offsets[i] <= length of the child array
• 0 <= offsets[i] + size[i] <= length of the child array

pub fn new( field: FieldRef, offsets: ScalarBuffer<OffsetSize>, sizes: ScalarBuffer<OffsetSize>, values: ArrayRef, nulls: Option<NullBuffer>, ) -> Self

Create a new GenericListViewArray from the provided parts

Panics

Panics if Self::try_new returns an error

pub fn new_null(field: FieldRef, len: usize) -> Self

Create a new GenericListViewArray of length len where all values are null

pub fn into_parts( self, ) -> (FieldRef, ScalarBuffer<OffsetSize>, ScalarBuffer<OffsetSize>, ArrayRef, Option<NullBuffer>)

Deconstruct this array into its constituent parts

pub fn offsets(&self) -> &ScalarBuffer<OffsetSize>

Returns a reference to the offsets of this list

Unlike Self::value_offsets this returns the ScalarBuffer allowing for zero-copy cloning

pub fn values(&self) -> &ArrayRef

Returns a reference to the values of this list

pub fn sizes(&self) -> &ScalarBuffer<OffsetSize>

Returns a reference to the sizes of this list

Unlike Self::value_sizes this returns the ScalarBuffer allowing for zero-copy cloning

pub fn value_type(&self) -> DataType

Returns a clone of the value type of this list.

pub unsafe fn value_unchecked(&self, i: usize) -> ArrayRef

Returns ith value of this list view array.

Safety

Caller must ensure that the index is within the array bounds

pub fn value(&self, i: usize) -> ArrayRef

Returns ith value of this list view array.

Panics

Panics if the index is out of bounds

pub fn value_offsets(&self) -> &[OffsetSize]

Returns the offset values in the offsets buffer

pub fn value_sizes(&self) -> &[OffsetSize]

Returns the sizes values in the offsets buffer

pub fn value_size(&self, i: usize) -> OffsetSize

Returns the size for value at index i.

pub fn value_offset(&self, i: usize) -> OffsetSize

Returns the offset for value at index i.

pub fn iter(&self) -> GenericListViewArrayIter<'_, OffsetSize>

Constructs a new iterator

pub fn slice(&self, offset: usize, length: usize) -> Self

Returns a zero-copy slice of this array with the indicated offset and length.

Trait Implementations

impl<OffsetSize: OffsetSizeTrait> Array for GenericListViewArray<OffsetSize>

fn as_any(&self) -> &dyn Any

Returns the array as Any so that it can be downcasted to a specific implementation.

fn to_data(&self) -> ArrayData

Returns the underlying data of this array

fn into_data(self) -> ArrayData

Returns the underlying data of this array

fn data_type(&self) -> &DataType

Returns a reference to the DataType of this array.

fn slice(&self, offset: usize, length: usize) -> ArrayRef

Returns a zero-copy slice of this array with the indicated offset and length.

fn len(&self) -> usize

Returns the length (i.e., number of elements) of this array.

fn is_empty(&self) -> bool

Returns whether this array is empty.

fn shrink_to_fit(&mut self)

Shrinks the capacity of any exclusively owned buffer as much as possible

fn offset(&self) -> usize

Returns the offset into the underlying data used by this array(-slice). Note that the underlying data can be shared by many arrays. This defaults to 0.

fn nulls(&self) -> Option<&NullBuffer>

Returns the null buffer of this array if any.

fn logical_null_count(&self) -> usize

Returns the total number of logical null values in this array.

fn get_buffer_memory_size(&self) -> usize

Returns the total number of bytes of memory pointed to by this array. The buffers store bytes in the Arrow memory format, and include the data as well as the validity map. Note that this does not always correspond to the exact memory usage of an array, since multiple arrays can share the same buffers or slices thereof.

fn get_array_memory_size(&self) -> usize

Returns the total number of bytes of memory occupied physically by this array. This value will always be greater than returned by get_buffer_memory_size() and includes the overhead of the data structures that contain the pointers to the various buffers.

fn logical_nulls(&self) -> Option<NullBuffer>

Returns a potentially computed NullBuffer that represents the logical null values of this array, if any.

fn is_null(&self, index: usize) -> bool

Returns whether the element at index is null according to Array::nulls

fn is_valid(&self, index: usize) -> bool

Returns whether the element at index is not null, the opposite of Self::is_null.

fn null_count(&self) -> usize

Returns the total number of physical null values in this array.

fn is_nullable(&self) -> bool

Returns false if the array is guaranteed to not contain any logical nulls

impl<OffsetSize: OffsetSizeTrait> ArrayAccessor for &GenericListViewArray<OffsetSize>

type Item = Arc<dyn Array>

The Arrow type of the element being accessed.

fn value(&self, index: usize) -> Self::Item

Returns the element at index i

unsafe fn value_unchecked(&self, index: usize) -> Self::Item

Returns the element at index i

impl<OffsetSize: Clone + OffsetSizeTrait> Clone for GenericListViewArray<OffsetSize>

fn clone(&self) -> GenericListViewArray<OffsetSize>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<OffsetSize: OffsetSizeTrait> Debug for GenericListViewArray<OffsetSize>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<OffsetSize: OffsetSizeTrait> From<ArrayData> for GenericListViewArray<OffsetSize>

fn from(data: ArrayData) -> Self

Converts to this type from the input type.

impl<OffsetSize: OffsetSizeTrait> From<FixedSizeListArray> for GenericListViewArray<OffsetSize>

fn from(value: FixedSizeListArray) -> Self

Converts to this type from the input type.

impl<OffsetSize: OffsetSizeTrait> From<GenericListViewArray<OffsetSize>> for ArrayData

fn from(array: GenericListViewArray<OffsetSize>) -> Self

Converts to this type from the input type.

impl<OffsetSize: OffsetSizeTrait> PartialEq for GenericListViewArray<OffsetSize>

fn eq(&self, other: &Self) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.