Enum IR

pub enum IR {
    PythonScan {
        options: PythonOptions,
    },
    Slice {
        input: Node,
        offset: i64,
        len: IdxSize,
    },
    Filter {
        input: Node,
        predicate: ExprIR,
    },
    Scan {
        sources: ScanSources,
        file_info: FileInfo,
        hive_parts: Option<Arc<Vec<HivePartitions>>>,
        predicate: Option<ExprIR>,
        output_schema: Option<SchemaRef>,
        scan_type: FileScan,
        file_options: FileScanOptions,
    },
    DataFrameScan {
        df: Arc<DataFrame>,
        schema: SchemaRef,
        output_schema: Option<SchemaRef>,
        filter: Option<ExprIR>,
    },
    SimpleProjection {
        input: Node,
        columns: SchemaRef,
    },
    Reduce {
        input: Node,
        exprs: Vec<ExprIR>,
        schema: SchemaRef,
    },
    Select {
        input: Node,
        expr: Vec<ExprIR>,
        schema: SchemaRef,
        options: ProjectionOptions,
    },
    Sort {
        input: Node,
        by_column: Vec<ExprIR>,
        slice: Option<(i64, usize)>,
        sort_options: SortMultipleOptions,
    },
    Cache {
        input: Node,
        id: usize,
        cache_hits: u32,
    },
    GroupBy {
        input: Node,
        keys: Vec<ExprIR>,
        aggs: Vec<ExprIR>,
        schema: SchemaRef,
        apply: Option<Arc<dyn DataFrameUdf>>,
        maintain_order: bool,
        options: Arc<GroupbyOptions>,
    },
    Join {
        input_left: Node,
        input_right: Node,
        schema: SchemaRef,
        left_on: Vec<ExprIR>,
        right_on: Vec<ExprIR>,
        options: Arc<JoinOptions>,
    },
    HStack {
        input: Node,
        exprs: Vec<ExprIR>,
        schema: SchemaRef,
        options: ProjectionOptions,
    },
    Distinct {
        input: Node,
        options: DistinctOptionsIR,
    },
    MapFunction {
        input: Node,
        function: FunctionIR,
    },
    Union {
        inputs: Vec<Node>,
        options: UnionOptions,
    },
    HConcat {
        inputs: Vec<Node>,
        schema: SchemaRef,
        options: HConcatOptions,
    },
    ExtContext {
        input: Node,
        contexts: Vec<Node>,
        schema: SchemaRef,
    },
    Sink {
        input: Node,
        payload: SinkType,
    },
    Invalid,
}

IR is a representation of DslPlan with Nodes which are allocated in an Arena In this IR the logical plan has access to the full dataset.

Variants

PythonScan

Available on crate feature python only.

Fields

options: PythonOptions

Slice

Fields

input: Node

offset: i64

len: IdxSize

Filter

Fields

input: Node

predicate: ExprIR

Scan

Fields

sources: ScanSources

file_info: FileInfo

hive_parts: Option<Arc<Vec<HivePartitions>>>

predicate: Option<ExprIR>

output_schema: Option<SchemaRef>

schema of the projected file

scan_type: FileScan

file_options: FileScanOptions

generic options that can be used for all file types.

DataFrameScan

Fields

df: Arc<DataFrame>

schema: SchemaRef

output_schema: Option<SchemaRef>

filter: Option<ExprIR>

SimpleProjection

Fields

input: Node

columns: SchemaRef

Reduce

Fields

input: Node

exprs: Vec<ExprIR>

schema: SchemaRef

Select

Fields

input: Node

expr: Vec<ExprIR>

schema: SchemaRef

options: ProjectionOptions

Sort

Fields

input: Node

by_column: Vec<ExprIR>

slice: Option<(i64, usize)>

sort_options: SortMultipleOptions

Cache

Fields

input: Node

id: usize

cache_hits: u32

How many hits the cache must be saved in memory.

GroupBy

Fields

input: Node

keys: Vec<ExprIR>

aggs: Vec<ExprIR>

schema: SchemaRef

apply: Option<Arc<dyn DataFrameUdf>>

maintain_order: bool

options: Arc<GroupbyOptions>

Join

Fields

input_left: Node

input_right: Node

schema: SchemaRef

left_on: Vec<ExprIR>

right_on: Vec<ExprIR>

options: Arc<JoinOptions>

HStack

Fields

input: Node

exprs: Vec<ExprIR>

schema: SchemaRef

options: ProjectionOptions

Distinct

Fields

input: Node

options: DistinctOptionsIR

MapFunction

Fields

input: Node

function: FunctionIR

Union

Fields

inputs: Vec<Node>

options: UnionOptions

HConcat

Horizontal concatenation

• Invariant: the names will be unique

Fields

inputs: Vec<Node>

schema: SchemaRef

options: HConcatOptions

ExtContext

Fields

input: Node

contexts: Vec<Node>

schema: SchemaRef

Sink

Fields

input: Node

payload: SinkType

Invalid

Implementations

impl IR

pub fn with_exprs_and_input(&self, exprs: Vec<ExprIR>, inputs: Vec<Node>) -> IR

Takes the expressions of an LP node and the inputs of that node and reconstruct

pub fn copy_exprs(&self, container: &mut Vec<ExprIR>)

Copy the exprs in this LP node to an existing container.

pub fn get_exprs(&self) -> Vec<ExprIR>

Get expressions in this node.

pub fn copy_inputs<T>(&self, container: &mut T)

where T: Extend<Node>,

Push inputs of the LP in of this node to an existing container. Most plans have typically one input. A join has two and a scan (CsvScan) or an in-memory DataFrame has none. A Union has multiple.

pub fn get_inputs(&self) -> UnitVec<Node>

pub fn get_inputs_vec(&self) -> Vec<Node>

impl IR

pub fn name(&self) -> &'static str

pub fn input_schema<'a>( &'a self, arena: &'a Arena<IR>, ) -> Option<Cow<'a, SchemaRef>>

pub fn schema<'a>(&'a self, arena: &'a Arena<IR>) -> Cow<'a, SchemaRef>

Get the schema of the logical plan node.

pub fn schema_with_cache<'a>( node: Node, arena: &'a Arena<IR>, cache: &mut PlHashMap<Node, Arc<Schema>>, ) -> Arc<Schema>

Get the schema of the logical plan node, using caching.

Trait Implementations

impl Clone for IR

fn clone(&self) -> IR

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for IR

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for IR

fn default() -> IR

Returns the “default value” for a type.