Struct DFSchema

pub struct DFSchema { /* private fields */ }

DFSchema wraps an Arrow schema and adds relation names.

The schema may hold the fields across multiple tables. Some fields may be qualified and some unqualified. A qualified field is a field that has a relation name associated with it.

Unqualified fields must be unique not only amongst themselves, but also must have a distinct name from any qualified field names. This allows finding a qualified field by name to be possible, so long as there aren’t multiple qualified fields with the same name.

There is an alias to Arc<DFSchema> named DFSchemaRef.

Creating qualified schemas

Use DFSchema::try_from_qualified_schema to create a qualified schema from an Arrow schema.

use datafusion_common::{DFSchema, Column};
use arrow_schema::{DataType, Field, Schema};

let arrow_schema = Schema::new(vec![
   Field::new("c1", DataType::Int32, false),
]);

let df_schema = DFSchema::try_from_qualified_schema("t1", &arrow_schema).unwrap();
let column = Column::from_qualified_name("t1.c1");
assert!(df_schema.has_column(&column));

// Can also access qualified fields with unqualified name, if it's unambiguous
let column = Column::from_qualified_name("c1");
assert!(df_schema.has_column(&column));

Creating unqualified schemas

Create an unqualified schema using TryFrom:

use datafusion_common::{DFSchema, Column};
use arrow_schema::{DataType, Field, Schema};

let arrow_schema = Schema::new(vec![
   Field::new("c1", DataType::Int32, false),
]);

let df_schema = DFSchema::try_from(arrow_schema).unwrap();
let column = Column::new_unqualified("c1");
assert!(df_schema.has_column(&column));

Converting back to Arrow schema

Use the Into trait to convert DFSchema into an Arrow schema:

use datafusion_common::DFSchema;
use arrow_schema::Schema;
use arrow::datatypes::Field;
use std::collections::HashMap;

let df_schema = DFSchema::from_unqualified_fields(vec![
   Field::new("c1", arrow::datatypes::DataType::Int32, false),
].into(),HashMap::new()).unwrap();
let schema = Schema::from(df_schema);
assert_eq!(schema.fields().len(), 1);

Implementations

impl DFSchema

pub fn empty() -> Self

Creates an empty DFSchema

pub fn as_arrow(&self) -> &Schema

Return a reference to the inner Arrow Schema

Note this does not have the qualifier information

pub fn inner(&self) -> &SchemaRef

Return a reference to the inner Arrow SchemaRef

Note this does not have the qualifier information

pub fn new_with_metadata( qualified_fields: Vec<(Option<TableReference>, Arc<Field>)>, metadata: HashMap<String, String>, ) -> Result<Self>

Create a DFSchema from an Arrow schema where all the fields have a given qualifier

pub fn from_unqualified_fields( fields: Fields, metadata: HashMap<String, String>, ) -> Result<Self>

Create a new DFSchema from a list of Arrow Fields

pub fn from_unqualifed_fields( fields: Fields, metadata: HashMap<String, String>, ) -> Result<Self>

👎Deprecated since 40.0.0: Please use from_unqualified_fields instead (this one’s name is a typo). This method is subject to be removed soon

Create a new DFSchema from a list of Arrow Fields

pub fn try_from_qualified_schema( qualifier: impl Into<TableReference>, schema: &Schema, ) -> Result<Self>

Create a DFSchema from an Arrow schema and a given qualifier

To create a schema from an Arrow schema without a qualifier, use DFSchema::try_from.

pub fn from_field_specific_qualified_schema( qualifiers: Vec<Option<TableReference>>, schema: &SchemaRef, ) -> Result<Self>

Create a DFSchema from an Arrow schema where all the fields have a given qualifier

pub fn check_names(&self) -> Result<()>

Check if the schema have some fields with the same name

pub fn with_functional_dependencies( self, functional_dependencies: FunctionalDependencies, ) -> Result<Self>

Assigns functional dependencies.

pub fn join(&self, schema: &DFSchema) -> Result<Self>

Create a new schema that contains the fields from this schema followed by the fields from the supplied schema. An error will be returned if there are duplicate field names.

pub fn merge(&mut self, other_schema: &DFSchema)

Modify this schema by appending the fields from the supplied schema, ignoring any duplicate fields.

pub fn fields(&self) -> &Fields

Get a list of fields

pub fn field(&self, i: usize) -> &Field

Returns an immutable reference of a specific Field instance selected using an offset within the internal fields vector

pub fn qualified_field(&self, i: usize) -> (Option<&TableReference>, &Field)

Returns an immutable reference of a specific Field instance selected using an offset within the internal fields vector and its qualifier

pub fn index_of_column_by_name( &self, qualifier: Option<&TableReference>, name: &str, ) -> Option<usize>

pub fn maybe_index_of_column(&self, col: &Column) -> Option<usize>

Find the index of the column with the given qualifier and name, returning None if not found

See Self::index_of_column for a version that returns an error if the column is not found

pub fn index_of_column(&self, col: &Column) -> Result<usize>

Find the index of the column with the given qualifier and name, returning Err if not found

See Self::maybe_index_of_column for a version that returns None if the column is not found

pub fn is_column_from_schema(&self, col: &Column) -> bool

Check if the column is in the current schema

pub fn field_with_name( &self, qualifier: Option<&TableReference>, name: &str, ) -> Result<&Field>

Find the field with the given name

pub fn qualified_field_with_name( &self, qualifier: Option<&TableReference>, name: &str, ) -> Result<(Option<&TableReference>, &Field)>

Find the qualified field with the given name

pub fn fields_with_qualified(&self, qualifier: &TableReference) -> Vec<&Field>

Find all fields having the given qualifier

pub fn fields_indices_with_qualified( &self, qualifier: &TableReference, ) -> Vec<usize>

Find all fields indices having the given qualifier

pub fn fields_with_unqualified_name(&self, name: &str) -> Vec<&Field>

Find all fields that match the given name

pub fn qualified_fields_with_unqualified_name( &self, name: &str, ) -> Vec<(Option<&TableReference>, &Field)>

Find all fields that match the given name and return them with their qualifier

pub fn columns_with_unqualified_name(&self, name: &str) -> Vec<Column>

Find all fields that match the given name and convert to column

pub fn columns(&self) -> Vec<Column>

Return all Columns for the schema

pub fn qualified_field_with_unqualified_name( &self, name: &str, ) -> Result<(Option<&TableReference>, &Field)>

Find the qualified field with the given unqualified name

pub fn field_with_unqualified_name(&self, name: &str) -> Result<&Field>

Find the field with the given name

pub fn field_with_qualified_name( &self, qualifier: &TableReference, name: &str, ) -> Result<&Field>

Find the field with the given qualified name

pub fn field_from_column(&self, column: &Column) -> Result<&Field>

Find the field with the given qualified column

pub fn qualified_field_from_column( &self, column: &Column, ) -> Result<(Option<&TableReference>, &Field)>

Find the field with the given qualified column

pub fn has_column_with_unqualified_name(&self, name: &str) -> bool

Find if the field exists with the given name

pub fn has_column_with_qualified_name( &self, qualifier: &TableReference, name: &str, ) -> bool

Find if the field exists with the given qualified name

pub fn has_column(&self, column: &Column) -> bool

Find if the field exists with the given qualified column

pub fn matches_arrow_schema(&self, arrow_schema: &Schema) -> bool

Check to see if unqualified field names matches field names in Arrow schema

pub fn check_arrow_schema_type_compatible( &self, arrow_schema: &Schema, ) -> Result<()>

Check to see if fields in 2 Arrow schemas are compatible

pub fn logically_equivalent_names_and_types(&self, other: &Self) -> bool

Returns true if the two schemas have the same qualified named fields with logically equivalent data types. Returns false otherwise.

Use DFSchema::equivalent_names_and_types for stricter semantic type equivalence checking.

pub fn equivalent_names_and_types(&self, other: &Self) -> bool

Returns true if the two schemas have the same qualified named fields with the same data types. Returns false otherwise.

This is a specialized version of Eq that ignores differences in nullability and metadata.

Use DFSchema::logically_equivalent_names_and_types for a weaker logical type checking, which for example would consider a dictionary encoded UTF8 array to be equivalent to a plain UTF8 array.

pub fn datatype_is_logically_equal(dt1: &DataType, dt2: &DataType) -> bool

Checks if two DataTypes are logically equal. This is a notably weaker constraint than datatype_is_semantically_equal in that a Dictionary<K,V> type is logically equal to a plain V type, but not semantically equal. Dictionary<K1, V1> is also logically equal to Dictionary<K2, V1>.

pub fn strip_qualifiers(self) -> Self

Strip all field qualifier in schema

pub fn replace_qualifier(self, qualifier: impl Into<TableReference>) -> Self

Replace all field qualifier with new value in schema

pub fn field_names(&self) -> Vec<String>

Get list of fully-qualified field names in this schema

pub fn metadata(&self) -> &HashMap<String, String>

Get metadata of this schema

pub fn functional_dependencies(&self) -> &FunctionalDependencies

Get functional dependencies

pub fn iter(&self) -> impl Iterator<Item = (Option<&TableReference>, &FieldRef)>

Iterate over the qualifiers and fields in the DFSchema

Trait Implementations

impl AsRef<Arc<Schema>> for DFSchema

Allow DFSchema to be converted into an Arrow &SchemaRef (to clone, for example)

fn as_ref(&self) -> &SchemaRef

Converts this type into a shared reference of the (usually inferred) input type.

impl AsRef<Schema> for DFSchema

Allow DFSchema to be converted into an Arrow &Schema

fn as_ref(&self) -> &Schema

Converts this type into a shared reference of the (usually inferred) input type.

impl Clone for DFSchema

fn clone(&self) -> DFSchema

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for DFSchema

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Display for DFSchema

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl ExprSchema for DFSchema

fn nullable(&self, col: &Column) -> Result<bool>

Is this column reference nullable?

fn data_type(&self, col: &Column) -> Result<&DataType>

What is the datatype of this column?

fn metadata(&self, col: &Column) -> Result<&HashMap<String, String>>

Returns the column’s optional metadata.

fn data_type_and_nullable(&self, col: &Column) -> Result<(&DataType, bool)>

Return the coulmn’s datatype and nullability

impl From<&DFSchema> for Schema

fn from(df_schema: &DFSchema) -> Self

Convert DFSchema reference into a Schema

impl From<DFSchema> for SchemaRef

fn from(df_schema: DFSchema) -> Self

Converts to this type from the input type.

impl From<DFSchema> for Schema

fn from(df_schema: DFSchema) -> Self

Convert DFSchema into a Schema

impl Hash for DFSchema

fn hash<H: Hasher>(&self, state: &mut H)

Feeds this value into the given Hasher.

fn hash_slice<H>(data: &[Self], state: &mut H)

where H: Hasher, Self: Sized,

Feeds a slice of this type into the given Hasher.

impl PartialEq for DFSchema

fn eq(&self, other: &DFSchema) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl TryFrom<Arc<Schema>> for DFSchema

type Error = DataFusionError

The type returned in the event of a conversion error.

fn try_from(schema: SchemaRef) -> Result<Self, Self::Error>

Performs the conversion.

impl TryFrom<Schema> for DFSchema

Create a DFSchema from an Arrow schema

type Error = DataFusionError

The type returned in the event of a conversion error.

fn try_from(schema: Schema) -> Result<Self, Self::Error>

Performs the conversion.

impl Eq for DFSchema

impl StructuralPartialEq for DFSchema