datafusion_physical_expr/expressions/
try_cast.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::any::Any;
19use std::fmt;
20use std::hash::Hash;
21use std::sync::Arc;
22
23use crate::PhysicalExpr;
24use arrow::compute;
25use arrow::compute::{cast_with_options, CastOptions};
26use arrow::datatypes::{DataType, Schema};
27use arrow::record_batch::RecordBatch;
28use compute::can_cast_types;
29use datafusion_common::format::DEFAULT_FORMAT_OPTIONS;
30use datafusion_common::{not_impl_err, Result, ScalarValue};
31use datafusion_expr::ColumnarValue;
32
33/// TRY_CAST expression casts an expression to a specific data type and returns NULL on invalid cast
34#[derive(Debug, Eq)]
35pub struct TryCastExpr {
36    /// The expression to cast
37    expr: Arc<dyn PhysicalExpr>,
38    /// The data type to cast to
39    cast_type: DataType,
40}
41
42// Manually derive PartialEq and Hash to work around https://github.com/rust-lang/rust/issues/78808
43impl PartialEq for TryCastExpr {
44    fn eq(&self, other: &Self) -> bool {
45        self.expr.eq(&other.expr) && self.cast_type == other.cast_type
46    }
47}
48
49impl Hash for TryCastExpr {
50    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
51        self.expr.hash(state);
52        self.cast_type.hash(state);
53    }
54}
55
56impl TryCastExpr {
57    /// Create a new CastExpr
58    pub fn new(expr: Arc<dyn PhysicalExpr>, cast_type: DataType) -> Self {
59        Self { expr, cast_type }
60    }
61
62    /// The expression to cast
63    pub fn expr(&self) -> &Arc<dyn PhysicalExpr> {
64        &self.expr
65    }
66
67    /// The data type to cast to
68    pub fn cast_type(&self) -> &DataType {
69        &self.cast_type
70    }
71}
72
73impl fmt::Display for TryCastExpr {
74    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
75        write!(f, "TRY_CAST({} AS {:?})", self.expr, self.cast_type)
76    }
77}
78
79impl PhysicalExpr for TryCastExpr {
80    /// Return a reference to Any that can be used for downcasting
81    fn as_any(&self) -> &dyn Any {
82        self
83    }
84
85    fn data_type(&self, _input_schema: &Schema) -> Result<DataType> {
86        Ok(self.cast_type.clone())
87    }
88
89    fn nullable(&self, _input_schema: &Schema) -> Result<bool> {
90        Ok(true)
91    }
92
93    fn evaluate(&self, batch: &RecordBatch) -> Result<ColumnarValue> {
94        let value = self.expr.evaluate(batch)?;
95        let options = CastOptions {
96            safe: true,
97            format_options: DEFAULT_FORMAT_OPTIONS,
98        };
99        match value {
100            ColumnarValue::Array(array) => {
101                let cast = cast_with_options(&array, &self.cast_type, &options)?;
102                Ok(ColumnarValue::Array(cast))
103            }
104            ColumnarValue::Scalar(scalar) => {
105                let array = scalar.to_array()?;
106                let cast_array = cast_with_options(&array, &self.cast_type, &options)?;
107                let cast_scalar = ScalarValue::try_from_array(&cast_array, 0)?;
108                Ok(ColumnarValue::Scalar(cast_scalar))
109            }
110        }
111    }
112
113    fn children(&self) -> Vec<&Arc<dyn PhysicalExpr>> {
114        vec![&self.expr]
115    }
116
117    fn with_new_children(
118        self: Arc<Self>,
119        children: Vec<Arc<dyn PhysicalExpr>>,
120    ) -> Result<Arc<dyn PhysicalExpr>> {
121        Ok(Arc::new(TryCastExpr::new(
122            Arc::clone(&children[0]),
123            self.cast_type.clone(),
124        )))
125    }
126}
127
128/// Return a PhysicalExpression representing `expr` casted to
129/// `cast_type`, if any casting is needed.
130///
131/// Note that such casts may lose type information
132pub fn try_cast(
133    expr: Arc<dyn PhysicalExpr>,
134    input_schema: &Schema,
135    cast_type: DataType,
136) -> Result<Arc<dyn PhysicalExpr>> {
137    let expr_type = expr.data_type(input_schema)?;
138    if expr_type == cast_type {
139        Ok(Arc::clone(&expr))
140    } else if can_cast_types(&expr_type, &cast_type) {
141        Ok(Arc::new(TryCastExpr::new(expr, cast_type)))
142    } else {
143        not_impl_err!("Unsupported TRY_CAST from {expr_type:?} to {cast_type:?}")
144    }
145}
146
147#[cfg(test)]
148mod tests {
149    use super::*;
150    use crate::expressions::col;
151    use arrow::array::{
152        Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray,
153    };
154    use arrow::{
155        array::{
156            Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
157            Int8Array, TimestampNanosecondArray, UInt32Array,
158        },
159        datatypes::*,
160    };
161
162    // runs an end-to-end test of physical type cast
163    // 1. construct a record batch with a column "a" of type A
164    // 2. construct a physical expression of TRY_CAST(a AS B)
165    // 3. evaluate the expression
166    // 4. verify that the resulting expression is of type B
167    // 5. verify that the resulting values are downcastable and correct
168    macro_rules! generic_decimal_to_other_test_cast {
169        ($DECIMAL_ARRAY:ident, $A_TYPE:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
170            let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
171            let batch = RecordBatch::try_new(
172                Arc::new(schema.clone()),
173                vec![Arc::new($DECIMAL_ARRAY)],
174            )?;
175            // verify that we can construct the expression
176            let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?;
177
178            // verify that its display is correct
179            assert_eq!(
180                format!("TRY_CAST(a@0 AS {:?})", $TYPE),
181                format!("{}", expression)
182            );
183
184            // verify that the expression's type is correct
185            assert_eq!(expression.data_type(&schema)?, $TYPE);
186
187            // compute
188            let result = expression
189                .evaluate(&batch)?
190                .into_array(batch.num_rows())
191                .expect("Failed to convert to array");
192
193            // verify that the array's data_type is correct
194            assert_eq!(*result.data_type(), $TYPE);
195
196            // verify that the data itself is downcastable
197            let result = result
198                .as_any()
199                .downcast_ref::<$TYPEARRAY>()
200                .expect("failed to downcast");
201
202            // verify that the result itself is correct
203            for (i, x) in $VEC.iter().enumerate() {
204                match x {
205                    Some(x) => assert_eq!(result.value(i), *x),
206                    None => assert!(!result.is_valid(i)),
207                }
208            }
209        }};
210    }
211
212    // runs an end-to-end test of physical type cast
213    // 1. construct a record batch with a column "a" of type A
214    // 2. construct a physical expression of TRY_CAST(a AS B)
215    // 3. evaluate the expression
216    // 4. verify that the resulting expression is of type B
217    // 5. verify that the resulting values are downcastable and correct
218    macro_rules! generic_test_cast {
219        ($A_ARRAY:ident, $A_TYPE:expr, $A_VEC:expr, $TYPEARRAY:ident, $TYPE:expr, $VEC:expr) => {{
220            let schema = Schema::new(vec![Field::new("a", $A_TYPE, true)]);
221            let a_vec_len = $A_VEC.len();
222            let a = $A_ARRAY::from($A_VEC);
223            let batch =
224                RecordBatch::try_new(Arc::new(schema.clone()), vec![Arc::new(a)])?;
225
226            // verify that we can construct the expression
227            let expression = try_cast(col("a", &schema)?, &schema, $TYPE)?;
228
229            // verify that its display is correct
230            assert_eq!(
231                format!("TRY_CAST(a@0 AS {:?})", $TYPE),
232                format!("{}", expression)
233            );
234
235            // verify that the expression's type is correct
236            assert_eq!(expression.data_type(&schema)?, $TYPE);
237
238            // compute
239            let result = expression
240                .evaluate(&batch)?
241                .into_array(batch.num_rows())
242                .expect("Failed to convert to array");
243
244            // verify that the array's data_type is correct
245            assert_eq!(*result.data_type(), $TYPE);
246
247            // verify that the len is correct
248            assert_eq!(result.len(), a_vec_len);
249
250            // verify that the data itself is downcastable
251            let result = result
252                .as_any()
253                .downcast_ref::<$TYPEARRAY>()
254                .expect("failed to downcast");
255
256            // verify that the result itself is correct
257            for (i, x) in $VEC.iter().enumerate() {
258                match x {
259                    Some(x) => assert_eq!(result.value(i), *x),
260                    None => assert!(!result.is_valid(i)),
261                }
262            }
263        }};
264    }
265
266    #[test]
267    fn test_try_cast_decimal_to_decimal() -> Result<()> {
268        // try cast one decimal data type to another decimal data type
269        let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
270        let decimal_array = create_decimal_array(&array, 10, 3);
271        generic_decimal_to_other_test_cast!(
272            decimal_array,
273            DataType::Decimal128(10, 3),
274            Decimal128Array,
275            DataType::Decimal128(20, 6),
276            [
277                Some(1_234_000),
278                Some(2_222_000),
279                Some(3_000),
280                Some(4_000_000),
281                Some(5_000_000),
282                None
283            ]
284        );
285
286        let decimal_array = create_decimal_array(&array, 10, 3);
287        generic_decimal_to_other_test_cast!(
288            decimal_array,
289            DataType::Decimal128(10, 3),
290            Decimal128Array,
291            DataType::Decimal128(10, 2),
292            [Some(123), Some(222), Some(0), Some(400), Some(500), None]
293        );
294
295        Ok(())
296    }
297
298    #[test]
299    fn test_try_cast_decimal_to_numeric() -> Result<()> {
300        // TODO we should add function to create Decimal128Array with value and metadata
301        // https://github.com/apache/arrow-rs/issues/1009
302        let array: Vec<i128> = vec![1, 2, 3, 4, 5];
303        let decimal_array = create_decimal_array(&array, 10, 0);
304        // decimal to i8
305        generic_decimal_to_other_test_cast!(
306            decimal_array,
307            DataType::Decimal128(10, 0),
308            Int8Array,
309            DataType::Int8,
310            [
311                Some(1_i8),
312                Some(2_i8),
313                Some(3_i8),
314                Some(4_i8),
315                Some(5_i8),
316                None
317            ]
318        );
319
320        // decimal to i16
321        let decimal_array = create_decimal_array(&array, 10, 0);
322        generic_decimal_to_other_test_cast!(
323            decimal_array,
324            DataType::Decimal128(10, 0),
325            Int16Array,
326            DataType::Int16,
327            [
328                Some(1_i16),
329                Some(2_i16),
330                Some(3_i16),
331                Some(4_i16),
332                Some(5_i16),
333                None
334            ]
335        );
336
337        // decimal to i32
338        let decimal_array = create_decimal_array(&array, 10, 0);
339        generic_decimal_to_other_test_cast!(
340            decimal_array,
341            DataType::Decimal128(10, 0),
342            Int32Array,
343            DataType::Int32,
344            [
345                Some(1_i32),
346                Some(2_i32),
347                Some(3_i32),
348                Some(4_i32),
349                Some(5_i32),
350                None
351            ]
352        );
353
354        // decimal to i64
355        let decimal_array = create_decimal_array(&array, 10, 0);
356        generic_decimal_to_other_test_cast!(
357            decimal_array,
358            DataType::Decimal128(10, 0),
359            Int64Array,
360            DataType::Int64,
361            [
362                Some(1_i64),
363                Some(2_i64),
364                Some(3_i64),
365                Some(4_i64),
366                Some(5_i64),
367                None
368            ]
369        );
370
371        // decimal to float32
372        let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
373        let decimal_array = create_decimal_array(&array, 10, 3);
374        generic_decimal_to_other_test_cast!(
375            decimal_array,
376            DataType::Decimal128(10, 3),
377            Float32Array,
378            DataType::Float32,
379            [
380                Some(1.234_f32),
381                Some(2.222_f32),
382                Some(0.003_f32),
383                Some(4.0_f32),
384                Some(5.0_f32),
385                None
386            ]
387        );
388        // decimal to float64
389        let decimal_array = create_decimal_array(&array, 20, 6);
390        generic_decimal_to_other_test_cast!(
391            decimal_array,
392            DataType::Decimal128(20, 6),
393            Float64Array,
394            DataType::Float64,
395            [
396                Some(0.001234_f64),
397                Some(0.002222_f64),
398                Some(0.000003_f64),
399                Some(0.004_f64),
400                Some(0.005_f64),
401                None
402            ]
403        );
404
405        Ok(())
406    }
407
408    #[test]
409    fn test_try_cast_numeric_to_decimal() -> Result<()> {
410        // int8
411        generic_test_cast!(
412            Int8Array,
413            DataType::Int8,
414            vec![1, 2, 3, 4, 5],
415            Decimal128Array,
416            DataType::Decimal128(3, 0),
417            [Some(1), Some(2), Some(3), Some(4), Some(5)]
418        );
419
420        // int16
421        generic_test_cast!(
422            Int16Array,
423            DataType::Int16,
424            vec![1, 2, 3, 4, 5],
425            Decimal128Array,
426            DataType::Decimal128(5, 0),
427            [Some(1), Some(2), Some(3), Some(4), Some(5)]
428        );
429
430        // int32
431        generic_test_cast!(
432            Int32Array,
433            DataType::Int32,
434            vec![1, 2, 3, 4, 5],
435            Decimal128Array,
436            DataType::Decimal128(10, 0),
437            [Some(1), Some(2), Some(3), Some(4), Some(5)]
438        );
439
440        // int64
441        generic_test_cast!(
442            Int64Array,
443            DataType::Int64,
444            vec![1, 2, 3, 4, 5],
445            Decimal128Array,
446            DataType::Decimal128(20, 0),
447            [Some(1), Some(2), Some(3), Some(4), Some(5)]
448        );
449
450        // int64 to different scale
451        generic_test_cast!(
452            Int64Array,
453            DataType::Int64,
454            vec![1, 2, 3, 4, 5],
455            Decimal128Array,
456            DataType::Decimal128(20, 2),
457            [Some(100), Some(200), Some(300), Some(400), Some(500)]
458        );
459
460        // float32
461        generic_test_cast!(
462            Float32Array,
463            DataType::Float32,
464            vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
465            Decimal128Array,
466            DataType::Decimal128(10, 2),
467            [Some(150), Some(250), Some(300), Some(112), Some(550)]
468        );
469
470        // float64
471        generic_test_cast!(
472            Float64Array,
473            DataType::Float64,
474            vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
475            Decimal128Array,
476            DataType::Decimal128(20, 4),
477            [
478                Some(15000),
479                Some(25000),
480                Some(30000),
481                Some(11235),
482                Some(55000)
483            ]
484        );
485        Ok(())
486    }
487
488    #[test]
489    fn test_cast_i32_u32() -> Result<()> {
490        generic_test_cast!(
491            Int32Array,
492            DataType::Int32,
493            vec![1, 2, 3, 4, 5],
494            UInt32Array,
495            DataType::UInt32,
496            [
497                Some(1_u32),
498                Some(2_u32),
499                Some(3_u32),
500                Some(4_u32),
501                Some(5_u32)
502            ]
503        );
504        Ok(())
505    }
506
507    #[test]
508    fn test_cast_i32_utf8() -> Result<()> {
509        generic_test_cast!(
510            Int32Array,
511            DataType::Int32,
512            vec![1, 2, 3, 4, 5],
513            StringArray,
514            DataType::Utf8,
515            [Some("1"), Some("2"), Some("3"), Some("4"), Some("5")]
516        );
517        Ok(())
518    }
519
520    #[test]
521    fn test_try_cast_utf8_i32() -> Result<()> {
522        generic_test_cast!(
523            StringArray,
524            DataType::Utf8,
525            vec!["a", "2", "3", "b", "5"],
526            Int32Array,
527            DataType::Int32,
528            [None, Some(2), Some(3), None, Some(5)]
529        );
530        Ok(())
531    }
532
533    #[test]
534    fn test_cast_i64_t64() -> Result<()> {
535        let original = vec![1, 2, 3, 4, 5];
536        let expected: Vec<Option<i64>> = original
537            .iter()
538            .map(|i| Some(Time64NanosecondArray::from(vec![*i]).value(0)))
539            .collect();
540        generic_test_cast!(
541            Int64Array,
542            DataType::Int64,
543            original,
544            TimestampNanosecondArray,
545            DataType::Timestamp(TimeUnit::Nanosecond, None),
546            expected
547        );
548        Ok(())
549    }
550
551    #[test]
552    fn invalid_cast() {
553        // Ensure a useful error happens at plan time if invalid casts are used
554        let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
555
556        let result = try_cast(
557            col("a", &schema).unwrap(),
558            &schema,
559            DataType::Interval(IntervalUnit::MonthDayNano),
560        );
561        result.expect_err("expected Invalid TRY_CAST");
562    }
563
564    // create decimal array with the specified precision and scale
565    fn create_decimal_array(array: &[i128], precision: u8, scale: i8) -> Decimal128Array {
566        let mut decimal_builder = Decimal128Builder::with_capacity(array.len());
567        for value in array {
568            decimal_builder.append_value(*value);
569        }
570        decimal_builder.append_null();
571        decimal_builder
572            .finish()
573            .with_precision_and_scale(precision, scale)
574            .unwrap()
575    }
576}
datafusion_physical_expr/expressions/try_cast.rs

datafusion_physical_expr/expressions/
try_cast.rs
