use std::any::Any;
use std::sync::Arc;
use arrow::datatypes::DataType;
use arrow::datatypes::DataType::Utf8;
use datafusion_common::cast::as_string_array;
use datafusion_common::{internal_err, Result, ScalarValue};
use datafusion_expr::expr::ScalarFunction;
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo};
use datafusion_expr::{lit, ColumnarValue, Expr, Volatility};
use datafusion_expr::{ScalarUDFImpl, Signature};
use crate::string::common::*;
use crate::string::concat;
#[derive(Debug)]
pub struct ConcatFunc {
signature: Signature,
}
impl Default for ConcatFunc {
fn default() -> Self {
ConcatFunc::new()
}
}
impl ConcatFunc {
pub fn new() -> Self {
use DataType::*;
Self {
signature: Signature::variadic(vec![Utf8], Volatility::Immutable),
}
}
}
impl ScalarUDFImpl for ConcatFunc {
fn as_any(&self) -> &dyn Any {
self
}
fn name(&self) -> &str {
"concat"
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
Ok(Utf8)
}
fn invoke(&self, args: &[ColumnarValue]) -> Result<ColumnarValue> {
let array_len = args
.iter()
.filter_map(|x| match x {
ColumnarValue::Array(array) => Some(array.len()),
_ => None,
})
.next();
if array_len.is_none() {
let mut result = String::new();
for arg in args {
if let ColumnarValue::Scalar(ScalarValue::Utf8(Some(v))) = arg {
result.push_str(v);
}
}
return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(result))));
}
let len = array_len.unwrap();
let mut data_size = 0;
let mut columns = Vec::with_capacity(args.len());
for arg in args {
match arg {
ColumnarValue::Scalar(ScalarValue::Utf8(maybe_value)) => {
if let Some(s) = maybe_value {
data_size += s.len() * len;
columns.push(ColumnarValueRef::Scalar(s.as_bytes()));
}
}
ColumnarValue::Array(array) => {
let string_array = as_string_array(array)?;
data_size += string_array.values().len();
let column = if array.is_nullable() {
ColumnarValueRef::NullableArray(string_array)
} else {
ColumnarValueRef::NonNullableArray(string_array)
};
columns.push(column);
}
_ => unreachable!(),
}
}
let mut builder = StringArrayBuilder::with_capacity(len, data_size);
for i in 0..len {
columns
.iter()
.for_each(|column| builder.write::<true>(column, i));
builder.append_offset();
}
Ok(ColumnarValue::Array(Arc::new(builder.finish(None))))
}
fn simplify(
&self,
args: Vec<Expr>,
_info: &dyn SimplifyInfo,
) -> Result<ExprSimplifyResult> {
simplify_concat(args)
}
}
pub fn simplify_concat(args: Vec<Expr>) -> Result<ExprSimplifyResult> {
let mut new_args = Vec::with_capacity(args.len());
let mut contiguous_scalar = "".to_string();
for arg in args.clone() {
match arg {
Expr::Literal(ScalarValue::Utf8(None) | ScalarValue::LargeUtf8(None)) => {}
Expr::Literal(
ScalarValue::Utf8(Some(v)) | ScalarValue::LargeUtf8(Some(v)),
) => contiguous_scalar += &v,
Expr::Literal(x) => {
return internal_err!(
"The scalar {x} should be casted to string type during the type coercion."
)
}
arg => {
if !contiguous_scalar.is_empty() {
new_args.push(lit(contiguous_scalar));
contiguous_scalar = "".to_string();
}
new_args.push(arg);
}
}
}
if !contiguous_scalar.is_empty() {
new_args.push(lit(contiguous_scalar));
}
if !args.eq(&new_args) {
Ok(ExprSimplifyResult::Simplified(Expr::ScalarFunction(
ScalarFunction {
func: concat(),
args: new_args,
},
)))
} else {
Ok(ExprSimplifyResult::Original(args))
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::utils::test::test_function;
use arrow::array::Array;
use arrow::array::{ArrayRef, StringArray};
#[test]
fn test_functions() -> Result<()> {
test_function!(
ConcatFunc::new(),
&[
ColumnarValue::Scalar(ScalarValue::from("aa")),
ColumnarValue::Scalar(ScalarValue::from("bb")),
ColumnarValue::Scalar(ScalarValue::from("cc")),
],
Ok(Some("aabbcc")),
&str,
Utf8,
StringArray
);
test_function!(
ConcatFunc::new(),
&[
ColumnarValue::Scalar(ScalarValue::from("aa")),
ColumnarValue::Scalar(ScalarValue::Utf8(None)),
ColumnarValue::Scalar(ScalarValue::from("cc")),
],
Ok(Some("aacc")),
&str,
Utf8,
StringArray
);
test_function!(
ConcatFunc::new(),
&[ColumnarValue::Scalar(ScalarValue::Utf8(None))],
Ok(Some("")),
&str,
Utf8,
StringArray
);
Ok(())
}
#[test]
fn concat() -> Result<()> {
let c0 =
ColumnarValue::Array(Arc::new(StringArray::from(vec!["foo", "bar", "baz"])));
let c1 = ColumnarValue::Scalar(ScalarValue::Utf8(Some(",".to_string())));
let c2 = ColumnarValue::Array(Arc::new(StringArray::from(vec![
Some("x"),
None,
Some("z"),
])));
let args = &[c0, c1, c2];
let result = ConcatFunc::new().invoke(args)?;
let expected =
Arc::new(StringArray::from(vec!["foo,x", "bar,", "baz,z"])) as ArrayRef;
match &result {
ColumnarValue::Array(array) => {
assert_eq!(&expected, array);
}
_ => panic!(),
}
Ok(())
}
}