use crate::aggregate::utils::down_cast_any_ref;
use crate::expressions::{format_state_name, Literal};
use crate::{AggregateExpr, PhysicalExpr};
use arrow::array::ArrayRef;
use arrow::datatypes::{DataType, Field};
use datafusion_common::cast::as_generic_string_array;
use datafusion_common::{not_impl_err, DataFusionError, Result, ScalarValue};
use datafusion_expr::Accumulator;
use std::any::Any;
use std::sync::Arc;
#[derive(Debug)]
pub struct StringAgg {
name: String,
data_type: DataType,
expr: Arc<dyn PhysicalExpr>,
delimiter: Arc<dyn PhysicalExpr>,
nullable: bool,
}
impl StringAgg {
pub fn new(
expr: Arc<dyn PhysicalExpr>,
delimiter: Arc<dyn PhysicalExpr>,
name: impl Into<String>,
data_type: DataType,
) -> Self {
Self {
name: name.into(),
data_type,
delimiter,
expr,
nullable: true,
}
}
}
impl AggregateExpr for StringAgg {
fn as_any(&self) -> &dyn Any {
self
}
fn field(&self) -> Result<Field> {
Ok(Field::new(
&self.name,
self.data_type.clone(),
self.nullable,
))
}
fn create_accumulator(&self) -> Result<Box<dyn Accumulator>> {
if let Some(delimiter) = self.delimiter.as_any().downcast_ref::<Literal>() {
match delimiter.value() {
ScalarValue::Utf8(Some(delimiter))
| ScalarValue::LargeUtf8(Some(delimiter)) => {
return Ok(Box::new(StringAggAccumulator::new(delimiter)));
}
ScalarValue::Null => {
return Ok(Box::new(StringAggAccumulator::new("")));
}
_ => return not_impl_err!("StringAgg not supported for {}", self.name),
}
}
not_impl_err!("StringAgg not supported for {}", self.name)
}
fn state_fields(&self) -> Result<Vec<Field>> {
Ok(vec![Field::new(
format_state_name(&self.name, "string_agg"),
self.data_type.clone(),
self.nullable,
)])
}
fn expressions(&self) -> Vec<Arc<dyn PhysicalExpr>> {
vec![self.expr.clone(), self.delimiter.clone()]
}
fn name(&self) -> &str {
&self.name
}
}
impl PartialEq<dyn Any> for StringAgg {
fn eq(&self, other: &dyn Any) -> bool {
down_cast_any_ref(other)
.downcast_ref::<Self>()
.map(|x| {
self.name == x.name
&& self.data_type == x.data_type
&& self.expr.eq(&x.expr)
&& self.delimiter.eq(&x.delimiter)
})
.unwrap_or(false)
}
}
#[derive(Debug)]
pub(crate) struct StringAggAccumulator {
values: Option<String>,
delimiter: String,
}
impl StringAggAccumulator {
pub fn new(delimiter: &str) -> Self {
Self {
values: None,
delimiter: delimiter.to_string(),
}
}
}
impl Accumulator for StringAggAccumulator {
fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
let string_array: Vec<_> = as_generic_string_array::<i64>(&values[0])?
.iter()
.filter_map(|v| v.as_ref().map(ToString::to_string))
.collect();
if !string_array.is_empty() {
let s = string_array.join(self.delimiter.as_str());
let v = self.values.get_or_insert("".to_string());
if !v.is_empty() {
v.push_str(self.delimiter.as_str());
}
v.push_str(s.as_str());
}
Ok(())
}
fn merge_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
self.update_batch(values)?;
Ok(())
}
fn state(&self) -> Result<Vec<ScalarValue>> {
Ok(vec![self.evaluate()?])
}
fn evaluate(&self) -> Result<ScalarValue> {
Ok(ScalarValue::LargeUtf8(self.values.clone()))
}
fn size(&self) -> usize {
std::mem::size_of_val(self)
+ self.values.as_ref().map(|v| v.capacity()).unwrap_or(0)
+ self.delimiter.capacity()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::expressions::tests::aggregate;
use crate::expressions::{col, create_aggregate_expr, try_cast};
use arrow::array::ArrayRef;
use arrow::datatypes::*;
use arrow::record_batch::RecordBatch;
use arrow_array::LargeStringArray;
use arrow_array::StringArray;
use datafusion_expr::type_coercion::aggregates::coerce_types;
use datafusion_expr::AggregateFunction;
fn assert_string_aggregate(
array: ArrayRef,
function: AggregateFunction,
distinct: bool,
expected: ScalarValue,
delimiter: String,
) {
let data_type = array.data_type();
let sig = function.signature();
let coerced =
coerce_types(&function, &[data_type.clone(), DataType::Utf8], &sig).unwrap();
let input_schema = Schema::new(vec![Field::new("a", data_type.clone(), true)]);
let batch =
RecordBatch::try_new(Arc::new(input_schema.clone()), vec![array]).unwrap();
let input = try_cast(
col("a", &input_schema).unwrap(),
&input_schema,
coerced[0].clone(),
)
.unwrap();
let delimiter = Arc::new(Literal::new(ScalarValue::from(delimiter)));
let schema = Schema::new(vec![Field::new("a", coerced[0].clone(), true)]);
let agg = create_aggregate_expr(
&function,
distinct,
&[input, delimiter],
&[],
&schema,
"agg",
)
.unwrap();
let result = aggregate(&batch, agg).unwrap();
assert_eq!(expected, result);
}
#[test]
fn string_agg_utf8() {
let a: ArrayRef = Arc::new(StringArray::from(vec!["h", "e", "l", "l", "o"]));
assert_string_aggregate(
a,
AggregateFunction::StringAgg,
false,
ScalarValue::LargeUtf8(Some("h,e,l,l,o".to_owned())),
",".to_owned(),
);
}
#[test]
fn string_agg_largeutf8() {
let a: ArrayRef = Arc::new(LargeStringArray::from(vec!["h", "e", "l", "l", "o"]));
assert_string_aggregate(
a,
AggregateFunction::StringAgg,
false,
ScalarValue::LargeUtf8(Some("h|e|l|l|o".to_owned())),
"|".to_owned(),
);
}
}