[feature](array-func)support array_match_all/any

be/src/vec/functions/array/function_array_register.cpp

@@ -57,6 +57,7 @@ void register_function_array_count(SimpleFunctionFactory&);
 void register_function_array_filter_function(SimpleFunctionFactory&);
 void register_function_array_splits(SimpleFunctionFactory&);
 void register_function_array_contains_all(SimpleFunctionFactory&);
+void register_function_array_match(SimpleFunctionFactory&);
 
 void register_function_array(SimpleFunctionFactory& factory) {
     register_function_array_shuffle(factory);
@@ -94,6 +95,7 @@ void register_function_array(SimpleFunctionFactory& factory) {
     register_function_array_filter_function(factory);
     register_function_array_splits(factory);
     register_function_array_contains_all(factory);
+    register_function_array_match(factory);
 }
 
 } // namespace doris::vectorized

be/src/vec/functions/array/varray_match_function.cpp

@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <vec/functions/simple_function_factory.h>
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/status.h"
+#include "vec/aggregate_functions/aggregate_function.h"
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_vector.h"
+#include "vec/columns/columns_number.h"
+#include "vec/common/assert_cast.h"
+#include "vec/core/block.h"
+#include "vec/core/column_numbers.h"
+#include "vec/core/column_with_type_and_name.h"
+#include "vec/utils/util.hpp"
+
+namespace doris::vectorized {
+
+///* bool array_match_all/any(array<boolean>) *///
+template <bool MATCH_ALL>
+class ArrayMatchFunction : public IFunction {
+public:
+    static constexpr auto name = MATCH_ALL ? "array_match_all" : "array_match_any";
+    static FunctionPtr create() { return std::make_shared<ArrayMatchFunction>(); }
+
+    std::string get_name() const override { return name; }
+
+    bool is_variadic() const override { return false; }
+
+    size_t get_number_of_arguments() const override { return 1; }
+
+    bool is_use_default_implementation_for_constants() const override { return false; }
+
+    bool use_default_implementation_for_nulls() const override { return false; }
+
+    DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
+        return make_nullable(std::make_shared<DataTypeUInt8>());
+    }
+
+    Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
+                        size_t result, size_t input_rows_count) const override {
+        // here is executed by array_map filtered and arg[0] is bool result column
+        const auto& [src_column, src_const] =
+                unpack_if_const(block.get_by_position(arguments[0]).column);
+        const ColumnArray* array_column = nullptr;
+        const UInt8* array_null_map = nullptr;
+        if (src_column->is_nullable()) {
+            auto nullable_array = assert_cast<const ColumnNullable*>(src_column.get());
+            array_column = assert_cast<const ColumnArray*>(&nullable_array->get_nested_column());
+            array_null_map = nullable_array->get_null_map_column().get_data().data();
+        } else {
+            array_column = assert_cast<const ColumnArray*>(src_column.get());
+        }
+
+        if (!array_column) {
+            return Status::RuntimeError("unsupported types for function {}({})", get_name(),
+                                        block.get_by_position(arguments[0]).type->get_name());
+        }
+
+        const auto& offsets = array_column->get_offsets();
+        ColumnPtr nested_column = nullptr;
+        const UInt8* nested_null_map = nullptr;
+        if (array_column->get_data().is_nullable()) {
+            const auto& nested_null_column =
+                    assert_cast<const ColumnNullable&>(array_column->get_data());
+            nested_null_map = nested_null_column.get_null_map_column().get_data().data();
+            nested_column = nested_null_column.get_nested_column_ptr();
+        } else {
+            nested_column = array_column->get_data_ptr();
+        }
+
+        if (!nested_column) {
+            return Status::RuntimeError("unsupported types for function {}({})", get_name(),
+                                        block.get_by_position(arguments[0]).type->get_name());
+        }
+
+        const auto& nested_data = assert_cast<const ColumnUInt8&>(*nested_column).get_data();
+
+        // result is nullable bool column for every array column
+        auto result_data_column = ColumnUInt8::create(input_rows_count, 1);
+        auto result_null_column = ColumnUInt8::create(input_rows_count, 0);
+
+        // iterate over all arrays with bool elements
+        for (int row = 0; row < input_rows_count; ++row) {
+            if (array_null_map && array_null_map[row]) {
+                // current array is null, this is always null
+                result_null_column->get_data()[row] = 1;
+                result_data_column->get_data()[row] = 0;
+            } else {
+                // we should calculate the bool result for current array
+                // has_null in current array
+                bool has_null_elem = false;
+                // res for current array
+                bool res_for_array = MATCH_ALL;
+                for (auto off = offsets[row - 1]; off < offsets[row]; ++off) {
+                    if (nested_null_map && nested_null_map[off]) {
+                        has_null_elem = true;
+                    } else {
+                        if (nested_data[off] != MATCH_ALL) { // not match
+                            res_for_array = !MATCH_ALL;
+                            break;
+                        } // default is MATCH_ALL
+                    }
+                }
+                result_null_column->get_data()[row] = has_null_elem && res_for_array == MATCH_ALL;
+                result_data_column->get_data()[row] = res_for_array;
+            }
+        }
+
+        // insert the result column to block
+        DCHECK(block.get_by_position(result).type->is_nullable());
+        ColumnPtr dst_column = ColumnNullable::create(std::move(result_data_column),
+                                                      std::move(result_null_column));
+        block.replace_by_position(result, std::move(dst_column));
+        return Status::OK();
+    }
+};
+
+void register_function_array_match(SimpleFunctionFactory& factory) {
+    factory.register_function<ArrayMatchFunction<true>>(); // MATCH_ALL = true means array_match_all
+    factory.register_function<
+            ArrayMatchFunction<false>>(); // MATCH_ALL = false means array_match_any
+}
+} // namespace doris::vectorized

fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java

@@ -46,8 +46,8 @@ public class LambdaFunctionCallExpr extends FunctionCallExpr {
     public static final ImmutableSet<String> LAMBDA_MAPPED_FUNCTION_SET = new ImmutableSortedSet.Builder(
             String.CASE_INSENSITIVE_ORDER).add("array_exists").add("array_sortby")
             .add("array_first_index").add("array_last_index").add("array_first").add("array_last").add("array_count")
-            .add("element_at").add("array_split").add("array_reverse_split")
-            .build();
+            .add("element_at").add("array_split").add("array_reverse_split").add("array_match_any")
+            .add("array_match_all").build();
 
     private static final Logger LOG = LogManager.getLogger(LambdaFunctionCallExpr.class);
 

fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java

@@ -49,6 +49,8 @@
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLast;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAll;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAny;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMax;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMin;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayPopBack;
@@ -506,6 +508,8 @@ public class BuiltinScalarFunctions implements FunctionHelper {
             scalar(ArrayLast.class, "array_last"),
             scalar(ArrayLastIndex.class, "array_last_index"),
             scalar(ArrayMap.class, "array_map"),
+            scalar(ArrayMatchAll.class, "array_match_all"),
+            scalar(ArrayMatchAny.class, "array_match_any"),
             scalar(ArrayMax.class, "array_max"),
             scalar(ArrayMin.class, "array_min"),
             scalar(ArrayPopBack.class, "array_popback"),

fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java

@@ -88,7 +88,6 @@
 import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.HighOrderFunction;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction;
 import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf;
@@ -507,11 +506,7 @@ public Expr visitScalarFunction(ScalarFunction function, PlanTranslatorContext c
 
         FunctionCallExpr functionCallExpr;
         // create catalog FunctionCallExpr without analyze again
-        if (function instanceof HighOrderFunction) {
-            functionCallExpr = new LambdaFunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
-        } else {
-            functionCallExpr = new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
-        }
+        functionCallExpr = new FunctionCallExpr(catalogFunction, new FunctionParams(false, arguments));
         functionCallExpr.setNullableFromNereids(function.nullable());
         return functionCallExpr;
     }

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMatchAll.java

@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.scalar;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.BooleanType;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * ScalarFunction 'array_match_all'.
+ */
+public class ArrayMatchAll extends ScalarFunction
+        implements HighOrderFunction, AlwaysNullable {
+
+    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BooleanType.INSTANCE).args(ArrayType.of(BooleanType.INSTANCE))
+    );
+
+    /**
+     * constructor with arguments.
+     * array_match_all(lambda, a1, ...) = array_match(a1, array_map(lambda, a1, ...))
+     */
+    public ArrayMatchAll(Expression arg) {
+        super("array_match_all", arg instanceof Lambda ? new ArrayMap(arg) : arg);
+    }
+
+    @Override
+    public ArrayMatchAll withChildren(List<Expression> children) {
+        if (children.size() != 1) {
+            throw new AnalysisException(
+                    String.format("The number of args of %s must be 1 but is %d", getName(), children.size()));
+        }
+        return new ArrayMatchAll(children.get(0));
+    }
+
+    @Override
+    public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+        return visitor.visitArrayMatchAll(this, context);
+    }
+
+    @Override
+    public List<FunctionSignature> getImplSignature() {
+        return SIGNATURES;
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMatchAny.java

@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.scalar;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.ArrayType;
+import org.apache.doris.nereids.types.BooleanType;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * ScalarFunction 'array_match_any'.
+ */
+public class ArrayMatchAny extends ScalarFunction
+        implements HighOrderFunction, AlwaysNullable {
+
+    public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
+            FunctionSignature.ret(BooleanType.INSTANCE).args(ArrayType.of(BooleanType.INSTANCE))
+    );
+
+    /**
+     * constructor with arguments.
+     * array_match_any(lambda, a1, ...) = array_match_any(a1, array_map(lambda, a1, ...))
+     */
+    public ArrayMatchAny(Expression arg) {
+        super("array_match_any", arg instanceof Lambda ? new ArrayMap(arg) : arg);
+    }
+
+    @Override
+    public ArrayMatchAny withChildren(List<Expression> children) {
+        if (children.size() != 1) {
+            throw new AnalysisException(
+                    String.format("The number of args of %s must be 1 but is %d", getName(), children.size()));
+        }
+        return new ArrayMatchAny(children.get(0));
+    }
+
+    @Override
+    public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+        return visitor.visitArrayMatchAny(this, context);
+    }
+
+    @Override
+    public List<FunctionSignature> getImplSignature() {
+        return SIGNATURES;
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java

@@ -49,6 +49,8 @@
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayJoin;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayLastIndex;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMap;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAll;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMatchAny;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMax;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayMin;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ArrayPopBack;
@@ -672,6 +674,14 @@ default R visitArrayMap(ArrayMap arraySort, C context) {
         return visitScalarFunction(arraySort, context);
     }
 
+    default R visitArrayMatchAll(ArrayMatchAll arrayMatchAll, C context) {
+        return visitScalarFunction(arrayMatchAll, context);
+    }
+
+    default R visitArrayMatchAny(ArrayMatchAny arrayMatchAny, C context) {
+        return visitScalarFunction(arrayMatchAny, context);
+    }
+
     default R visitArrayRepeat(ArrayRepeat arrayRepeat, C context) {
         return visitScalarFunction(arrayRepeat, context);
     }