ARROW-13568: [Gandiva] Support null data type for gandiva.

c_glib/test/gandiva/test-null-literal-node.rb

@@ -22,11 +22,8 @@ def setup
 
   def test_invalid_type
     return_type = Arrow::NullDataType.new
-    message =
-      "[gandiva][null-literal-node][new] " +
-      "failed to create: <#{return_type}>"
-    assert_raise(Arrow::Error::Invalid.new(message)) do
-      Gandiva::NullLiteralNode.new(return_type)
+    literal_node = Gandiva::NullLiteralNode.new(return_type)
+    assert_equal(return_type, literal_node.return_type)
     end
   end
 

cpp/src/gandiva/CMakeLists.txt

@@ -69,6 +69,7 @@ set(SRC_FILES
     expression_registry.cc
     exported_funcs_registry.cc
     filter.cc
+    null_ops.cc
     function_ir_builder.cc
     function_registry.cc
     function_registry_arithmetic.cc
@@ -236,6 +237,7 @@ add_gandiva_test(internals-test
                  random_generator_holder_test.cc
                  hash_utils_test.cc
                  gdv_function_stubs_test.cc
+                 null_ops_test.cc
                  EXTRA_DEPENDENCIES
                  LLVM::LLVM_INTERFACE
                  ${GANDIVA_OPENSSL_LIBS}

cpp/src/gandiva/annotator.cc

@@ -77,13 +77,21 @@ void Annotator::PrepareBuffersForField(const FieldDescriptor& desc,
     ++buffer_idx;
   }
 
-  uint8_t* data_buf = const_cast<uint8_t*>(array_data.buffers[buffer_idx]->data());
-  eval_batch->SetBuffer(desc.data_idx(), data_buf, array_data.offset);
+  if (array_data.type->id() == arrow::Type::NA) {
+    eval_batch->SetBuffer(desc.data_idx(), nullptr, array_data.offset);
+  } else {
+    uint8_t* data_buf = const_cast<uint8_t*>(array_data.buffers[buffer_idx]->data());
+    eval_batch->SetBuffer(desc.data_idx(), data_buf, array_data.offset);
+  }
   if (is_output) {
     // pass in the Buffer object for output data buffers. Can be used for resizing.
-    uint8_t* data_buf_ptr =
-        reinterpret_cast<uint8_t*>(array_data.buffers[buffer_idx].get());
-    eval_batch->SetBuffer(desc.data_buffer_ptr_idx(), data_buf_ptr, array_data.offset);
+    if (array_data.type->id() == arrow::Type::NA) {
+      eval_batch->SetBuffer(desc.data_buffer_ptr_idx(), nullptr, array_data.offset);
+    } else {
+      uint8_t* data_buf_ptr =
+          reinterpret_cast<uint8_t*>(array_data.buffers[buffer_idx].get());
+      eval_batch->SetBuffer(desc.data_buffer_ptr_idx(), data_buf_ptr, array_data.offset);
+    }
   }
 }
 

cpp/src/gandiva/dex.h

@@ -205,6 +205,14 @@ class GANDIVA_EXPORT LiteralDex : public Dex {
   LiteralHolder holder_;
 };
 
+/// decomposed expression for a null literal.
+class GANDIVA_EXPORT NullLiteralDex : public Dex {
+ public:
+  NullLiteralDex() {}
+
+  void Accept(DexVisitor& visitor) override { visitor.Visit(*this); }
+};
+
 /// decomposed if-else expression.
 class GANDIVA_EXPORT IfDex : public Dex {
  public:

cpp/src/gandiva/dex_visitor.h

@@ -31,6 +31,7 @@ class VectorReadFixedLenValueDex;
 class VectorReadVarLenValueDex;
 class LocalBitMapValidityDex;
 class LiteralDex;
+class NullLiteralDex;
 class TrueDex;
 class FalseDex;
 class NonNullableFuncDex;
@@ -54,6 +55,7 @@ class GANDIVA_EXPORT DexVisitor {
   virtual void Visit(const TrueDex& dex) = 0;
   virtual void Visit(const FalseDex& dex) = 0;
   virtual void Visit(const LiteralDex& dex) = 0;
+  virtual void Visit(const NullLiteralDex& dex) = 0;
   virtual void Visit(const NonNullableFuncDex& dex) = 0;
   virtual void Visit(const NullableNeverFuncDex& dex) = 0;
   virtual void Visit(const NullableInternalFuncDex& dex) = 0;
@@ -80,6 +82,7 @@ class GANDIVA_EXPORT DexDefaultVisitor : public DexVisitor {
   VISIT_DCHECK(TrueDex)
   VISIT_DCHECK(FalseDex)
   VISIT_DCHECK(LiteralDex)
+  VISIT_DCHECK(NullLiteralDex)
   VISIT_DCHECK(NonNullableFuncDex)
   VISIT_DCHECK(NullableNeverFuncDex)
   VISIT_DCHECK(NullableInternalFuncDex)

cpp/src/gandiva/exported_funcs.h

@@ -32,6 +32,12 @@ class ExportedFuncsBase {
   virtual void AddMappings(Engine* engine) const = 0;
 };
 
+// Class for exporting Null functions
+class ExportedNullFunctions : public ExportedFuncsBase {
+  void AddMappings(Engine* engine) const override;
+};
+REGISTER_EXPORTED_FUNCS(ExportedNullFunctions);
+
 // Class for exporting Stub functions
 class ExportedStubFunctions : public ExportedFuncsBase {
   void AddMappings(Engine* engine) const override;

cpp/src/gandiva/expr_decomposer.cc

@@ -225,6 +225,13 @@ Status ExprDecomposer::Visit(const LiteralNode& node) {
   return Status::OK();
 }
 
+Status ExprDecomposer::Visit(const NullLiteralNode& node) {
+  auto value_dex = std::make_shared<NullLiteralDex>();
+  auto validity_dex = std::make_shared<FalseDex>();
+  result_ = std::make_shared<ValueValidityPair>(validity_dex, value_dex);
+  return Status::OK();
+}
+
 // The bolow functions use a stack to detect :
 // a. nested if-else expressions.
 //    In such cases,  the local bitmap can be re-used.

cpp/src/gandiva/expr_decomposer.h

@@ -64,6 +64,7 @@ class GANDIVA_EXPORT ExprDecomposer : public NodeVisitor {
   Status Visit(const FunctionNode& node) override;
   Status Visit(const IfNode& node) override;
   Status Visit(const LiteralNode& node) override;
+  Status Visit(const NullLiteralNode& node) override;
   Status Visit(const BooleanNode& node) override;
   Status Visit(const InExpressionNode<int32_t>& node) override;
   Status Visit(const InExpressionNode<int64_t>& node) override;

cpp/src/gandiva/expr_validator.cc

@@ -42,11 +42,14 @@ Status ExprValidator::Validate(const ExpressionPtr& expr) {
 }
 
 Status ExprValidator::Visit(const FieldNode& node) {
-  auto llvm_type = types_->IRType(node.return_type()->id());
-  ARROW_RETURN_IF(llvm_type == nullptr,
-                  Status::ExpressionValidationError("Field ", node.field()->name(),
-                                                    " has unsupported data type ",
-                                                    node.return_type()->name()));
+  auto return_type = node.return_type();
+  if (return_type->id() != arrow::Type::NA) {
+    auto llvm_type = types_->DataVecType(node.return_type());
+    ARROW_RETURN_IF(llvm_type == nullptr,
+                    Status::ExpressionValidationError("Field ", node.field()->name(),
+                                                      " has unsupported data type ",
+                                                      node.return_type()->name()));
+  }
 
   // Ensure that field is found in schema
   auto field_in_schema_entry = field_map_.find(node.field()->name());
@@ -120,6 +123,15 @@ Status ExprValidator::Visit(const LiteralNode& node) {
   return Status::OK();
 }
 
+Status ExprValidator::Visit(const NullLiteralNode& node) {
+  auto llvm_type = types_->DataVecType(node.return_type());
+  ARROW_RETURN_IF(llvm_type != nullptr,
+                  Status::ExpressionValidationError("Should be data type ",
+                                                    node.return_type()->name()));
+
+  return Status::OK();
+}
+
 Status ExprValidator::Visit(const BooleanNode& node) {
   ARROW_RETURN_IF(
       node.children().size() < 2,

cpp/src/gandiva/expr_validator.h

@@ -57,6 +57,7 @@ class ExprValidator : public NodeVisitor {
   Status Visit(const FunctionNode& node) override;
   Status Visit(const IfNode& node) override;
   Status Visit(const LiteralNode& node) override;
+  Status Visit(const NullLiteralNode& node) override;
   Status Visit(const BooleanNode& node) override;
   Status Visit(const InExpressionNode<int32_t>& node) override;
   Status Visit(const InExpressionNode<int64_t>& node) override;

cpp/src/gandiva/function_registry.cc

@@ -20,6 +20,7 @@
 #include "gandiva/function_registry_datetime.h"
 #include "gandiva/function_registry_hash.h"
 #include "gandiva/function_registry_math_ops.h"
+#include "gandiva/function_registry_null.h"
 #include "gandiva/function_registry_string.h"
 #include "gandiva/function_registry_timestamp_arithmetic.h"
 
@@ -65,6 +66,9 @@ SignatureMap FunctionRegistry::InitPCMap() {
   auto v6 = GetDateTimeArithmeticFunctionRegistry();
   pc_registry_.insert(std::end(pc_registry_), v6.begin(), v6.end());
 
+  auto v8 = GetNullFunctionRegistry();
+  pc_registry_.insert(std::end(pc_registry_), v8.begin(), v8.end());
+
   for (auto& elem : pc_registry_) {
     for (auto& func_signature : elem.signatures()) {
       map.insert(std::make_pair(&(func_signature), &elem));

cpp/src/gandiva/function_registry_common.h

@@ -44,6 +44,7 @@ using arrow::int32;
 using arrow::int64;
 using arrow::int8;
 using arrow::month_interval;
+using arrow::null;
 using arrow::uint16;
 using arrow::uint32;
 using arrow::uint64;

cpp/src/gandiva/function_registry_null.h

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <vector>
+
+#include "gandiva/native_function.h"
+
+namespace gandiva {
+
+std::vector<NativeFunction> GetNullFunctionRegistry() {
+  static std::vector<NativeFunction> null_fn_registry_ = {
+      NativeFunction("equal",
+                     {"not_equal", "less_than", "less_than_or_equal_to", "greater_than",
+                      "greater_than_or_equal_to"},
+                     DataTypeVector{null(), null()}, null(), kResultNullNever,
+                     "compare_null_null"),
+      NativeFunction("isnull", {}, DataTypeVector{null()}, boolean(), kResultNullNever,
+                     "isnull_null"),
+      NativeFunction("isnotnull", {}, DataTypeVector{null()}, boolean(), kResultNullNever,
+                     "isnotnull_null")};
+  return null_fn_registry_;
+}
+
+}  // namespace gandiva

cpp/src/gandiva/llvm_generator.cc

@@ -170,6 +170,9 @@ llvm::Value* LLVMGenerator::GetDataReference(llvm::Value* arg_addrs, int idx,
   llvm::Value* load = LoadVectorAtIndex(arg_addrs, idx, name);
   llvm::Type* base_type = types()->DataVecType(field->type());
   llvm::Value* ret;
+  if (base_type == nullptr) {
+    return nullptr;
+  }
   if (base_type->isPointerTy()) {
     ret = ir_builder()->CreateIntToPtr(load, base_type, name + "_darray");
   } else {
@@ -363,6 +366,8 @@ Status LLVMGenerator::CodeGenExprValue(DexPtr value_expr, int buffer_count,
     AddFunctionCall("gdv_fn_populate_varlen_vector", types()->i32_type(),
                     {arg_context_ptr, output_buffer_ptr_ref, output_offset_ref, loop_var,
                      output_value->data(), output_value->length()});
+  } else if (output_type_id == arrow::Type::NA) {
+    // Do nothing when data type is null
   } else {
     return Status::NotImplemented("output type ", output->Type()->ToString(),
                                   " not supported");
@@ -452,6 +457,10 @@ void LLVMGenerator::ComputeBitMapsForExpr(const CompiledExpr& compiled_expr,
   // Extract the destination bitmap address.
   int out_idx = compiled_expr.output()->validity_idx();
   uint8_t* dst_bitmap = eval_batch.GetBuffer(out_idx);
+  if (dst_bitmap == nullptr) {
+    // Return when dst_bitmap is null meaning data type is null
+    return;
+  }
   // Compute the destination bitmap.
   if (selection_vector == nullptr) {
     accumulator.ComputeResult(dst_bitmap);
@@ -491,7 +500,7 @@ llvm::Value* LLVMGenerator::AddFunctionCall(const std::string& full_name,
 
   // build a call to the llvm function.
   llvm::Value* value;
-  if (ret_type->isVoidTy()) {
+  if (ret_type == nullptr || ret_type->isVoidTy()) {
     // void functions can't have a name for the call.
     value = ir_builder()->CreateCall(fn, args);
   } else {
@@ -556,6 +565,9 @@ void LLVMGenerator::Visitor::Visit(const VectorReadFixedLenValueDex& dex) {
       break;
     }
 
+    case arrow::Type::NA:
+      break;
+
     default: {
       auto slot_offset = CreateGEP(builder, slot_ref, slot_index);
       slot_value = CreateLoad(builder, slot_offset, dex.FieldName());
@@ -720,6 +732,13 @@ void LLVMGenerator::Visitor::Visit(const LiteralDex& dex) {
   result_.reset(new LValue(value, len));
 }
 
+void LLVMGenerator::Visitor::Visit(const NullLiteralDex& dex) {
+  llvm::Value* value = nullptr;
+  llvm::Value* len = nullptr;
+  ADD_VISITOR_TRACE("visit Literal null");
+  result_.reset(new LValue(value, len));
+}
+
 void LLVMGenerator::Visitor::Visit(const NonNullableFuncDex& dex) {
   const std::string& function_name = dex.func_descriptor()->name();
   ADD_VISITOR_TRACE("visit NonNullableFunc base function " + function_name);
@@ -1133,6 +1152,9 @@ LValuePtr LLVMGenerator::Visitor::BuildIfElse(llvm::Value* condition,
   // Emit the merge block.
   builder->SetInsertPoint(merge_bb);
   auto llvm_type = types->IRType(result_type->id());
+  if (llvm_type == nullptr) {
+    return nullptr;
+  }
   llvm::PHINode* result_value = builder->CreatePHI(llvm_type, 2, "res_value");
   result_value->addIncoming(then_lvalue->data(), then_bb);
   result_value->addIncoming(else_lvalue->data(), else_bb);
@@ -1248,10 +1270,11 @@ std::vector<llvm::Value*> LLVMGenerator::Visitor::BuildParams(
     // build value.
     DexPtr value_expr = pair->value_expr();
     value_expr->Accept(*this);
-    LValue& result_ref = *result();
-
-    // append all the parameters corresponding to this LValue.
-    result_ref.AppendFunctionParams(&params);
+    if (auto result_ptr = result()) {
+      LValue& result_ref = *result_ptr;
+      // append all the parameters corresponding to this LValue.
+      result_ref.AppendFunctionParams(&params);
+    }
 
     // build validity.
     if (with_validity) {

cpp/src/gandiva/llvm_generator.h

@@ -100,6 +100,7 @@ class GANDIVA_EXPORT LLVMGenerator {
     void Visit(const TrueDex& dex) override;
     void Visit(const FalseDex& dex) override;
     void Visit(const LiteralDex& dex) override;
+    void Visit(const NullLiteralDex& dex) override;
     void Visit(const NonNullableFuncDex& dex) override;
     void Visit(const NullableNeverFuncDex& dex) override;
     void Visit(const NullableInternalFuncDex& dex) override;

cpp/src/gandiva/node.h

@@ -23,7 +23,6 @@
 #include <vector>
 
 #include "arrow/status.h"
-
 #include "gandiva/arrow.h"
 #include "gandiva/func_descriptor.h"
 #include "gandiva/gandiva_aliases.h"
@@ -94,6 +93,20 @@ class GANDIVA_EXPORT LiteralNode : public Node {
   bool is_null_;
 };
 
+/// \brief Node in the expression tree, representing a NullLiteralNode.
+class GANDIVA_EXPORT NullLiteralNode : public Node {
+ public:
+  NullLiteralNode() : Node(arrow::null()) {}
+
+  Status Accept(NodeVisitor& visitor) const override { return visitor.Visit(*this); }
+
+  std::string ToString() const override {
+    std::stringstream ss;
+    ss << "(const " << return_type()->ToString() << ") null";
+    return ss.str();
+  }
+};
+
 /// \brief Node in the expression tree, representing an arrow field.
 class GANDIVA_EXPORT FieldNode : public Node {
  public:

cpp/src/gandiva/node_visitor.h

@@ -31,6 +31,7 @@ class FieldNode;
 class FunctionNode;
 class IfNode;
 class LiteralNode;
+class NullLiteralNode;
 class BooleanNode;
 template <typename Type>
 class InExpressionNode;
@@ -44,6 +45,7 @@ class GANDIVA_EXPORT NodeVisitor {
   virtual Status Visit(const FunctionNode& node) = 0;
   virtual Status Visit(const IfNode& node) = 0;
   virtual Status Visit(const LiteralNode& node) = 0;
+  virtual Status Visit(const NullLiteralNode& node) = 0;
   virtual Status Visit(const BooleanNode& node) = 0;
   virtual Status Visit(const InExpressionNode<int32_t>& node) = 0;
   virtual Status Visit(const InExpressionNode<int64_t>& node) = 0;