Assert for non-empty nulls

java/pom.xml

@@ -158,6 +158,7 @@
         <maven.compiler.target>1.8</maven.compiler.target>
         <junit.version>5.4.2</junit.version>
         <ai.rapids.refcount.debug>false</ai.rapids.refcount.debug>
+        <ai.rapids.allow.nonempty.nulls>true</ai.rapids.allow.nonempty.nulls>
         <ai.rapids.cudf.nvtx.enabled>false</ai.rapids.cudf.nvtx.enabled>
         <native.build.path>${basedir}/target/cmake-build</native.build.path>
         <skipNativeCopy>false</skipNativeCopy>
@@ -550,6 +551,7 @@
                     <redirectTestOutputToFile>true</redirectTestOutputToFile>
                     <systemPropertyVariables>
                         <ai.rapids.refcount.debug>${ai.rapids.refcount.debug}</ai.rapids.refcount.debug>
+                        <ai.rapids.allow.nonempty.nulls>${ai.rapids.allow.nonempty.nulls}</ai.rapids.allow.nonempty.nulls>
                         <ai.rapids.cudf.nvtx.enabled>${ai.rapids.cudf.nvtx.enabled}</ai.rapids.cudf.nvtx.enabled>
                     </systemPropertyVariables>
                 </configuration>

java/src/main/java/ai/rapids/cudf/AssertEmptyNulls.java

@@ -0,0 +1,18 @@
+package ai.rapids.cudf;
+
+public class AssertEmptyNulls {
+  /*
+  We have to read the system property to see if we should by-pass this assert. Ideally we should
+  just use `-da:AssertNonEmptyNulls` as per Java standard, but the reason for doing this is because
+  surefire-plugin doesn't expose anyway to disable assertions at the class level or package level
+  instead we can only disable asserts at the ClassLoader level. Therefore, in order for the tests to
+  pass we need to set a flag from the pom to know we are running unit-tests to allow non-empty nulls
+ */
+  private static final boolean ALLOW_NON_EMPTY_NULLS =
+      Boolean.getBoolean("ai.rapids.allow.nonempty.nulls");
+  public static void assertEmptyNulls(ColumnView cv) {
+    if (cv.type.isNestedType() && !ALLOW_NON_EMPTY_NULLS) {
+      assert !cv.hasNonEmptyNulls() : "Column has non-empty nulls";
+    }
+  }
+}

java/src/main/java/ai/rapids/cudf/ColumnView.java

@@ -52,6 +52,7 @@ public class ColumnView implements AutoCloseable, BinaryOperable {
     this.rows = ColumnView.getNativeRowCount(viewHandle);
     this.nullCount = ColumnView.getNativeNullCount(viewHandle);
     this.offHeap = null;
+    AssertEmptyNulls.assertEmptyNulls(this);
   }
 
 
@@ -67,6 +68,7 @@ protected ColumnView(ColumnVector.OffHeapState state) {
     type = DType.fromNative(ColumnView.getNativeTypeId(viewHandle), ColumnView.getNativeTypeScale(viewHandle));
     rows = ColumnView.getNativeRowCount(viewHandle);
     nullCount = ColumnView.getNativeNullCount(viewHandle);
+    AssertEmptyNulls.assertEmptyNulls(this);
   }
 
   /**

java/src/test/java/ai/rapids/cudf/ColumnVectorTest.java

@@ -45,6 +45,8 @@
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 public class ColumnVectorTest extends CudfTestBase {
+  private static final boolean ALLOW_NON_EMPTY_NULLS =
+      Boolean.getBoolean("ai.rapids.allow.nonempty.nulls");
 
   public static final double PERCENTAGE = 0.0001;
 
@@ -742,6 +744,7 @@ void testSpark32BitMurmur3HashListsAndNestedLists() {
 
   @Test
   void testAndNullReconfigureNulls() {
+    assumeTrue(ALLOW_NON_EMPTY_NULLS);
     try (ColumnVector v0 = ColumnVector.fromBoxedInts(0, 100, null, null, Integer.MIN_VALUE, null);
          ColumnVector v1 = ColumnVector.fromBoxedInts(0, 100, 1, 2, Integer.MIN_VALUE, null);
          ColumnVector intResult = v1.mergeAndSetValidity(BinaryOp.BITWISE_AND, v0);
@@ -757,6 +760,7 @@ void testAndNullReconfigureNulls() {
 
   @Test
   void testOrNullReconfigureNulls() {
+    assumeTrue(ALLOW_NON_EMPTY_NULLS);
     try (ColumnVector v0 = ColumnVector.fromBoxedInts(0, 100, null, null, Integer.MIN_VALUE, null);
          ColumnVector v1 = ColumnVector.fromBoxedInts(0, 100, 1, 2, Integer.MIN_VALUE, null);
          ColumnVector v2 = ColumnVector.fromBoxedInts(0, 100, 1, 2, Integer.MIN_VALUE, Integer.MAX_VALUE);
@@ -6746,6 +6750,7 @@ private ColumnView[] getColumnViewWithNonEmptyNulls() {
 
   @Test
   void testPurgeNonEmptyNullsList() {
+    assumeTrue(ALLOW_NON_EMPTY_NULLS);
     ColumnView[] values = getColumnViewWithNonEmptyNulls();
     try (ColumnView colWithNonEmptyNulls = values[1];
          ColumnView input = values[0];
@@ -6761,6 +6766,7 @@ void testPurgeNonEmptyNullsList() {
 
   @Test
   void testPurgeNonEmptyNullsStruct() {
+    assumeTrue(ALLOW_NON_EMPTY_NULLS);
     ColumnView[] values = getColumnViewWithNonEmptyNulls();
     try (ColumnView listCol = values[1];
          ColumnView input = values[0];