remove breaking behavior on nulls

rapidsai · Nov 14, 2022 · 0e45d13 · 0e45d13
1 parent bea7fc3
commit 0e45d13
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 10 deletions.
diff --git a/cpp/src/io/utilities/parsing_utils.cuh b/cpp/src/io/utilities/parsing_utils.cuh
@@ -578,9 +578,9 @@ struct ConvertFunctor {
       return as_hex ? cudf::io::parse_numeric<T, 16>(begin, end, opts)
                     : cudf::io::parse_numeric<T>(begin, end, opts);
     }();
-    if (value.has_value()) { static_cast<T*>(out_buffer)[row] = *value; }
+    static_cast<T*>(out_buffer)[row] = value.value_or(std::numeric_limits<T>::quiet_NaN());
 
-    return value.has_value();
+    return true;
   }
 
   /**
@@ -630,9 +630,9 @@ struct ConvertFunctor {
       }
       return cudf::io::parse_numeric<T>(begin, end, opts);
     }();
-    if (value.has_value()) { static_cast<T*>(out_buffer)[row] = *value; }
+    static_cast<T*>(out_buffer)[row] = value.value_or(std::numeric_limits<T>::quiet_NaN());
 
-    return value.has_value();
+    return true;
   }
 
   /**
@@ -659,7 +659,7 @@ struct ConvertFunctor {
       }
       return cudf::io::parse_numeric<T>(begin, end, opts);
     }();
-    if (value.has_value()) { static_cast<T*>(out_buffer)[row] = *value; }
+    static_cast<T*>(out_buffer)[row] = value.value_or(std::numeric_limits<T>::quiet_NaN());
 
     return value.has_value() and !std::isnan(*value);
   }

diff --git a/cpp/tests/io/csv_test.cpp b/cpp/tests/io/csv_test.cpp
@@ -1170,7 +1170,8 @@ TEST_F(CsvReaderTest, InvalidFloatingPoint)
 
   const auto col_data = cudf::test::to_host<float>(view.column(0));
   // col_data.first contains the column data
-  // ignore all data because it is all nulls.
+  for (const auto& elem : col_data.first)
+    ASSERT_TRUE(std::isnan(elem));
   // col_data.second contains the bitmasks
   ASSERT_EQ(0u, col_data.second[0]);
 }

diff --git a/cpp/tests/io/json_test.cpp b/cpp/tests/io/json_test.cpp
@@ -914,7 +914,8 @@ TEST_P(JsonReaderParamTest, InvalidFloatingPoint)
 
   const auto col_data = cudf::test::to_host<float>(result.tbl->view().column(0));
   // col_data.first contains the column data
-  // ignore all data because it is all nulls.
+  for (const auto& elem : col_data.first)
+    ASSERT_TRUE(std::isnan(elem));
   // col_data.second contains the bitmasks
   ASSERT_EQ(0u, col_data.second[0]);
 }
@@ -1495,7 +1496,7 @@ TEST_P(JsonReaderParamTest, JsonDtypeParsing)
 
   auto int_col = int_wrapper{
     {0, 0, int_NA, 1, 1, int_NA, int_NA, int_NA, int_NA, 1, 0, int_NA, 1, 0, int_NA, int_NA},
-    make_validity(validity)};
+    cudf::test::iterators::nulls_at(std::vector<int>{8})};
   auto float_col = float_wrapper{{0.0,
                                   0.0,
                                   double_NA,
@@ -1534,7 +1535,7 @@ TEST_P(JsonReaderParamTest, JsonDtypeParsing)
                                 false,
                                 bool_NA,
                                 bool_NA},
-                               make_validity(validity)};
+                               cudf::test::iterators::nulls_at(std::vector<int>{8})};
 
   // Types to test
   const std::vector<data_type> dtypes = {

diff --git a/python/cudf/cudf/tests/test_csv.py b/python/cudf/cudf/tests/test_csv.py
@@ -834,7 +834,7 @@ def test_csv_reader_bools_NA():
     expected = pd.DataFrame(
         {
             "text": ["true", "false", "foo", "bar", "qux"],
-            "int": [1.0, 0.0, 1.0, 0.0, np.nan],
+            "int": [1, 0, 1, 0, 0],
         }
     )
     # breaking behaviour is np.nan for qux