implement - nil datatype - polar null type column

lib/explorer/shared.ex

@@ -73,6 +73,7 @@ defmodule Explorer.Shared do
   def normalise_dtype(:u16), do: {:u, 16}
   def normalise_dtype(:u32), do: {:u, 32}
   def normalise_dtype(:u64), do: {:u, 64}
+  def normalise_dtype(:null), do: :null
   def normalise_dtype(_dtype), do: nil
 
   @doc """

native/explorer/src/datatypes/ex_dtypes.rs

@@ -43,6 +43,7 @@ impl TryFrom<&ExTimeUnit> for TimeUnit {
 
 #[derive(NifTaggedEnum)]
 pub enum ExSeriesDtype {
+    Null,
     Binary,
     Boolean,
     Category,
@@ -63,6 +64,7 @@ impl TryFrom<&DataType> for ExSeriesDtype {
 
     fn try_from(value: &DataType) -> Result<Self, Self::Error> {
         match value {
+            DataType::Null => Ok(ExSeriesDtype::Null),
             DataType::Binary => Ok(ExSeriesDtype::Binary),
             DataType::Boolean => Ok(ExSeriesDtype::Boolean),
             DataType::Categorical(_, _) => Ok(ExSeriesDtype::Category),
@@ -128,6 +130,7 @@ impl TryFrom<&ExSeriesDtype> for DataType {
 
     fn try_from(value: &ExSeriesDtype) -> Result<Self, Self::Error> {
         match value {
+            ExSeriesDtype::Null => Ok(DataType::Null),
             ExSeriesDtype::Binary => Ok(DataType::Binary),
             ExSeriesDtype::Boolean => Ok(DataType::Boolean),
             ExSeriesDtype::Category => {

native/explorer/src/encoding.rs

@@ -505,6 +505,17 @@ macro_rules! series_to_list {
     };
 }
 
+#[inline]
+fn null_series_to_list<'b>(s: &Series, env: Env<'b>) -> Result<Term<'b>, ExplorerError> {
+    let nil_as_c_arg = atom::nil().to_term(env).as_c_arg();
+    let env_as_c_arg = env.as_c_arg();
+    let mut list = unsafe { list::make_list(env_as_c_arg, &[]) };
+    for _n in 0..s.len() {
+        list = unsafe { list::make_list_cell(env_as_c_arg, nil_as_c_arg, list) }
+    }
+    Ok(unsafe { Term::new(env, list) })
+}
+
 macro_rules! series_to_iovec {
     ($resource:ident, $s:ident, $env:ident, $convert_function:ident, $in_type:ty) => {{
         Ok(unsafe_iterator_series_to_list!(
@@ -593,6 +604,7 @@ pub fn term_from_value<'b>(v: AnyValue, env: Env<'b>) -> Result<Term<'b>, Explor
 
 pub fn list_from_series(s: ExSeries, env: Env) -> Result<Term, ExplorerError> {
     match s.dtype() {
+        DataType::Null => null_series_to_list(&s, env),
         DataType::Boolean => series_to_list!(s, env, bool),
 
         DataType::Int8 => series_to_list!(s, env, i8),

test/explorer/series_test.exs

@@ -3428,6 +3428,31 @@ defmodule Explorer.SeriesTest do
   end
 
   describe "cast/2" do
+    test "integer series to null" do
+      s = Series.from_list([1, 2, 3])
+      s1 = Series.cast(s, :null)
+      assert Series.to_list(s1) == [nil, nil, nil]
+      assert Series.dtype(s1) == :null
+    end
+
+    test "string series to null" do
+      s = Series.from_list(["a", "b", "c"])
+      s1 = Series.cast(s, :null)
+      assert Series.to_list(s1) == [nil, nil, nil]
+      assert Series.dtype(s1) == :null
+    end
+
+    test "error when to_iovec with :null dtype" do
+      assert_raise ArgumentError,
+                   ~r"cannot convert series of dtype :null into iovec",
+                   fn ->
+                     [1, 2, nil]
+                     |> Series.from_list()
+                     |> Series.cast(:null)
+                     |> Series.to_iovec()
+                   end
+    end
+
     test "integer series to string" do
       s = Series.from_list([1, 2, 3])
       s1 = Series.cast(s, :string)