Implement Series.from_list of null type (#808)

lib/explorer/data_frame.ex

@@ -5063,7 +5063,7 @@ defmodule Explorer.DataFrame do
 
   defp types_are_numeric_compatible?(types, name, type) do
     types[name] != type and types[name] in Shared.numeric_types() and
-      type in Shared.numeric_types()
+      (type == :null or type in Shared.numeric_types())
   end
 
   defp cast_numeric_columns_to_float(dfs, changed_types) do

lib/explorer/polars_backend/native.ex

@@ -336,6 +336,7 @@ defmodule Explorer.PolarsBackend.Native do
   def s_name(_s), do: err()
   def s_nil_count(_s), do: err()
   def s_not(_s), do: err()
+  def s_from_list_null(_name, _val), do: err()
   def s_from_list_bool(_name, _val), do: err()
   def s_from_list_date(_name, _val), do: err()
   def s_from_list_time(_name, _val), do: err()

lib/explorer/polars_backend/shared.ex

@@ -153,6 +153,7 @@ defmodule Explorer.PolarsBackend.Shared do
       {:datetime, precision} -> Native.s_from_list_datetime(name, list, Atom.to_string(precision))
       {:duration, precision} -> Native.s_from_list_duration(name, list, Atom.to_string(precision))
       :binary -> Native.s_from_list_binary(name, list)
+      :null -> Native.s_from_list_null(name, length(list))
     end
   end
 

lib/explorer/series.ex

@@ -286,12 +286,12 @@ defmodule Explorer.Series do
         f64 [1.0, 2.0, -Inf, 4.0]
       >
 
-  Trying to create a "nil" series will, by default, result in a series of floats:
+  Trying to create a "nil" series will, by default, result in a series of null type:
 
       iex> Explorer.Series.from_list([nil, nil])
       #Explorer.Series<
         Polars[2]
-        f64 [nil, nil]
+        null [nil, nil]
       >
 
   You can specify the desired `dtype` for a series with the `:dtype` option.

lib/explorer/shared.ex

@@ -262,13 +262,8 @@ defmodule Explorer.Shared do
   """
   def dtype_from_list!(list, preferable_type \\ nil) do
     initial_type =
-      if leaf_dtype(preferable_type) in ([
-                                           :numeric,
-                                           :binary,
-                                           {:f, 32},
-                                           {:f, 64},
-                                           :category
-                                         ] ++ @integer_types),
+      if leaf_dtype(preferable_type) in ([:null, :numeric, :binary, {:f, 32}, {:f, 64}, :category] ++
+                                           @integer_types),
          do: preferable_type
 
     type =
@@ -314,6 +309,8 @@ defmodule Explorer.Shared do
   defp type(item, :category) when is_binary(item), do: :category
   defp type(item, _type) when is_binary(item), do: :string
 
+  defp type(nil, nil), do: :null
+  defp type(nil, :null), do: :null
   defp type(item, _type) when is_nil(item), do: nil
   defp type([], _type), do: nil
   defp type([_item | _] = items, type), do: {:list, result_list_type(items, type)}
@@ -354,6 +351,8 @@ defmodule Explorer.Shared do
 
   defp new_type_matches?(type, type), do: true
 
+  defp new_type_matches?(:null, _type), do: true
+
   defp new_type_matches?(nil, _new_type), do: true
 
   defp new_type_matches?({:struct, types}, {:struct, new_types}) do

native/explorer/src/lib.rs

@@ -389,6 +389,7 @@ rustler::init!(
         s_not,
         s_log,
         s_log_natural,
+        s_from_list_null,
         s_from_list_bool,
         s_from_list_date,
         s_from_list_time,

native/explorer/src/series.rs

@@ -22,6 +22,12 @@ pub fn s_as_str(data: ExSeries) -> Result<String, ExplorerError> {
     Ok(format!("{:?}", data.resource.0))
 }
 
+#[rustler::nif(schedule = "DirtyCpu")]
+pub fn s_from_list_null(name: &str, length: usize) -> ExSeries {
+    let s = Series::new_null(name, length);
+    ExSeries::new(Series::new(name, s))
+}
+
 macro_rules! from_list {
     ($name:ident, $type:ty) => {
         #[rustler::nif(schedule = "DirtyCpu")]

test/explorer/series/list_test.exs

@@ -4,6 +4,13 @@ defmodule Explorer.Series.ListTest do
   alias Explorer.Series
 
   describe "from_list/2" do
+    test "list of list of nulls" do
+      series = Series.from_list([[nil, nil], [nil]])
+      assert series.dtype == {:list, :null}
+      assert series[0] == [nil, nil]
+      assert Series.to_list(series) == [[nil, nil], [nil]]
+    end
+
     test "list of lists of one integer" do
       series = Series.from_list([[1]])
 

test/explorer/series/struct_test.exs

@@ -4,6 +4,23 @@ defmodule Explorer.Series.StructTest do
   alias Explorer.Series
 
   describe "from_list/2" do
+    test "allows struct of all nil value" do
+      s =
+        Series.from_list([
+          %{a: nil, b: nil},
+          %{a: 3, b: nil},
+          %{a: 5, b: nil}
+        ])
+
+      assert s.dtype == {:struct, %{"a" => {:s, 64}, "b" => :null}}
+
+      assert Series.to_list(s) == [
+               %{"a" => nil, "b" => nil},
+               %{"a" => 3, "b" => nil},
+               %{"a" => 5, "b" => nil}
+             ]
+    end
+
     test "allows struct values" do
       s = Series.from_list([%{a: 1}, %{a: 3}, %{a: 5}])
 

test/explorer/series_test.exs

@@ -30,6 +30,20 @@ defmodule Explorer.SeriesTest do
   end
 
   describe "from_list/1" do
+    test "with nils" do
+      s = Series.from_list([nil, nil, nil])
+
+      assert Series.to_list(s) === [nil, nil, nil]
+      assert Series.dtype(s) == :null
+    end
+
+    test "with non nils and dtype :null" do
+      s = Series.from_list([1, 2, 3], dtype: :null)
+
+      assert Series.to_list(s) === [nil, nil, nil]
+      assert Series.dtype(s) == :null
+    end
+
     test "with integers" do
       s = Series.from_list([1, 2, 3])