Handle NaNs when comparing BigInt against Float (#13293)

spec/std/big/big_int_spec.cr

@@ -61,6 +61,19 @@ describe "BigInt" do
     1.1.should_not eq(1.to_big_i)
 
     [1.1, 1.to_big_i, 3.to_big_i, 2.2].sort.should eq([1, 1.1, 2.2, 3])
+
+    (1.to_big_i <=> Float64::NAN).should be_nil
+    (1.to_big_i <=> Float32::NAN).should be_nil
+    (Float64::NAN <=> 1.to_big_i).should be_nil
+    (Float32::NAN <=> 1.to_big_i).should be_nil
+
+    typeof(1.to_big_i <=> Float64::NAN).should eq(Int32?)
+    typeof(1.to_big_i <=> Float32::NAN).should eq(Int32?)
+    typeof(Float64::NAN <=> 1.to_big_i).should eq(Int32?)
+    typeof(Float32::NAN <=> 1.to_big_i).should eq(Int32?)
+
+    typeof(1.to_big_i <=> 1.to_big_f).should eq(Int32)
+    typeof(1.to_big_f <=> 1.to_big_i).should eq(Int32)
   end
 
   it "divides and calculates the modulo" do

src/big/big_int.cr

@@ -121,8 +121,8 @@ struct BigInt < Int
     end
   end
 
-  def <=>(other : Float)
-    LibGMP.cmp_d(mpz, other)
+  def <=>(other : Float::Primitive)
+    LibGMP.cmp_d(mpz, other) unless other.nan?
   end
 
   def +(other : BigInt) : BigInt
@@ -811,7 +811,8 @@ struct Float
   include Comparable(BigInt)
 
   def <=>(other : BigInt)
-    -(other <=> self)
+    cmp = other <=> self
+    -cmp if cmp
   end
 
   # Returns a `BigInt` representing this float (rounded using `floor`).