Fix some edge-case in type_intersection_env

This commit fixes examples reported in JuliaLang#43064. We should not erase the current `envout` value, as `subtype_unionall` needs it to check whether we are trying to assign a different one.
N5N3 · Aug 10, 2022 · 7c76a97 · 7c76a97
1 parent 4025b99
commit 7c76a97
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 15 deletions.
diff --git a/src/subtype.c b/src/subtype.c
@@ -793,14 +793,16 @@ static int subtype_unionall(jl_value_t *t, jl_unionall_t *u, jl_stenv_t *e, int8
                 val = (jl_value_t*)u->var;
             else
                 val = (jl_value_t*)jl_new_typevar(u->var->name, vb.lb, vb.ub);
-            jl_value_t *oldval = e->envout[e->envidx];
-            // if we try to assign different variable values (due to checking
-            // multiple union members), consider the value unknown.
-            if (oldval && !jl_egal(oldval, val))
-                e->envout[e->envidx] = (jl_value_t*)u->var;
-            else
-                e->envout[e->envidx] = fix_inferred_var_bound(u->var, val);
+            if (ans) {
+                jl_value_t *oldval = e->envout[e->envidx];
+                // if we try to assign different variable values (due to checking
+                // multiple union members), consider the value unknown.
+                if (ans && oldval && !jl_egal(oldval, val))
+                    e->envout[e->envidx] = (jl_value_t*)u->var;
+                else
+                    e->envout[e->envidx] = fix_inferred_var_bound(u->var, val);
             // TODO: substitute the value (if any) of this variable into previous envout entries
+            }
         }
     }
     else {
@@ -1393,10 +1395,16 @@ static int exists_subtype(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, jl_value_
         e->Lunions.more = 0;
         if (subtype(x, y, e, param))
             return 1;
-        restore_env(e, saved, se);
         int set = e->Runions.more;
-        if (!set)
+        if (set) {
+            e->envidx++;
+            restore_env(e, saved, se);
+            e->envidx--;
+        }
+        else {
+            restore_env(e, saved, se);
             return 0;
+        }
         for (int i = set; i <= lastset; i++)
             statestack_set(&e->Runions, i, 0);
         lastset = set - 1;

diff --git a/test/arrayops.jl b/test/arrayops.jl
@@ -1203,10 +1203,9 @@ end
     @test o == fill(1, 3, 4)
 
     # issue #18524
-    # m = mapslices(x->tuple(x), [1 2; 3 4], dims=1) # see variations of this below
-    # ERROR: fatal error in type inference (type bound), https://github.com/JuliaLang/julia/issues/43064
-    # @test m[1,1] == ([1,3],)
-    # @test m[1,2] == ([2,4],)
+    m = mapslices(x->tuple(x), [1 2; 3 4], dims=1) # see variations of this below
+    @test m[1,1] == ([1,3],)
+    @test m[1,2] == ([2,4],)
 
     r = rand(Int8, 4,5,2)
     @test vec(mapslices(repr, r, dims=(2,1))) == map(repr, eachslice(r, dims=3))
@@ -1216,8 +1215,6 @@ end
     # failures
     @test_broken @inferred(mapslices(tuple, [1 2; 3 4], dims=1)) == [([1, 3],)  ([2, 4],)]
     @test_broken @inferred(mapslices(transpose, r, dims=(1,3))) == permutedims(r, (3,2,1))
-    # ERROR: fatal error in type inference (type bound), https://github.com/JuliaLang/julia/issues/43064
-    @test_broken @inferred(mapslices(x -> tuple(x), [1 2; 3 4], dims=1)) == [([1, 3],)  ([2, 4],)]
 
     # re-write, #40996
     @test_throws ArgumentError mapslices(identity, rand(2,3), dims=0) # previously BoundsError

diff --git a/test/subtype.jl b/test/subtype.jl
@@ -1996,3 +1996,15 @@ let T = TypeVar(:T, Real),
     @test !(UnionAll(T, UnionAll(V, UnionAll(T, Type{Pair{T, V}}))) <: UnionAll(T, UnionAll(V, Type{Pair{T, V}})))
     @test !(UnionAll(T, UnionAll(V, UnionAll(T, S))) <: UnionAll(T, UnionAll(V, S)))
 end
+
+# issue #43064
+let
+    TT = Tuple{Type{T},Union{Real,Missing,Nothing}} where {T}
+    T1 = Union{Type{Int8},Type{Int16}}
+    _, E = intersection_env(Tuple{T1,Missing}, TT)
+    @test only(E) isa TypeVar
+    _, E = intersection_env(Tuple{T1,Nothing}, TT)
+    @test only(E) isa TypeVar
+    _, E = intersection_env(Tuple{T1,Int}, TT)
+    @test only(E) isa TypeVar
+end