fix obvious_subtype bug with egal objects (JuliaLang#45771)

When egal objects contain identical typevars with different environments, the resulting subtyping might not be so obvious. Fix JuliaLang#45703
pcjentsch · Aug 18, 2022 · a714bb5 · a714bb5
1 parent aa952ae
commit a714bb5
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/src/subtype.c b/src/subtype.c
@@ -1542,8 +1542,15 @@ static int obvious_subtype(jl_value_t *x, jl_value_t *y, jl_value_t *y0, int *su
         *subtype = 1;
         return 1;
     }
-    if (jl_is_unionall(x))
-        x = jl_unwrap_unionall(x);
+    while (jl_is_unionall(x)) {
+        if (!jl_is_unionall(y)) {
+            if (obvious_subtype(jl_unwrap_unionall(x), y, y0, subtype) && !*subtype)
+                return 1;
+            return 0;
+        }
+        x = ((jl_unionall_t*)x)->body;
+        y = ((jl_unionall_t*)y)->body;
+    }
     if (jl_is_unionall(y))
         y = jl_unwrap_unionall(y);
     if (x == (jl_value_t*)jl_typeofbottom_type->super)

diff --git a/test/subtype.jl b/test/subtype.jl
@@ -1987,3 +1987,12 @@ let A = Tuple{typeof(identity), Type{Union{}}},
     B = Tuple{typeof(identity), typeof(Union{})}
     @test A == B && (Base.isdispatchtuple(A) == Base.isdispatchtuple(B))
 end
+
+# issue #45703
+# requires assertions enabled (to catch discrepancy in obvious_subtype)
+let T = TypeVar(:T, Real),
+    V = TypeVar(:V, AbstractVector{T}),
+    S = Type{Pair{T, V}}
+    @test !(UnionAll(T, UnionAll(V, UnionAll(T, Type{Pair{T, V}}))) <: UnionAll(T, UnionAll(V, Type{Pair{T, V}})))
+    @test !(UnionAll(T, UnionAll(V, UnionAll(T, S))) <: UnionAll(T, UnionAll(V, S)))
+end