From 666aeb0d4ae0732d456ce561841901ff6d5d902d Mon Sep 17 00:00:00 2001
From: Jameson Nash <vtjnash@gmail.com>
Date: Tue, 6 Dec 2022 09:48:37 -0500
Subject: [PATCH] intersect: fix a minor soundness issue with supertypes

When doing intersection, we might end up with a value in `env` (as the
only possible *value* for that parameter) without properly considering
that the parameter might be a TypeVar.
---
 src/subtype.c   | 46 ++++++----------------------------------------
 test/docs.jl    |  1 +
 test/subtype.jl | 11 ++++++++++-
 3 files changed, 17 insertions(+), 41 deletions(-)

diff --git a/src/subtype.c b/src/subtype.c
index cbb11520190cb..3d185fcf977a3 100644
--- a/src/subtype.c
+++ b/src/subtype.c
@@ -2882,48 +2882,14 @@ static void flip_vars(jl_stenv_t *e)
 // intersection where xd nominally inherits from yd
 static jl_value_t *intersect_sub_datatype(jl_datatype_t *xd, jl_datatype_t *yd, jl_stenv_t *e, int R, int param)
 {
+    // attempt to populate additional constraints into `e`
+    // if that attempt fails, then return bottom
+    // otherwise return xd (finish_unionall will later handle propagating those constraints)
     jl_value_t *isuper = R ? intersect((jl_value_t*)yd, (jl_value_t*)xd->super, e, param) :
                              intersect((jl_value_t*)xd->super, (jl_value_t*)yd, e, param);
-    if (isuper == jl_bottom_type) return jl_bottom_type;
-    if (jl_nparams(xd) == 0 || jl_nparams(xd->super) == 0 || !jl_has_free_typevars((jl_value_t*)xd))
-        return (jl_value_t*)xd;
-    jl_value_t *super_pattern=NULL;
-    JL_GC_PUSH2(&isuper, &super_pattern);
-    jl_value_t *wrapper = xd->name->wrapper;
-    super_pattern = jl_rewrap_unionall((jl_value_t*)((jl_datatype_t*)jl_unwrap_unionall(wrapper))->super,
-                                       wrapper);
-    int envsz = jl_subtype_env_size(super_pattern);
-    jl_value_t *ii = jl_bottom_type;
-    {
-        jl_value_t **env;
-        JL_GC_PUSHARGS(env, envsz);
-        jl_stenv_t tempe;
-        init_stenv(&tempe, env, envsz);
-        tempe.intersection = tempe.ignore_free = 1;
-        if (subtype_in_env(isuper, super_pattern, &tempe)) {
-            jl_value_t *wr = wrapper;
-            int i;
-            for(i=0; i<envsz; i++) {
-                // if a parameter is not constrained by the supertype, use the original
-                // parameter value from `x`. this is detected by the value in `env` being
-                // the exact typevar from the type's `wrapper`, or a free typevar.
-                jl_value_t *ei = env[i];
-                if (ei == (jl_value_t*)((jl_unionall_t*)wr)->var ||
-                    (jl_is_typevar(ei) && lookup(e, (jl_tvar_t*)ei) == NULL))
-                    env[i] = jl_tparam(xd,i);
-                wr = ((jl_unionall_t*)wr)->body;
-            }
-            JL_TRY {
-                ii = jl_apply_type(wrapper, env, envsz);
-            }
-            JL_CATCH {
-                ii = jl_bottom_type;
-            }
-        }
-        JL_GC_POP();
-    }
-    JL_GC_POP();
-    return ii;
+    if (isuper == jl_bottom_type)
+        return jl_bottom_type;
+    return (jl_value_t*)xd;
 }
 
 static jl_value_t *intersect_invariant(jl_value_t *x, jl_value_t *y, jl_stenv_t *e)
diff --git a/test/docs.jl b/test/docs.jl
index 4399722e864c1..6707278c53847 100644
--- a/test/docs.jl
+++ b/test/docs.jl
@@ -970,6 +970,7 @@ abstract type $(curmod_prefix)Undocumented.at1{T>:Integer, N}
 
 ```
 $(curmod_prefix)Undocumented.mt6{Integer, N}
+$(curmod_prefix)Undocumented.st5{T>:Integer, N}
 ```
 
 # Supertype Hierarchy
diff --git a/test/subtype.jl b/test/subtype.jl
index 70f3dd864cdbe..59e5b82fdc8c0 100644
--- a/test/subtype.jl
+++ b/test/subtype.jl
@@ -2188,7 +2188,16 @@ for T in (B46871{Int, N} where {N}, B46871{Int}) # intentional duplication
 end
 abstract type C38497{e,g<:Tuple,i} end
 struct Q38497{o,e<:NTuple{o},g} <: C38497{e,g,Array{o}} end
-@testintersect(Q38497{<:Any, Tuple{Int}}, C38497, Q38497{1, Tuple{Int}, <:Tuple})
+@testintersect(Q38497{<:Any, Tuple{Int}}, C38497, Q38497{<:Any, Tuple{Int}, <:Tuple})
+# n.b. the only concrete instance of this type is Q38497{1, Tuple{Int}, <:Tuple} (since NTuple{o} also adds an ::Int constraint)
+# but this abstract type is also part of the intersection abstractly
+
+abstract type X38497{T<:Number} end
+abstract type Y38497{T>:Integer} <: X38497{T} end
+struct Z38497{T>:Int} <: Y38497{T} end
+@testintersect(Z38497, X38497, Z38497{T} where Int<:T<:Number)
+@testintersect(Z38497, Y38497, Z38497{T} where T>:Integer)
+@testintersect(X38497, Y38497, Y38497{T} where Integer<:T<:Number)
 
 #issue #33138
 @test Vector{Vector{Tuple{T,T}} where Int<:T<:Int} <: Vector{Vector{Tuple{S1,S1} where S<:S1<:S}} where S