fix #30394, an unsoundness in ml_matches (#30396)

This fixes a corner case where a bug is caused, counter-intuitively,
by an over-estimated intersection.
We have method signatures A and B, with A<B (A is a strict subtype).
We have a dispatch tuple X, where X<:B and !(X<:A).
However, intersection returns X for intersect(X,A). Since there
appears to be a match there and A<B, ml_matches skips the match with B.
The fix just requires dispatch tuples to be a subtype of a signature
in order to match at all.

(cherry picked from commit b167bc2)

src/gf.c

@@ -2395,6 +2395,12 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio
                 closure->max_valid = ml->max_world;
         }
     }
+    // In some corner cases type intersection is conservative and returns something
+    // for intersect(A, B) even though A is a dispatch tuple and !(A <: B).
+    // For dispatch purposes in such a case we know there's no match. This check
+    // fixes issue #30394.
+    if (jl_is_dispatch_tupletype(closure->match.type) && !closure->match.issubty)
+        return 1;
     // a method is shadowed if type <: S <: m->sig where S is the
     // signature of another applicable method
     /*

test/compiler/compiler.jl

@@ -2077,3 +2077,24 @@ function f29036(s, i)
     val
 end
 @test Base.return_types(f29036, (String, Int)) == Any[Char]
+
+# issue #30394
+mutable struct Base30394
+    a::Int
+end
+
+mutable struct Foo30394
+    foo_inner::Base30394
+    Foo30394() = new(Base30394(1))
+end
+
+mutable struct Foo30394_2
+    foo_inner::Foo30394
+    Foo30394_2() = new(Foo30394())
+end
+
+f30394(foo::T1, ::Type{T2}) where {T2, T1 <: T2} = foo
+
+f30394(foo, T2) = f30394(foo.foo_inner, T2)
+
+@test Base.return_types(f30394, (Foo30394_2, Type{Base30394})) == Any[Base30394]