From 1ab7be6d89146cc15b9563f563c7526a5f2a2f08 Mon Sep 17 00:00:00 2001
From: Jameson Nash <vtjnash@gmail.com>
Date: Wed, 2 Mar 2022 16:31:15 -0500
Subject: [PATCH] optimizer: bail out of inlining if ir_inline_unionsplit will
 fail

Intersection cannot deal with this `metharg`, so it does not simplify
the type at all when handling this case. This can cause us to run into
an assertion later, where we assume the intersection of a non-Varags
type will always return a simple DataType without Varargs.

Fixes #44238

    atype = Tuple{typeof(Base.similar),
      Tuple{Union{Polyhedra.Polyhedron{T}, Polyhedra.Representation{T}} where T},
      Array{_A, 1} where _A,
      Array{_C, 1} where _C,
      Array{_B, 1} where _B}

    metharg = Tuple{typeof(Base.similar),
      Tuple{Vararg{Union{Polyhedra.Polyhedron{T}, Polyhedra.Representation{T}} where T}},
      Vararg{Union{Union{AbstractArray{var"#s14", 1}, Polyhedra.AbstractRepIterator{var"#s13", var"#s14"} where var"#s13", Polyhedra.AllRepIterator{var"#s14", var"#s14", LinElemT, LRT, RT} where RT<:Polyhedra.AbstractRepIterator{var"#s14", var"#s14"} where LRT<:Polyhedra.AbstractRepIterator{var"#s14", LinElemT} where LinElemT where var"#s14"} where var"#s14"<:(Polyhedra.HyperPlane{T, AT} where AT<:AbstractArray{T, 1}), Union{AbstractArray{var"#s14", 1}, Polyhedra.AbstractRepIterator{var"#s13", var"#s14"} where var"#s13", Polyhedra.AllRepIterator{var"#s14", var"#s14", LinElemT, LRT, RT} where RT<:Polyhedra.AbstractRepIterator{var"#s14", var"#s14"} where LRT<:Polyhedra.AbstractRepIterator{var"#s14", LinElemT} where LinElemT where var"#s14"} where var"#s14"<:(Polyhedra.HalfSpace{T, AT} where AT<:AbstractArray{T, 1}), Union{AbstractArray{var"#s14", 1}, Polyhedra.AbstractRepIterator{var"#s13", var"#s14"} where var"#s13", Polyhedra.AllRepIterator{var"#s14", var"#s14", LinElemT, LRT, RT} where RT<:Polyhedra.AbstractRepIterator{var"#s14", var"#s14"} where LRT<:Polyhedra.AbstractRepIterator{var"#s14", LinElemT} where LinElemT where var"#s14"} where var"#s14"<:AbstractArray{T, 1}, Union{AbstractArray{var"#s14", 1}, Polyhedra.AbstractRepIterator{var"#s13", var"#s14"} where var"#s13", Polyhedra.AllRepIterator{var"#s14", var"#s14", LinElemT, LRT, RT} where RT<:Polyhedra.AbstractRepIterator{var"#s14", var"#s14"} where LRT<:Polyhedra.AbstractRepIterator{var"#s14", LinElemT} where LinElemT where var"#s14"} where var"#s14"<:(Polyhedra.Line{T, AT} where AT<:AbstractArray{T, 1}), Union{AbstractArray{var"#s14", 1}, Polyhedra.AbstractRepIterator{var"#s13", var"#s14"} where var"#s13", Polyhedra.AllRepIterator{var"#s14", var"#s14", LinElemT, LRT, RT} where RT<:Polyhedra.AbstractRepIterator{var"#s14", var"#s14"} where LRT<:Polyhedra.AbstractRepIterator{var"#s14", LinElemT} where LinElemT where var"#s14"} where var"#s14"<:(Polyhedra.Ray{T, AT} where AT<:AbstractArray{T, 1})} where T}}

Currently `typeintersection(atype, metharg) === metharg`
---
 base/compiler/ssair/inlining.jl | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl
index fc3c3a60115e6..a8b3ce14ef82b 100644
--- a/base/compiler/ssair/inlining.jl
+++ b/base/compiler/ssair/inlining.jl
@@ -820,9 +820,10 @@ end
 function analyze_method!(match::MethodMatch, argtypes::Vector{Any},
                          flag::UInt8, state::InliningState)
     method = match.method
+    spec_types = match.spec_types
     methsig = method.sig
 
-    # Check that we habe the correct number of arguments
+    # Check that we have the correct number of arguments
     na = Int(method.nargs)
     npassedargs = length(argtypes)
     if na != npassedargs && !(na > 0 && method.isva)
@@ -832,6 +833,13 @@ function analyze_method!(match::MethodMatch, argtypes::Vector{Any},
         # call this function
         return nothing
     end
+    if !match.fully_covers
+        # type-intersection was not able to give us a simple list of types, so
+        # ir_inline_unionsplit won't be able to deal with inlining this
+        if !(spec_types isa DataType && length(spec_types.parameters) == length(argtypes) && !isvarargtype(spec_types.parameters[end]))
+            return nothing
+        end
+    end
 
     # Bail out if any static parameters are left as TypeVar
     validate_sparams(match.sparams) || return nothing