fix JuliaLang#40336, limit single-level nested Type signature

Previously `type_more_complex` returns `false` for `Type{Type{...}}`
compared against `Type{...}`. AFAIU this should return `true`, and
the change will fix JuliaLang#40336.

base/compiler/typelimits.jl

@@ -263,6 +263,7 @@ function type_more_complex(@nospecialize(t), @nospecialize(c), sources::SimpleVe
         end
         if isType(t) # allow taking typeof any source type anywhere as Type{...}, as long as it isn't nesting Type{Type{...}}
             tt = unwrap_unionall(t.parameters[1])
+            tt === c && return true # nesting one level
             if isa(tt, DataType) && !isType(tt)
                 is_derived_type_from_any(tt, sources, depth) || return true
                 return false

test/compiler/inference.jl

@@ -44,6 +44,13 @@ let t = Tuple{Ref{T},T,T} where T, c = Tuple{Ref, T, T} where T # #36407
     @test t <: Core.Compiler.limit_type_size(t, c, Union{}, 1, 100)
 end
 
+let # 40336
+    t = Type{Type{Int}}
+    c = Type{Int}
+    r = Core.Compiler.limit_type_size(t, c, c, 100, 100)
+    @test t !== r && t <: r
+end
+
 @test Core.Compiler.unionlen(Union{}) == 1
 @test Core.Compiler.unionlen(Int8) == 1
 @test Core.Compiler.unionlen(Union{Int8, Int16}) == 2
@@ -3171,8 +3178,6 @@ end
 end
 
 @testset "constant prop' for union split signature" begin
-    anonymous_module() = Core.eval(@__MODULE__, :(module $(gensym()) end))::Module
-
     # indexing into tuples really relies on constant prop', and we will get looser result
     # (`Union{Int,String,Char}`) if constant prop' doesn't happen for splitunion signatures
     tt = (Union{Tuple{Int,String},Tuple{Int,Char}},)
@@ -3191,7 +3196,7 @@ end
         b
     end == Any[Union{String,Char}]
 
-    @test (@eval anonymous_module() begin
+    @test (@eval Module() begin
         struct F32
             val::Float32
             _v::Int
@@ -3205,7 +3210,7 @@ end
         end
     end) == Any[Union{Float32,Float64}]
 
-    @test (@eval anonymous_module() begin
+    @test (@eval Module() begin
         struct F32
             val::Float32
             _v
@@ -3243,3 +3248,25 @@ end
         Some(0x2)
     end
 end == [Union{Some{Float64}, Some{Int}, Some{UInt8}}]
+
+# https://github.com/JuliaLang/julia/issues/40336
+@testset "make sure a call with signatures with recursively nested Types terminates" begin
+    @test @eval Module() begin
+        f(@nospecialize(t)) = f(Type{t})
+
+        code_typed() do
+            f(Int)
+        end
+        true
+    end
+
+    @test @eval Module() begin
+        f(@nospecialize(t)) = tdepth(t) == 10 ? t : f(Type{t})
+        tdepth(@nospecialize(t)) = isempty(t.parameters) ? 1 : 1+tdepth(t.parameters[1])
+
+        code_typed() do
+            f(Int)
+        end
+        true
+    end
+end