fix bug with max_values in union! (#30315)

(cherry picked from commit f49cb42)

base/abstractset.jl

@@ -67,9 +67,11 @@ function union!(s::AbstractSet, sets...)
 end
 
 max_values(::Type) = typemax(Int)
-max_values(T::Type{<:Union{Nothing,BitIntegerSmall}}) = 1 << (8*sizeof(T))
-max_values(T::Union) = max(max_values(T.a), max_values(T.b))
+max_values(T::Union{map(X -> Type{X}, BitIntegerSmall_types)...}) = 1 << (8*sizeof(T))
+# saturated addition to prevent overflow with typemax(Int)
+max_values(T::Union) = max(max_values(T.a), max_values(T.b), max_values(T.a) + max_values(T.b))
 max_values(::Type{Bool}) = 2
+max_values(::Type{Nothing}) = 1
 
 function union!(s::AbstractSet{T}, itr) where T
     haslength(itr) && sizehint!(s, length(s) + length(itr))

test/sets.jl

@@ -613,6 +613,27 @@ end
     end
 end
 
+@testset "optimized union! with max_values" begin
+    # issue #30315
+    T = Union{Nothing, Bool}
+    @test Base.max_values(T) == 3
+    d = Set{T}()
+    union!(d, (nothing, true, false))
+    @test length(d) == 3
+    @test d == Set((nothing, true, false))
+    @test nothing in d
+    @test true    in d
+    @test false   in d
+
+    for X = (Int8, Int16, Int32, Int64)
+        @test Base.max_values(Union{Nothing, X}) == (sizeof(X) < sizeof(Int) ?
+                                                     2^(8*sizeof(X)) + 1 :
+                                                     typemax(Int))
+    end
+    # this does not account for non-empty intersections of the unioned types
+    @test Base.max_values(Union{Int8,Int16}) == 2^8 + 2^16
+end
+
 struct OpenInterval{T}
     lower::T
     upper::T