diff --git a/test/numbers.jl b/test/numbers.jl
index 019ffcffcb695..c447a730b4f01 100644
--- a/test/numbers.jl
+++ b/test/numbers.jl
@@ -2771,7 +2771,8 @@ let types = (Base.BitInteger_types..., BigInt, Bool,
     for S in types
         for op in (+, -)
             if S === Float16 # type instability here?
-                @test_broken @inferred Base.promote_op(op, S)
+                # broken, fixme then remove this branch
+                @test_throws ErrorException @inferred(Base.promote_op(op, S))
                 T = Base.promote_op(op, S)
             else
                 T = @inferred Base.promote_op(op, S)
@@ -2785,11 +2786,31 @@ let types = (Base.BitInteger_types..., BigInt, Bool,
 
     for R in types, S in types
         for op in (+, -, *, /, ^)
-            if R in (Float16, Complex{Float16}) || S in (Float16, Complex{Float16})
-                @test_broken @inferred Base.promote_op(op, R, S)
+            if R === Float16 || S === Float16
+                # broken, fixme then remove this branch
+                @test_throws ErrorException @inferred(Base.promote_op(op, R, S))
                 T = Base.promote_op(op, R, S)
-                @test_broken @inferred op(one(R), one(S))
-                t = op(one(R), one(S))
+                if ((R === Bool || S === Bool) && op in (+, *)) ||
+                       ((S in (Rational{Int}, Complex{Float16})) && op === ^) ||
+                       (R === Complex{Float16} && op === ^)
+                    @test_throws ErrorException @inferred(op(one(R), one(S)))
+                    t = op(one(R), one(S))
+                else
+                    t = @inferred op(one(R), one(S))
+                end
+            elseif R === Complex{Float16} || S === Complex{Float16}
+                # broken, fixme then remove this branch too
+                if (R === Bool && op in (+, *, /, ^)) ||
+                        (S === Bool && op in (+, *)) ||
+                        (S in (R, Rational{Int}) && op === ^)
+                    @test_throws ErrorException @inferred(Base.promote_op(op, R, S))
+                    T = Base.promote_op(op, R, S)
+                    @test_throws ErrorException @inferred(op(one(R), one(S)))
+                    t = op(one(R), one(S))
+                else
+                    T = @inferred Base.promote_op(op, R, S)
+                    t = @inferred op(one(R), one(S))
+                end
             else
                 T = @inferred Base.promote_op(op, R, S)
                 t = @inferred op(one(R), one(S))
@@ -2805,7 +2826,8 @@ let types = (Base.BitInteger_types..., BigInt, Bool,
     for S in types, T in types
         for op in (<, >, <=, >=, (==))
             if S === Float16 || T === Float16
-                @test_broken @inferred(Base.promote_op(op, S, T))
+                # broken, fixme then remove this branch
+                @test_throws ErrorException @inferred(Base.promote_op(op, S, T))
                 @test Base.promote_op(op, S, T) === Bool
             else
                 @test @inferred(Base.promote_op(op, S, T)) === Bool