diff --git a/test/openssl/test_bn.rb b/test/openssl/test_bn.rb
index 0e9b13087..ad4141d74 100644
--- a/test/openssl/test_bn.rb
+++ b/test/openssl/test_bn.rb
@@ -296,7 +296,10 @@ def test_get_flags_and_set_flags
 
     # mod_exp fails when m is even and any argument has CONSTTIME flag
     m = OpenSSL::BN.new(98)
-    assert_raise(OpenSSL::BNError) { b.mod_exp(e, m) }
+    assert_raise(OpenSSL::BNError) do
+      b.mod_exp(e, m)
+      raise OpenSSL::BNError if !OpenSSL.errors.empty?
+    end
 
     # It looks like flags cannot be removed once enabled
     e.set_flags(0)