add example of unstable batchnorm

ROCm · Oct 19, 2023 · b2ce275 · b2ce275
1 parent 92deecf
commit b2ce275
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 3 deletions.
diff --git a/test/gtest/bn.hpp b/test/gtest/bn.hpp
@@ -159,9 +159,9 @@ struct BNBwdTest : public ::testing::TestWithParam<std::tuple<BNTestCase, miopen
                               DscaleDbiasDataType,
                               MeanVarDataType>(bn_bwd_test_data);
 
-        test::CompareTensor<DxDataType>(bn_bwd_test_data.output, bn_bwd_test_data.ref_out, 5e-4);
-        test::CompareTensor<DxDataType>(bn_bwd_test_data.dScale, bn_bwd_test_data.dScale_ref, 5e-4);
-        test::CompareTensor<DxDataType>(bn_bwd_test_data.dBias, bn_bwd_test_data.dBias_ref, 5e-4);
+        test::CompareTensor<DxDataType>(bn_bwd_test_data.output, bn_bwd_test_data.ref_out, 1e-4);
+        test::CompareTensor<DxDataType>(bn_bwd_test_data.dScale, bn_bwd_test_data.dScale_ref, 1e-4);
+        test::CompareTensor<DxDataType>(bn_bwd_test_data.dBias, bn_bwd_test_data.dBias_ref, 1e-4);
     }
 
     BNTestCase bn_config;

diff --git a/test/gtest/bn_test_data.hpp b/test/gtest/bn_test_data.hpp
@@ -35,6 +35,12 @@
 #include "tensor_util.hpp"
 #include "get_handle.hpp"
 
+// setting this to "enabled" makes ./bin/test_bn_bwd --gtest_filter=*BnBwdCKFloat/0 failed
+// and ./bin/test_bn_bwd --gtest_filter=*BnBwdCKFloat/24 (25 and 26) passed
+// while setting it to "true" makes opposite: 0 starts to pass and 24, 25, 26 start to fail
+// the other cases are not affected
+// the same problem happens for half precision as well
+MIOPEN_DECLARE_ENV_VAR(MIOPEN_DEBUG_UNSTABLE_BN)
 struct BNTestCase
 {
     size_t N;
@@ -301,6 +307,14 @@ struct BNBwdTestData : public BNTestData<XDataType, DyDataType, TConfig>
     void InitTensorsWithRandValue()
     {
         auto gen_value = [](auto...) {
+            if(miopen::IsEnabled(MIOPEN_DEBUG_UNSTABLE_BN{}))
+            {
+                // just advace PRNG to get slighly different sequence. but
+                // but with the same probability distribution and so on
+                prng::gen_canonical<int>();
+                prng::gen_canonical<int>();
+                prng::gen_canonical<int>();
+            }
             return prng::gen_descreet_uniform_sign<ScaleDataType>(1e-2, 100);
         };
         dy.generate(gen_value);