From 27d04f53e266b64d72b4aeddc1eb2cd9bb3f7cf7 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Tue, 19 Mar 2024 15:50:13 -0700 Subject: [PATCH] Adjust test tolerance (#19947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description Improve the precision of tests. Changes include: (1) Update checkers.cc to use consistent default tolerance. (2) Allow different default tolerances for different providers at runtime (Previously, threshold of a test is decided during compiling). (3) Explicitly set absolute and relative error tolerances for tests that failed to pass new default threshold. #### Default Thresholds Change Note that the formula of testing is `abs(expected - value) < absolute + relative * expected` Default test thresholds when both absolute and relative tolerance are not set: type | provider | absolute (before) | absolute (after) | relative (before) | relative (after) -- | -- | -- | -- | -- | -- double | CPU | 0.001 | 0.00001 | 0 | 0.00001 double | CUDA | 0.005 | 0.00001 | 0 | 0.00001 double | TRT | 0.005 | 0.00001 | 0 | 0.00001 double | ROCM | 0.005 | 0.00001 | 0 | 0.00001 double | DML | 0.005 | 0.00001 | 0 | 0.00001   |   |   |   |   |   float | CPU | 0.0001 | 0.00001 | 0 | 0.0001 float | CUDA | 0.005 | 0.00001 | 0 | 0.0001 float | TRT | 0.005 | 0.00001 | 0 | 0.0001 float | ROCM | 0.005 | 0.00001 | 0 | 0.0001 float | DML | 0.005 | 0.00001 | 0 | 0.0001 float | Training* | 0.005 | 0.001 | 0 | 0.0001   |   |   |   |   |   half | CPU | 0.001 | 0.0025 | 0 | 0.001 half | CUDA | 0.005 | 0.0025 | 0 | 0.001 half | TRT | 0.005 | 0.0025 | 0 | 0.001 half | ROCM | 0.005 | 0.0025 | 0 | 0.001 half | DML | 0.02 | 0.005 | 0 | 0.001 half | Training* | 0.005 | 0.005 | 0 | 0.001   |   |   |   |   |   bfloat16 | CPU | 0.0001 | 0.02 | 0 | 0.01 bfloat16 | CUDA | 0.0001 | 0.02 | 0.05 | 0.01 bfloat16 | TRT | 0.0001 | 0.02 | 0.05 | 0.01 bfloat16 | ROCM | 0.0001 | 0.02 | 0.05 | 0.01 bfloat16 | DML | 0.0001 | 0.02 | 0.05 | 0.01 bfloat16 | Training* | 0.0001 | 0.02 | 0.05 | 0.01 *Training mean a build flag ENABLE_TRAINING_CORE is defined. The provider can be any one. #### Threshold for provider Previously, the threshold might change according to build flags: ``` #if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) constexpr float threshold = 0.005f; #else constexpr float threshold = 0.0001f; #endif ``` For a cpu only build, the threshold is 0.0001. For a cuda build, the threshold for CPU provider (some tests in cuda build actually run with CPU provider) is changed to 0.005. After this change, the threshold only depends on data type and provider used in the test. It will not change by build flags for non-training builds. Default thresholds for training might be different from inference (please refer to the above table). There are a few factors there: Training has gradient outputs; TF32 is not disabled in training; Some training tests has iterations, and error might accumulate. How to set different thresholds based on these factors could be a future task. --- .../test/contrib_ops/attention_op_test.cc | 9 ++ .../contrib_ops/decoder_attention_op_test.cc | 7 +- ...oder_masked_multihead_attention_op_test.cc | 6 +- onnxruntime/test/contrib_ops/fft_op_test.cc | 2 + .../test/contrib_ops/gemm_fastgelu_op_test.cc | 6 +- .../test/contrib_ops/gridsample_test.cc | 1 + .../test/contrib_ops/layer_norm_op_test.cc | 6 + .../matmul_integer_to_float_test.cc | 2 +- onnxruntime/test/contrib_ops/moe_test.cc | 2 + .../packed_multihead_attention_op_test.cc | 2 + .../contrib_ops/quantize_attention_op_test.cc | 2 + onnxruntime/test/providers/base_tester.cc | 14 +++ onnxruntime/test/providers/base_tester.h | 11 ++ onnxruntime/test/providers/checkers.cc | 117 ++++++++++-------- .../cpu/activation/activation_op_test.h | 5 + .../cpu/math/element_wise_ops_test.cc | 9 +- .../providers/cpu/math/logsoftmax_test.cc | 9 +- .../providers/cpu/nn/batch_norm_op_test.cc | 9 +- .../test/providers/cpu/nn/pool_op_test.cc | 1 + .../cpu/object_detection/roialign_test.cc | 2 + .../cpu/rnn/deep_cpu_lstm_op_test.cc | 2 + .../providers/cpu/tensor/affine_grid_test.cc | 17 +++ .../mean_variance_normalization_test.cc | 5 + .../test/gradient/optimizer_ops_test.cc | 19 +++ .../cpu/nn/batchnorm_internal_test.cc | 2 + .../cuda/batch_norm_internal_test.cc | 1 + 26 files changed, 204 insertions(+), 64 deletions(-) diff --git a/onnxruntime/test/contrib_ops/attention_op_test.cc b/onnxruntime/test/contrib_ops/attention_op_test.cc index 7fe70fd2d6f09..a8e2fccdd0462 100644 --- a/onnxruntime/test/contrib_ops/attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/attention_op_test.cc @@ -227,6 +227,12 @@ static void RunAttentionTest( tester.AddOptionalInputEdge(); } + if (use_float16) { + tester.SetOutputTolerance(0.005f); + } else { + tester.SetOutputTolerance(0.001f, 0.001f); + } + if (enable_cuda) { std::vector> execution_providers; execution_providers.push_back(DefaultCudaExecutionProvider()); @@ -254,6 +260,9 @@ static void RunAttentionTest( if (enable_dml) { std::vector> execution_providers; execution_providers.push_back(DefaultDmlExecutionProvider()); + if (use_float16) { + tester.SetOutputTolerance(0.02f); + } tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } } diff --git a/onnxruntime/test/contrib_ops/decoder_attention_op_test.cc b/onnxruntime/test/contrib_ops/decoder_attention_op_test.cc index 88a2bdf6a4849..8a37ef921fd2b 100644 --- a/onnxruntime/test/contrib_ops/decoder_attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/decoder_attention_op_test.cc @@ -31,10 +31,8 @@ static void RunAttentionTest( const std::vector* new_value_cache = nullptr, const std::vector* key_cache = nullptr, const std::vector* value_cache = nullptr, - const std::initializer_list* key_padding_mask_data = nullptr, - bool use_float16 = false) { - int min_cuda_architecture = use_float16 ? 530 : 0; - bool enable_cuda = HasCudaEnvironment(min_cuda_architecture); + const std::initializer_list* key_padding_mask_data = nullptr) { + bool enable_cuda = HasCudaEnvironment(0); bool enable_rocm = (nullptr != DefaultRocmExecutionProvider().get()); bool enable_cpu = false; @@ -99,6 +97,7 @@ static void RunAttentionTest( tester.AddOutput("new_key_cache", output_cache_dims, *new_key_cache); tester.AddOutput("new_value_cache", output_cache_dims, *new_value_cache); } + tester.SetOutputTolerance(0.001f, 0.001f); std::vector> execution_providers; if (enable_cuda) { diff --git a/onnxruntime/test/contrib_ops/decoder_masked_multihead_attention_op_test.cc b/onnxruntime/test/contrib_ops/decoder_masked_multihead_attention_op_test.cc index acaae2dcd9712..17c9e8592f64e 100644 --- a/onnxruntime/test/contrib_ops/decoder_masked_multihead_attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/decoder_masked_multihead_attention_op_test.cc @@ -754,9 +754,10 @@ TEST(DecoderMaskedSelfAttentionTest, Test_fp32) { // Output(s) tester.AddOutput("output", input_dims, output); - tester.AddOutput("present", past_dims, present); + tester.SetOutputTolerance(0.001f, 0.001f); + // Run - Regular kernel execution path { std::vector> execution_providers; @@ -897,9 +898,10 @@ TEST(DecoderMaskedSelfAttentionTest, Test_fp16) { // Output(s) tester.AddOutput("output", input_dims, output); - tester.AddOutput("present", past_dims, present); + tester.SetOutputTolerance(0.005f); + // Run - Regular kernel execution path { std::vector> execution_providers; diff --git a/onnxruntime/test/contrib_ops/fft_op_test.cc b/onnxruntime/test/contrib_ops/fft_op_test.cc index 56a6466c760f6..7a6b6cca6425a 100644 --- a/onnxruntime/test/contrib_ops/fft_op_test.cc +++ b/onnxruntime/test/contrib_ops/fft_op_test.cc @@ -25,6 +25,7 @@ TEST(ContribOpTest, Rfft) { // Target values conputed using PyTorch torch.fft.rfft(X, dim=-1, norm="backward") test.AddInput("X", {4, 4}, {0.8129f, 1.3108f, -0.8790f, -1.2046f, 0.1661f, -0.9831f, 0.5879f, 0.4918f, 1.2506f, 0.7244f, -2.6260f, -1.1268f, -1.6885f, 1.0439f, -0.2595f, 1.8780f}); test.AddOutput("Y", {4, 3, 2}, {0.0400f, 0.0000f, 1.6919f, -2.5154f, -0.1722f, 0.0000f, 0.2627f, 0.0000f, -0.4218f, 1.4748f, 1.2454f, 0.0000f, -1.7779f, 0.0000f, 3.8766f, -1.8512f, -0.9730f, 0.0000f, 0.9740f, 0.0000f, -1.4290f, 0.8341f, -4.8699f, 0.0000f}); + test.SetOutputTolerance(0.0001f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } @@ -45,6 +46,7 @@ TEST(ContribOpTest, Irfft) { test.AddAttribute("normalized", static_cast(0)); test.AddInput("X", {4, 3, 2}, {0.0400f, 0.0000f, 1.6919f, -2.5154f, -0.1722f, 0.0000f, 0.2627f, 0.0000f, -0.4218f, 1.4748f, 1.2454f, 0.0000f, -1.7779f, 0.0000f, 3.8766f, -1.8512f, -0.9730f, 0.0000f, 0.9740f, 0.0000f, -1.4290f, 0.8341f, -4.8699f, 0.0000f}); test.AddOutput("Y", {4, 4}, {0.8129f, 1.3108f, -0.8790f, -1.2046f, 0.1661f, -0.9831f, 0.5879f, 0.4918f, 1.2506f, 0.7244f, -2.6260f, -1.1268f, -1.6885f, 1.0439f, -0.2595f, 1.8780f}); + test.SetOutputTolerance(0.0001f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers); } } // namespace test diff --git a/onnxruntime/test/contrib_ops/gemm_fastgelu_op_test.cc b/onnxruntime/test/contrib_ops/gemm_fastgelu_op_test.cc index a24f3b6b441e1..d9d2681dd3b3f 100644 --- a/onnxruntime/test/contrib_ops/gemm_fastgelu_op_test.cc +++ b/onnxruntime/test/contrib_ops/gemm_fastgelu_op_test.cc @@ -50,6 +50,8 @@ static void RunGemmFastGeluGpuTest(const std::vector& input_data, const s tester.AddOutput("Y", output_dims, output_data); } + tester.SetOutputTolerance(use_float16 ? 0.005f : 0.0025f); + tester.Config(run_with_tunable_op) .RunWithConfig(); } @@ -154,7 +156,7 @@ TEST(GemmFastGeluTest, GemmFastGeluWithoutBiasFloat16) { RunGemmFastGeluGpuTest(input_data, weight_data, bias_data, output_data, input_dims, weight_dims, bias_dims, output_dims, - false); + false, true); } TEST(GemmFastGeluTest, GemmFastGeluWithBiasFloat16) { @@ -189,7 +191,7 @@ TEST(GemmFastGeluTest, GemmFastGeluWithBiasFloat16) { RunGemmFastGeluGpuTest(input_data, weight_data, bias_data, output_data, input_dims, weight_dims, bias_dims, output_dims, - true); + true, true); } TEST(GemmFastGeluTest, GemmFastGeluWithBias_bfloat16) { diff --git a/onnxruntime/test/contrib_ops/gridsample_test.cc b/onnxruntime/test/contrib_ops/gridsample_test.cc index 46ed04301a9e8..d970178e29ab8 100644 --- a/onnxruntime/test/contrib_ops/gridsample_test.cc +++ b/onnxruntime/test/contrib_ops/gridsample_test.cc @@ -126,6 +126,7 @@ TEST(GridsampleContribOpTest, gridsample_mode_bicubic) { 0.5000f, 0.5000f, 1.0000f, 1.0000f}); test.AddAttribute("mode", "bicubic"); test.AddOutput("Y", {1, 1, 2, 4}, {-0.1406f, 0.3828f, 1.7556f, 2.9688f, 2.9688f, 1.7556f, 5.1445f, 1.3906f}); + test.SetOutputTolerance(0.0001f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider}); } diff --git a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc index 98fb62e435f31..655c4951f262d 100644 --- a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc @@ -160,6 +160,7 @@ TEST(LayerNormTest, LayerNorm_Scale_Bias) { test.AddInput("gamma", {2}, {-0.6953f, 5.1824f}); test.AddInput("bias", {2}, {0.6435f, -0.3964f}); test.AddOutput("output", dims, {-0.0516f, -5.5776f, -0.0518f, -5.5788f, -0.0518f, -5.5788f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -172,6 +173,8 @@ TEST(LayerNormTest, LayerNorm_Scale_Bias_Float16Input) { test.AddInput("gamma", {2}, {-0.6953f, 5.1824f}); test.AddInput("bias", {2}, {0.6435f, -0.3964f}); test.AddOutput("output", dims, {-0.0516f, -5.5776f, -0.0518f, -5.5788f, -0.0518f, -5.5788f}); + test.SetOutputTolerance(0.0001f); + // TRT, DNNL, OpenVINO and NNAPI, CoreML don't support this combination of datatypes test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kDnnlExecutionProvider, kQnnExecutionProvider, @@ -228,6 +231,9 @@ TEST(LayerNormTest, LayerNorm17_double) { test.AddInput("x", dims, {1.0, 2.0, 3.0, 4.0, 5.0, 6.0}); test.AddInput("gamma", {3}, {1.0, 1.0, 1.0}); test.AddOutput("output", dims, {-1.2247, 0.0, 1.2247, -1.2247, 0.0, 1.2247}); + + test.SetOutputTolerance(0.0001f); + // DNNL does not support double test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kDnnlExecutionProvider}); } diff --git a/onnxruntime/test/contrib_ops/matmul_integer_to_float_test.cc b/onnxruntime/test/contrib_ops/matmul_integer_to_float_test.cc index 72a5ba4dcefbf..8d7629b5fda1c 100644 --- a/onnxruntime/test/contrib_ops/matmul_integer_to_float_test.cc +++ b/onnxruntime/test/contrib_ops/matmul_integer_to_float_test.cc @@ -127,7 +127,7 @@ void TestMatMulIntegerToFloat(bool is_matrix_b_constant, if (std::is_same_v) { test.AddOutput("Y", {M, N}, Y_data); - test.SetOutputAbsErr("Y", 0.0001f); + test.SetOutputAbsErr("Y", 0.001f); test.SetOutputRelErr("Y", 0.02f); } else { test.AddOutput("Y", {M, N}, ToFloat16(Y_data)); diff --git a/onnxruntime/test/contrib_ops/moe_test.cc b/onnxruntime/test/contrib_ops/moe_test.cc index ebb0261deefa5..e88ef7794cd07 100644 --- a/onnxruntime/test/contrib_ops/moe_test.cc +++ b/onnxruntime/test/contrib_ops/moe_test.cc @@ -47,6 +47,7 @@ static void RunMoETest( tester.AddInput("fc1_experts_bias", fc1_experts_bias_dims, ToFloat16(fc1_experts_bias)); tester.AddInput("fc2_experts_bias", fc2_experts_bias_dims, ToFloat16(fc2_experts_bias)); tester.AddOutput("output", output_dims, ToFloat16(output_data)); + tester.SetOutputTolerance(0.005f); } else { tester.AddInput("input", input_dims, input); tester.AddInput("router_probs", router_probs_dims, router_probs); @@ -55,6 +56,7 @@ static void RunMoETest( tester.AddInput("fc1_experts_bias", fc1_experts_bias_dims, fc1_experts_bias); tester.AddInput("fc2_experts_bias", fc2_experts_bias_dims, fc2_experts_bias); tester.AddOutput("output", output_dims, output_data); + tester.SetOutputTolerance(0.001f); } std::vector> execution_providers; diff --git a/onnxruntime/test/contrib_ops/packed_multihead_attention_op_test.cc b/onnxruntime/test/contrib_ops/packed_multihead_attention_op_test.cc index 22253955566f2..5f811c8cf35f6 100644 --- a/onnxruntime/test/contrib_ops/packed_multihead_attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/packed_multihead_attention_op_test.cc @@ -107,6 +107,7 @@ static void RunPackedMultiHeadAttentionTest( } tester.AddOutput("output", output_dims, ToFloat16(output_data)); + tester.SetOutputTolerance(0.005f); } else { if (is_packed_qkv) { tester.AddInput("query", packed_qkv_dims, query_data); @@ -131,6 +132,7 @@ static void RunPackedMultiHeadAttentionTest( } tester.AddOutput("output", output_dims, output_data); + tester.SetOutputTolerance(0.001f, 0.001f); } std::vector> execution_providers; diff --git a/onnxruntime/test/contrib_ops/quantize_attention_op_test.cc b/onnxruntime/test/contrib_ops/quantize_attention_op_test.cc index fd222583ac67f..54dd831fe2fc2 100644 --- a/onnxruntime/test/contrib_ops/quantize_attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_attention_op_test.cc @@ -90,11 +90,13 @@ void RunQAttention(const std::vector& input_data, tester.AddInput("input_scale", {1}, ToFloat16({input_quant_params.scale})); tester.AddInput("weight_scale", {1}, ToFloat16({weight_quant_params.scale})); tester.AddOutput("output", output_dims, ToFloat16(output_data)); + tester.SetOutputTolerance(0.01f); } else { tester.AddInput("bias", bias_dims, bias_data); tester.AddInput("input_scale", {1}, {input_quant_params.scale}); tester.AddInput("weight_scale", {1}, {weight_quant_params.scale}); tester.AddOutput("output", output_dims, output_data); + tester.SetOutputTolerance(0.005f); } if (mask_index_data.size() > 0) { diff --git a/onnxruntime/test/providers/base_tester.cc b/onnxruntime/test/providers/base_tester.cc index e94f8c2673be3..8d84c689cd23e 100644 --- a/onnxruntime/test/providers/base_tester.cc +++ b/onnxruntime/test/providers/base_tester.cc @@ -120,6 +120,20 @@ void BaseTester::SetOutputRelErr(const char* name, float v) { it->validation_params.relative_error = optional(v); } +void BaseTester::SetOutputTolerance(float abs_error, float rel_error) { + for (auto& output : output_data_) { + if (output.def.Exists()) { + if (abs_error >= 0.0f) { + output.validation_params.absolute_error = optional(abs_error); + } + + if (rel_error >= 0.0f) { + output.validation_params.relative_error = optional(rel_error); + } + } + } +} + std::vector BaseTester::GetDimsForProto(gsl::span dims) { std::vector dims_for_proto{dims.begin(), dims.end()}; if (add_symbolic_dim_to_tensor_data_ >= 0 && diff --git a/onnxruntime/test/providers/base_tester.h b/onnxruntime/test/providers/base_tester.h index 5607e58315a12..c276ae494df43 100644 --- a/onnxruntime/test/providers/base_tester.h +++ b/onnxruntime/test/providers/base_tester.h @@ -519,9 +519,20 @@ class BaseTester { custom_session_registries_.push_back(registry); } + // For floating types (double/float/half/bfloat16), tolerance is similar to numpy.isclose: + // absolute(expected_value - actual_value) <= abs_error + rel_error * absolute(expected_value) + // For integer types, tolerance parameters are ignored except the following cases: + // For uint8, tolerance is only applied to NNAPI/XNNPACK/DML providers. + // For int8, only abs_error is used, and rel_error is ignored. See checkers.cc for detail. + // If abs_error or rel_error is not set, a default value is used (search DefaultTolerance for detail). void SetOutputAbsErr(const char* name, float v); void SetOutputRelErr(const char* name, float v); + // Set absolute and relative tolerance for all existed outputs. + // Negative value will be ignored. + // Note that it will not set tolerance for new outputs added after this call. + void SetOutputTolerance(float abs_error, float rel_error = -1.0f); + // Number of times to call InferenceSession::Run. The same feeds are used each time. // e.g. used to verify the generator ops behave as expected void SetNumRunCalls(int n) { diff --git a/onnxruntime/test/providers/checkers.cc b/onnxruntime/test/providers/checkers.cc index c97e6d9de4911..47c18c478dd9c 100644 --- a/onnxruntime/test/providers/checkers.cc +++ b/onnxruntime/test/providers/checkers.cc @@ -20,46 +20,87 @@ struct DefaultTolerance; template <> struct DefaultTolerance { - static constexpr float absolute = 1e-6f; + static constexpr float absolute = 1e-5f; static constexpr float relative = 1e-5f; + + // Allow to have different default absolute tolerance for different providers. + static float get_absolute(const std::string& /*provider_type*/) { + return absolute; + } }; template <> struct DefaultTolerance { +#if defined(ENABLE_TRAINING) + static constexpr float absolute = 1e-3f; +#else static constexpr float absolute = 1e-5f; +#endif + static constexpr float relative = 1e-4f; + + static float get_absolute(const std::string& /*provider_type*/) { + return absolute; + } }; template <> struct DefaultTolerance { - // The thresholds are estimated with PyTorch script like the following: +#if defined(ENABLE_TRAINING) + static constexpr float absolute = 0.005f; +#else + // The thresholds for inference are estimated with PyTorch script like the following: // x = torch.rand(1000, 1000) // absolute = ((x + 1e-6).to(torch.float16) - x).abs().max() * 10 // x[abs(x) < absolute] = absolute // relative = ((x - x.to(torch.float16)) / x).abs().max() * 2 static constexpr float absolute = 0.0025f; +#endif + static constexpr float relative = 0.001f; + + static float get_absolute(const std::string& provider_type) { + if (provider_type == kDmlExecutionProvider) { + return 0.005f; + } + return absolute; + } }; template <> struct DefaultTolerance { + // The thresholds for inference are estimated with PyTorch script like the following: + // x = torch.rand(1000, 1000) + // absolute = ((x + 1e-6).to(torch.bfloat16) - x).abs().max() * 10 + // x[abs(x) < absolute] = absolute + // relative = ((x - x.to(torch.bfloat16)) / x).abs().max() * 2 static constexpr float absolute = 0.02f; static constexpr float relative = 0.01f; + + static float get_absolute(const std::string& /*provider_type*/) { + return absolute; + } +}; + +struct ToleranceParams { + float absolute; + float relative; }; template -T get_tolerance(float absolute, float relative, T expected_value) { +ToleranceParams get_tolerance_params(const ValidateOutputParams& params, const std::string& provider_type) { + ToleranceParams new_params; + new_params.absolute = params.absolute_error.has_value() ? *(params.absolute_error) : DefaultTolerance::get_absolute(provider_type); + new_params.relative = params.relative_error.has_value() ? *(params.relative_error) : DefaultTolerance::relative; + return new_params; +} + +template +T get_tolerance(const ToleranceParams& params, T expected_value) { static_assert(std::is_floating_point::value, "T must be a floating point type"); // The formula is similar to numpy.isclose: https://numpy.org/doc/stable/reference/generated/numpy.isclose.html - return static_cast(absolute) + static_cast(relative) * std::abs(expected_value); -} - -template // D is the original data type -T get_tolerance(const ValidateOutputParams& params, T expected_value) { - float absolute = (params.absolute_error.has_value() ? *(params.absolute_error) : DefaultTolerance::absolute); - float relative = (params.relative_error.has_value() ? *(params.relative_error) : DefaultTolerance::relative); - return get_tolerance(absolute, relative, expected_value); + return static_cast(params.absolute) + static_cast(params.relative) * std::abs(expected_value); } template @@ -201,7 +242,10 @@ struct TensorCheck { cur_actual = actual.template Data(); } - const bool has_abs_err = params.absolute_error.has_value(); + // When absolute error is less than 1 for int8, it has same effect as no tolerance. + const bool has_abs_err = params.absolute_error.has_value() && *(params.absolute_error) >= 1.0f; + + // TODO: the relative error is not used for int8 yet. if (has_abs_err) { double threshold = *(params.absolute_error); @@ -221,11 +265,9 @@ struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, - const std::string& /*provider_type*/) const { + const std::string& provider_type) const { auto size = actual.Shape().Size(); - const bool has_tolerance = params.absolute_error.has_value() || params.relative_error.has_value(); - // deal with rare cases in which order of output data from a kernel MAY be // undefined Tensor expected_sorted, actual_sorted; @@ -240,10 +282,7 @@ struct TensorCheck { cur_actual = actual.Data(); } - double threshold = 0.001; -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) - threshold = 0.005; -#endif + auto tolerance_params = get_tolerance_params(params, provider_type); for (int64_t i = 0; i < size; ++i) { // NOTE: Check isnan first to work around MSVC linker bug when /LTCG:incremental is specified. @@ -253,7 +292,7 @@ struct TensorCheck { } else if (std::isinf(cur_expected[i])) { // Test infinity for equality EXPECT_EQ(cur_expected[i], cur_actual[i]) << "Expected infinity. i:" << i; } else { - double tolerance = has_tolerance ? get_tolerance(params, cur_expected[i]) : threshold; + double tolerance = get_tolerance(tolerance_params, cur_expected[i]); EXPECT_NEAR(cur_expected[i], cur_actual[i], tolerance) << "i:" << i; } } @@ -264,9 +303,7 @@ template void InternalNumericalCheck(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, - const std::string& /*provider_type*/) { - const bool has_tolerance = params.absolute_error.has_value() || params.relative_error.has_value(); - + const std::string& provider_type) { // deal with rare cases in which order of output data from a kernel MAY be // undefined Tensor expected_sorted, actual_sorted; @@ -282,11 +319,7 @@ void InternalNumericalCheck(const Tensor& expected, cur_actual = actual.Data(); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) - constexpr float threshold = 0.005f; -#else - constexpr float threshold = 0.0001f; -#endif + auto tolerance_params = get_tolerance_params(params, provider_type); for (int64_t i = 0; i < size; ++i) { // NOTE: Check isnan first to work around MSVC linker bug when /LTCG:incremental is specified. @@ -296,7 +329,7 @@ void InternalNumericalCheck(const Tensor& expected, } else if (std::isinf(cur_expected[i])) { // Test infinity for equality EXPECT_EQ(cur_expected[i], cur_actual[i]) << "Expected infinity. i:" << i; } else { - T tolerance = has_tolerance ? get_tolerance(params, cur_expected[i]) : threshold; + T tolerance = get_tolerance(tolerance_params, cur_expected[i]); EXPECT_NEAR(cur_expected[i], cur_actual[i], tolerance) << "i:" << i; } } @@ -317,7 +350,7 @@ struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, - const std::string& /*provider_type*/) const { + const std::string& provider_type) const { auto* cur_expected = expected.Data(); auto* cur_actual = actual.Data(); auto size = actual.Shape().Size(); @@ -333,21 +366,15 @@ struct TensorCheck { sort_expected_and_actual_buffers(f_expected, f_actual); } - const bool has_tolerance = params.absolute_error.has_value() || params.relative_error.has_value(); + auto tolerance_params = get_tolerance_params(params, provider_type); - float threshold = 0.001f; -#if defined(USE_TENSORRT) || defined(ENABLE_TRAINING_CORE) || defined(USE_CUDA) || defined(USE_ROCM) - threshold = 0.005f; -#elif defined(USE_DML) - threshold = 0.02f; -#endif for (int64_t i = 0; i < size; ++i) { if (std::isnan(f_expected[i])) { EXPECT_TRUE(std::isnan(f_expected[i])) << "Expected NaN. i:" << i; } else if (std::isinf(f_expected[i])) { // Test infinity for equality EXPECT_EQ(f_expected[i], f_actual[i]) << "Expected infinity. i:" << i; } else { - float tolerance = has_tolerance ? get_tolerance(params, f_expected[i]) : threshold; + float tolerance = get_tolerance(tolerance_params, f_expected[i]); EXPECT_NEAR(f_expected[i], f_actual[i], tolerance) << "i:" << i; } } @@ -359,7 +386,7 @@ struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, - const std::string& /*provider_type*/) const { + const std::string& provider_type) const { auto* cur_expected = expected.Data(); auto* cur_actual = actual.Data(); auto size = actual.Shape().Size(); @@ -375,13 +402,7 @@ struct TensorCheck { sort_expected_and_actual_buffers(f_expected, f_actual); } - const bool has_tolerance = params.absolute_error.has_value() || params.relative_error.has_value(); - - float abs_threshold = 0.0001f; - float rel_threshold = 0.001f; -#if defined(USE_TENSORRT) || defined(ENABLE_TRAINING_CORE) || defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_DML) || defined(USE_DNNL) - rel_threshold = 0.05f; // expect at least 95% close -#endif + auto tolerance_params = get_tolerance_params(params, provider_type); for (int64_t i = 0; i < size; ++i) { if (std::isnan(f_expected[i])) { @@ -389,9 +410,7 @@ struct TensorCheck { } else if (std::isinf(f_expected[i])) { // Test infinity for equality EXPECT_EQ(f_expected[i], f_actual[i]) << "Expected infinity. i:" << i; } else { - float tolerance = has_tolerance - ? get_tolerance(params, f_expected[i]) - : get_tolerance(abs_threshold, rel_threshold, f_expected[i]); + float tolerance = get_tolerance(tolerance_params, f_expected[i]); EXPECT_NEAR(f_expected[i], f_actual[i], tolerance) << "i:" << i; } } diff --git a/onnxruntime/test/providers/cpu/activation/activation_op_test.h b/onnxruntime/test/providers/cpu/activation/activation_op_test.h index 984b8f4437a3b..9a74d763a13e3 100644 --- a/onnxruntime/test/providers/cpu/activation/activation_op_test.h +++ b/onnxruntime/test/providers/cpu/activation/activation_op_test.h @@ -69,6 +69,11 @@ inline void TestActivationOp(const char* szOp, const std::vector> test.SetOutputRelErr("Y", .000001f); } #endif + + if (strcmp(szOp, "QuickGelu") == 0) { + test.SetOutputTolerance(0.0001f); + } + test.Run(OpTester::ExpectResult::kExpectSuccess, "", excluded_providers); } } diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index 0e99b2306873e..c73dfcbce1b53 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -2632,7 +2632,7 @@ TEST(MathOpTest, Mean_8) { #endif template -void TrigFloatTest(OpTester& test, std::initializer_list input) { +void TrigFloatTest(OpTester& test, std::initializer_list input, float abs_error = -1.0f) { std::vector dims{static_cast(input.size())}; std::vector output; @@ -2642,6 +2642,10 @@ void TrigFloatTest(OpTester& test, std::initializer_list input) { test.AddInput("X", dims, input); test.AddOutput("Y", dims, output); + if (abs_error >= 0.0f) { + test.SetOutputTolerance(abs_error); + } + test.Run(); } @@ -2719,7 +2723,8 @@ TEST(MathOpTest, Tan) { TEST(MathOpTest, Asin) { OpTester test("Asin"); - TrigFloatTest<::asinf>(test, {-1.0f, -0.5f, 0.0f, 0.5f, 1.0f}); + float abs_error = DefaultDmlExecutionProvider().get() != nullptr ? 0.0001f : -1.0f; + TrigFloatTest<::asinf>(test, {-1.0f, -0.5f, 0.0f, 0.5f, 1.0f}, abs_error); } TEST(MathOpTest, Acos) { diff --git a/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc b/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc index 273503e7bf6af..f057e4a071bd9 100644 --- a/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/logsoftmax_test.cc @@ -15,7 +15,8 @@ static void RunTest(const std::vector& x_vals, int64_t axis = 1, bool is_tensorrt_supported = true, OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, - const std::string& error_msg = "") { + const std::string& error_msg = "", + float tolerance = 0.0f) { OpTester tester("LogSoftmax", opset); if (opset < 13) { @@ -31,6 +32,10 @@ static void RunTest(const std::vector& x_vals, tester.AddInput("X", dimensions, x_vals); tester.AddOutput("Y", dimensions, expected_vals); + if (tolerance != 0.0f) { + tester.SetOutputAbsErr("Y", tolerance); + } + std::unordered_set excluded_providers; if (!is_tensorrt_supported) { excluded_providers.insert(kTensorrtExecutionProvider); @@ -62,7 +67,7 @@ TEST(LogSoftmaxOperator, LargeNumber) { -3.4401896f, -2.4401896f, -1.44018972f, -0.44018969f}; std::vector dimensions = {2, 4}; - RunTest(x_vals, expected_vals, dimensions); + RunTest(x_vals, expected_vals, dimensions, 7, 1, true, OpTester::ExpectResult::kExpectSuccess, "", 0.0005f); } // np.random.seed(123) # Use a seed so we can replicate the input and expected values here and in python diff --git a/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc b/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc index 3d30fc62a945d..d91a1de3faa6e 100644 --- a/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/batch_norm_op_test.cc @@ -905,14 +905,16 @@ TEST(BatchNormTest, ForwardTrainingTestWithSavedOutputsOpset9) { test.AddInput("var", channel_dims, {1.0f, 2.0f}); test.AddOutput("Y", input_output_dims, {0.0131f, 0.5210f, 1.7244f, 0.1387f, -0.2708f, -0.1191f, 1.2089f, -0.0922f, -0.9548f, -1.5203f, 0.9077f, -0.8298f, 0.5796f, -0.4501f, -2.0921f, 1.2358f}); - test.AddOutput("running_mean", channel_dims, {-0.1754f, 0.303106f}); test.AddOutput("running_var", channel_dims, {0.696052f, 1.41316f}); + // mean and variance of X across channel dimension // With Opset9 we output saved_inv_std instead of saved_var to match CUDA EP test.AddOutput("saved_mean", channel_dims, {-0.306f, 0.114562f}); test.AddOutput("saved_inv_std", channel_dims, {1.2288f, 0.861317f}); + test.SetOutputTolerance(0.0001f); + // exclude CUDA Execution Provider due to flakiness // exclude TRT and OpenVINO for same reasons as seen in TestBatchNorm() test.Run(OpTester::ExpectResult::kExpectSuccess, "", @@ -938,10 +940,11 @@ TEST(BatchNormTest, ForwardTrainingTestOpset14) { test.AddInput("var", channel_dims, {1.0f, 2.0f}); test.AddOutput("Y", input_output_dims, {0.0131f, 0.5210f, 1.7244f, 0.1387f, -0.2708f, -0.1191f, 1.2089f, -0.0922f, -0.9548f, -1.5203f, 0.9077f, -0.8298f, 0.5796f, -0.4501f, -2.0921f, 1.2358f}); - test.AddOutput("running_mean", channel_dims, {-0.1754f, 0.303106f}); test.AddOutput("running_var", channel_dims, {0.696052f, 1.41316f}); + test.SetOutputTolerance(0.0001f); + // exclude CUDA Execution Provider due to flakiness // exclude TRT and OpenVINO for same reasons as seen in TestBatchNorm() test.Run(OpTester::ExpectResult::kExpectSuccess, "", @@ -970,6 +973,8 @@ TEST(BatchNormTest, ForwardTrainingTestOpset15) { test.AddOutput("running_mean", channel_dims, {-0.1754f, 0.303106f}); test.AddOutput("running_var", channel_dims, {0.696052f, 1.41316f}); + test.SetOutputTolerance(0.0001f); + // Same exclusions as the opset 14 test test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kCudaNHWCExecutionProvider, kRocmExecutionProvider, diff --git a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc index e24cda17166ed..f98b18ddb17eb 100644 --- a/onnxruntime/test/providers/cpu/nn/pool_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/pool_op_test.cc @@ -888,6 +888,7 @@ TEST(PoolTest, AveragePool_IncludePadPixel) { test.AddInput("X", x_dims, x_vals); test.AddOutput("Y", expected_dims, expected_vals); + test.SetOutputTolerance(0.0001f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaNHWCExecutionProvider, kTensorrtExecutionProvider}); } diff --git a/onnxruntime/test/providers/cpu/object_detection/roialign_test.cc b/onnxruntime/test/providers/cpu/object_detection/roialign_test.cc index 0bff46edccc12..58a616717316e 100644 --- a/onnxruntime/test/providers/cpu/object_detection/roialign_test.cc +++ b/onnxruntime/test/providers/cpu/object_detection/roialign_test.cc @@ -464,6 +464,7 @@ static void BasicTest() { 0.3661f, 0.2349f, }); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -690,6 +691,7 @@ TEST(RoiAlignTest, MaxModePositive) { });*/ test.Run(); } + TEST(RoiAlignTest, AvgModeNegativeInvalidMode) { // TODO: Unskip when fixed #41968513 if (DefaultDmlExecutionProvider().get() != nullptr) { diff --git a/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc b/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc index 7e81fc80ddf85..e73a1b492cc05 100644 --- a/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc +++ b/onnxruntime/test/providers/cpu/rnn/deep_cpu_lstm_op_test.cc @@ -143,6 +143,8 @@ static void RunLstmTest(const std::vector& X_data, test.AddOptionalOutputEdge(); } + test.SetOutputTolerance(0.0001f); + // TensorRT failed on LSTM tests test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } diff --git a/onnxruntime/test/providers/cpu/tensor/affine_grid_test.cc b/onnxruntime/test/providers/cpu/tensor/affine_grid_test.cc index e37e784f28930..1ffe6c73d4fa4 100644 --- a/onnxruntime/test/providers/cpu/tensor/affine_grid_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/affine_grid_test.cc @@ -13,6 +13,7 @@ TEST(AffineGridTest, 2d) { test.AddInput("size", {4}, {1, 1, 2, 3}); test.AddOutput("grid", {1, 2, 3, 2}, {-0.6667f, -0.5000f, 0.0000f, -0.5000f, 0.6667f, -0.5000f, -0.6667f, 0.5000f, 0.0000f, 0.5000f, 0.6667f, 0.5000f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -24,6 +25,7 @@ TEST(AffineGridTest, test_2d_0) { test.AddInput("theta", {1, 2, 3}, {1.477212f, -0.173648f, 0.300000f, 0.173648f, 0.492404f, -0.500000f}); test.AddInput("size", {4}, {1, 1, 3, 2}); test.AddOutput("grid", {1, 3, 2, 2}, {-0.3228f, -0.9151f, 1.1544f, -0.7414f, -0.4386f, -0.5868f, 1.0386f, -0.4132f, -0.5544f, -0.2586f, 0.9228f, -0.0849f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -33,6 +35,7 @@ TEST(AffineGridTest, test_2d_1) { test.AddInput("theta", {2, 2, 3}, {1.477212f, -0.173648f, 0.300000f, 0.173648f, 0.492404f, -0.500000f, 1.477212f, -0.173648f, 0.300000f, 0.173648f, 0.492404f, -0.500000f}); test.AddInput("size", {4}, {2, 10, 2, 3}); test.AddOutput("grid", {2, 2, 3, 2}, {-0.5980f, -0.8620f, 0.3868f, -0.7462f, 1.3716f, -0.6304f, -0.7716f, -0.3696f, 0.2132f, -0.2538f, 1.1980f, -0.1380f, -0.5980f, -0.8620f, 0.3868f, -0.7462f, 1.3716f, -0.6304f, -0.7716f, -0.3696f, 0.2132f, -0.2538f, 1.1980f, -0.1380f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -42,6 +45,7 @@ TEST(AffineGridTest, test_2d_2) { test.AddInput("theta", {1, 2, 3}, {1.500000f, -0.866025f, -0.500000f, 0.866025f, 2.750000f, -0.500000f}); test.AddInput("size", {4}, {1, 1, 3, 2}); test.AddOutput("grid", {1, 3, 2, 2}, {-0.6726f, -2.7663f, 0.8274f, -1.9003f, -1.2500f, -0.9330f, 0.2500f, -0.0670f, -1.8274f, 0.9003f, -0.3274f, 1.7663f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -51,6 +55,7 @@ TEST(AffineGridTest, test_2d_3) { test.AddInput("theta", {2, 2, 3}, {1.500000f, -0.866025f, -0.500000f, 0.866025f, 2.750000f, -0.500000f, 1.500000f, -0.866025f, -0.500000f, 0.866025f, 2.750000f, -0.500000f}); test.AddInput("size", {4}, {2, 10, 2, 3}); test.AddOutput("grid", {2, 2, 3, 2}, {-1.0670f, -2.4524f, -0.0670f, -1.8750f, 0.9330f, -1.2976f, -1.9330f, 0.2976f, -0.9330f, 0.8750f, 0.0670f, 1.4524f, -1.0670f, -2.4524f, -0.0670f, -1.8750f, 0.9330f, -1.2976f, -1.9330f, 0.2976f, -0.9330f, 0.8750f, 0.0670f, 1.4524f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -60,6 +65,7 @@ TEST(AffineGridTest, test_2d_4) { test.AddInput("theta", {1, 2, 3}, {1.477212f, -0.173648f, 0.300000f, 0.173648f, 0.492404f, -0.500000f}); test.AddInput("size", {4}, {1, 1, 3, 2}); test.AddOutput("grid", {1, 3, 2, 2}, {-1.0036f, -1.1661f, 1.9509f, -0.8188f, -1.1772f, -0.6736f, 1.7772f, -0.3264f, -1.3509f, -0.1812f, 1.6036f, 0.1661f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -69,6 +75,7 @@ TEST(AffineGridTest, test_2d_5) { test.AddInput("theta", {2, 2, 3}, {1.477212f, -0.173648f, 0.300000f, 0.173648f, 0.492404f, -0.500000f, 1.477212f, -0.173648f, 0.300000f, 0.173648f, 0.492404f, -0.500000f}); test.AddInput("size", {4}, {2, 10, 2, 3}); test.AddOutput("grid", {2, 2, 3, 2}, {-1.0036f, -1.1661f, 0.4736f, -0.9924f, 1.9509f, -0.8188f, -1.3509f, -0.1812f, 0.1264f, -0.0076f, 1.6036f, 0.1661f, -1.0036f, -1.1661f, 0.4736f, -0.9924f, 1.9509f, -0.8188f, -1.3509f, -0.1812f, 0.1264f, -0.0076f, 1.6036f, 0.1661f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -78,6 +85,7 @@ TEST(AffineGridTest, test_2d_6) { test.AddInput("theta", {1, 2, 3}, {1.500000f, -0.866025f, -0.500000f, 0.866025f, 2.750000f, -0.500000f}); test.AddInput("size", {4}, {1, 1, 3, 2}); test.AddOutput("grid", {1, 3, 2, 2}, {-1.1340f, -4.1160f, 1.8660f, -2.3840f, -2.0000f, -1.3660f, 1.0000f, 0.3660f, -2.8660f, 1.3840f, 0.1340f, 3.1160f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -87,6 +95,7 @@ TEST(AffineGridTest, test_2d_7) { test.AddInput("theta", {2, 2, 3}, {1.500000f, -0.866025f, -0.500000f, 0.866025f, 2.750000f, -0.500000f, 1.500000f, -0.866025f, -0.500000f, 0.866025f, 2.750000f, -0.500000f}); test.AddInput("size", {4}, {2, 10, 2, 3}); test.AddOutput("grid", {2, 2, 3, 2}, {-1.1340f, -4.1160f, 0.3660f, -3.2500f, 1.8660f, -2.3840f, -2.8660f, 1.3840f, -1.3660f, 2.2500f, 0.1340f, 3.1160f, -1.1340f, -4.1160f, 0.3660f, -3.2500f, 1.8660f, -2.3840f, -2.8660f, 1.3840f, -1.3660f, 2.2500f, 0.1340f, 3.1160f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -96,6 +105,7 @@ TEST(AffineGridTest, test_3d_0) { test.AddInput("theta", {1, 3, 4}, {1.409539f, 0.000000f, 0.513030f, 0.300000f, 0.118782f, 1.969615f, -0.326352f, -0.500000f, -0.168412f, 0.086824f, 0.462708f, 1.800000f}); test.AddInput("size", {5}, {1, 1, 3, 2, 2}); test.AddOutput("grid", {1, 3, 2, 2, 3}, {-0.7468f, -1.3266f, 1.5323f, 0.6627f, -1.2078f, 1.3639f, -0.7468f, 0.6430f, 1.6191f, 0.6627f, 0.7618f, 1.4507f, -0.4048f, -1.5442f, 1.8408f, 1.0048f, -1.4254f, 1.6724f, -0.4048f, 0.4254f, 1.9276f, 1.0048f, 0.5442f, 1.7592f, -0.0627f, -1.7618f, 2.1493f, 1.3468f, -1.6430f, 1.9809f, -0.0627f, 0.2078f, 2.2361f, 1.3468f, 0.3266f, 2.0677f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -105,6 +115,7 @@ TEST(AffineGridTest, test_3d_1) { test.AddInput("theta", {2, 3, 4}, {1.409539f, 0.000000f, 0.513030f, 0.300000f, 0.118782f, 1.969615f, -0.326352f, -0.500000f, -0.168412f, 0.086824f, 0.462708f, 1.800000f, 1.409539f, 0.000000f, 0.513030f, 0.300000f, 0.118782f, 1.969615f, -0.326352f, -0.500000f, -0.168412f, 0.086824f, 0.462708f, 1.800000f}); test.AddInput("size", {5}, {2, 10, 2, 2, 3}); test.AddOutput("grid", {2, 2, 2, 3, 3}, {-0.8962f, -1.4008f, 1.6375f, 0.0435f, -1.3216f, 1.5252f, 0.9832f, -1.2424f, 1.4130f, -0.8962f, 0.5688f, 1.7243f, 0.0435f, 0.6480f, 1.6121f, 0.9832f, 0.7272f, 1.4998f, -0.3832f, -1.7272f, 2.1002f, 0.5565f, -1.6480f, 1.9879f, 1.4962f, -1.5688f, 1.8757f, -0.3832f, 0.2424f, 2.1870f, 0.5565f, 0.3216f, 2.0748f, 1.4962f, 0.4008f, 1.9625f, -0.8962f, -1.4008f, 1.6375f, 0.0435f, -1.3216f, 1.5252f, 0.9832f, -1.2424f, 1.4130f, -0.8962f, 0.5688f, 1.7243f, 0.0435f, 0.6480f, 1.6121f, 0.9832f, 0.7272f, 1.4998f, -0.3832f, -1.7272f, 2.1002f, 0.5565f, -1.6480f, 1.9879f, 1.4962f, -1.5688f, 1.8757f, -0.3832f, 0.2424f, 2.1870f, 0.5565f, 0.3216f, 2.0748f, 1.4962f, 0.4008f, 1.9625f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -114,6 +125,7 @@ TEST(AffineGridTest, test_3d_2) { test.AddInput("theta", {1, 3, 4}, {0.259808f, 0.000000f, -0.150000f, -0.500000f, -1.299038f, 1.500000f, -2.250000f, -0.500000f, 1.375000f, 4.763140f, 2.381570f, 0.300000f}); test.AddInput("size", {5}, {1, 1, 3, 2, 2}); test.AddOutput("grid", {1, 3, 2, 2, 3}, {-0.5299f, 0.8995f, -4.3568f, -0.2701f, -0.3995f, -2.9818f, -0.5299f, 2.3995f, 0.4064f, -0.2701f, 1.1005f, 1.7814f, -0.6299f, -0.6005f, -2.7691f, -0.3701f, -1.8995f, -1.3941f, -0.6299f, 0.8995f, 1.9941f, -0.3701f, -0.3995f, 3.3691f, -0.7299f, -2.1005f, -1.1814f, -0.4701f, -3.3995f, 0.1936f, -0.7299f, -0.6005f, 3.5818f, -0.4701f, -1.8995f, 4.9568f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -123,6 +135,7 @@ TEST(AffineGridTest, test_3d_3) { test.AddInput("theta", {2, 3, 4}, {0.259808f, 0.000000f, -0.150000f, -0.500000f, -1.299038f, 1.500000f, -2.250000f, -0.500000f, 1.375000f, 4.763140f, 2.381570f, 0.300000f, 0.259808f, 0.000000f, -0.150000f, -0.500000f, -1.299038f, 1.500000f, -2.250000f, -0.500000f, 1.375000f, 4.763140f, 2.381570f, 0.300000f}); test.AddInput("size", {5}, {2, 10, 2, 2, 3}); test.AddOutput("grid", {2, 2, 2, 3, 3}, {-0.5982f, 0.7410f, -4.1890f, -0.4250f, -0.1250f, -3.2724f, -0.2518f, -0.9910f, -2.3557f, -0.5982f, 2.2410f, 0.5741f, -0.4250f, 1.3750f, 1.4908f, -0.2518f, 0.5090f, 2.4075f, -0.7482f, -1.5090f, -1.8075f, -0.5750f, -2.3750f, -0.8908f, -0.4018f, -3.2410f, 0.0259f, -0.7482f, -0.0090f, 2.9557f, -0.5750f, -0.8750f, 3.8724f, -0.4018f, -1.7410f, 4.7890f, -0.5982f, 0.7410f, -4.1890f, -0.4250f, -0.1250f, -3.2724f, -0.2518f, -0.9910f, -2.3557f, -0.5982f, 2.2410f, 0.5741f, -0.4250f, 1.3750f, 1.4908f, -0.2518f, 0.5090f, 2.4075f, -0.7482f, -1.5090f, -1.8075f, -0.5750f, -2.3750f, -0.8908f, -0.4018f, -3.2410f, 0.0259f, -0.7482f, -0.0090f, 2.9557f, -0.5750f, -0.8750f, 3.8724f, -0.4018f, -1.7410f, 4.7890f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -132,6 +145,7 @@ TEST(AffineGridTest, test_3d_4) { test.AddInput("theta", {1, 3, 4}, {1.409539f, 0.000000f, 0.513030f, 0.300000f, 0.118782f, 1.969615f, -0.326352f, -0.500000f, -0.168412f, 0.086824f, 0.462708f, 1.800000f}); test.AddInput("size", {5}, {1, 1, 3, 2, 2}); test.AddOutput("grid", {1, 3, 2, 2, 3}, {-1.6226f, -2.2620f, 1.4189f, 1.1965f, -2.0245f, 1.0821f, -1.6226f, 1.6772f, 1.5925f, 1.1965f, 1.9147f, 1.2557f, -1.1095f, -2.5884f, 1.8816f, 1.7095f, -2.3508f, 1.5448f, -1.1095f, 1.3508f, 2.0552f, 1.7095f, 1.5884f, 1.7184f, -0.5965f, -2.9147f, 2.3443f, 2.2226f, -2.6772f, 2.0075f, -0.5965f, 1.0245f, 2.5179f, 2.2226f, 1.2620f, 2.1811f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -141,6 +155,7 @@ TEST(AffineGridTest, test_3d_5) { test.AddInput("theta", {2, 3, 4}, {1.409539f, 0.000000f, 0.513030f, 0.300000f, 0.118782f, 1.969615f, -0.326352f, -0.500000f, -0.168412f, 0.086824f, 0.462708f, 1.800000f, 1.409539f, 0.000000f, 0.513030f, 0.300000f, 0.118782f, 1.969615f, -0.326352f, -0.500000f, -0.168412f, 0.086824f, 0.462708f, 1.800000f}); test.AddInput("size", {5}, {2, 10, 2, 2, 3}); test.AddOutput("grid", {2, 2, 2, 3, 3}, {-1.6226f, -2.2620f, 1.4189f, -0.2130f, -2.1433f, 1.2505f, 1.1965f, -2.0245f, 1.0821f, -1.6226f, 1.6772f, 1.5925f, -0.2130f, 1.7960f, 1.4241f, 1.1965f, 1.9147f, 1.2557f, -0.5965f, -2.9147f, 2.3443f, 0.8130f, -2.7960f, 2.1759f, 2.2226f, -2.6772f, 2.0075f, -0.5965f, 1.0245f, 2.5179f, 0.8130f, 1.1433f, 2.3495f, 2.2226f, 1.2620f, 2.1811f, -1.6226f, -2.2620f, 1.4189f, -0.2130f, -2.1433f, 1.2505f, 1.1965f, -2.0245f, 1.0821f, -1.6226f, 1.6772f, 1.5925f, -0.2130f, 1.7960f, 1.4241f, 1.1965f, 1.9147f, 1.2557f, -0.5965f, -2.9147f, 2.3443f, 0.8130f, -2.7960f, 2.1759f, 2.2226f, -2.6772f, 2.0075f, -0.5965f, 1.0245f, 2.5179f, 0.8130f, 1.1433f, 2.3495f, 2.2226f, 1.2620f, 2.1811f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -150,6 +165,7 @@ TEST(AffineGridTest, test_3d_6) { test.AddInput("theta", {1, 3, 4}, {0.259808f, 0.000000f, -0.150000f, -0.500000f, -1.299038f, 1.500000f, -2.250000f, -0.500000f, 1.375000f, 4.763140f, 2.381570f, 0.300000f}); test.AddInput("size", {5}, {1, 1, 3, 2, 2}); test.AddOutput("grid", {1, 3, 2, 2, 3}, {-0.6098f, 1.5490f, -8.2197f, -0.0902f, -1.0490f, -5.4697f, -0.6098f, 4.5490f, 1.3066f, -0.0902f, 1.9510f, 4.0566f, -0.7598f, -0.7010f, -5.8381f, -0.2402f, -3.2990f, -3.0881f, -0.7598f, 2.2990f, 3.6881f, -0.2402f, -0.2990f, 6.4381f, -0.9098f, -2.9510f, -3.4566f, -0.3902f, -5.5490f, -0.7066f, -0.9098f, 0.0490f, 6.0697f, -0.3902f, -2.5490f, 8.8197f}); + test.SetOutputTolerance(0.0001f); test.Run(); } @@ -159,6 +175,7 @@ TEST(AffineGridTest, test_3d_7) { test.AddInput("theta", {2, 3, 4}, {0.259808f, 0.000000f, -0.150000f, -0.500000f, -1.299038f, 1.500000f, -2.250000f, -0.500000f, 1.375000f, 4.763140f, 2.381570f, 0.300000f, 0.259808f, 0.000000f, -0.150000f, -0.500000f, -1.299038f, 1.500000f, -2.250000f, -0.500000f, 1.375000f, 4.763140f, 2.381570f, 0.300000f}); test.AddInput("size", {5}, {2, 10, 2, 2, 3}); test.AddOutput("grid", {2, 2, 2, 3, 3}, {-0.6098f, 1.5490f, -8.2197f, -0.3500f, 0.2500f, -6.8447f, -0.0902f, -1.0490f, -5.4697f, -0.6098f, 4.5490f, 1.3066f, -0.3500f, 3.2500f, 2.6816f, -0.0902f, 1.9510f, 4.0566f, -0.9098f, -2.9510f, -3.4566f, -0.6500f, -4.2500f, -2.0816f, -0.3902f, -5.5490f, -0.7066f, -0.9098f, 0.0490f, 6.0697f, -0.6500f, -1.2500f, 7.4447f, -0.3902f, -2.5490f, 8.8197f, -0.6098f, 1.5490f, -8.2197f, -0.3500f, 0.2500f, -6.8447f, -0.0902f, -1.0490f, -5.4697f, -0.6098f, 4.5490f, 1.3066f, -0.3500f, 3.2500f, 2.6816f, -0.0902f, 1.9510f, 4.0566f, -0.9098f, -2.9510f, -3.4566f, -0.6500f, -4.2500f, -2.0816f, -0.3902f, -5.5490f, -0.7066f, -0.9098f, 0.0490f, 6.0697f, -0.6500f, -1.2500f, 7.4447f, -0.3902f, -2.5490f, 8.8197f}); + test.SetOutputTolerance(0.0001f); test.Run(); } } // namespace test diff --git a/onnxruntime/test/providers/cpu/tensor/mean_variance_normalization_test.cc b/onnxruntime/test/providers/cpu/tensor/mean_variance_normalization_test.cc index b6720ae2a9a7d..8dcb15cbc6926 100644 --- a/onnxruntime/test/providers/cpu/tensor/mean_variance_normalization_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/mean_variance_normalization_test.cc @@ -5,6 +5,7 @@ #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" +#include "test/util/include/default_providers.h" namespace onnxruntime::test { @@ -155,6 +156,10 @@ TEST(MeanVarianceNormalizationTest, AxesSubsets5D) { test.AddInput("input", shape, X.data(), X.size()); test.AddOutput("output", shape, Y.data(), Y.size()); + if (DefaultDmlExecutionProvider().get() != nullptr) { + test.SetOutputTolerance(0.001f); + } + test.Run(); }; diff --git a/orttraining/orttraining/test/gradient/optimizer_ops_test.cc b/orttraining/orttraining/test/gradient/optimizer_ops_test.cc index bfb59f1525e47..18c1364f5d1f6 100644 --- a/orttraining/orttraining/test/gradient/optimizer_ops_test.cc +++ b/orttraining/orttraining/test/gradient/optimizer_ops_test.cc @@ -144,6 +144,8 @@ TEST(OptimizerTest, AdamBiasCorrection) { test.AddOutput("Moment_2_Out", {3}, {1.7400e-04f, 8.9966e-04f, 1.5102e-03f}); test.AddOutput("W_Out", {3}, {-1.4634f, -0.6416f, -1.2121f}); + test.SetOutputTolerance(0.0001f); + test.AddAttribute("do_bias_correction", static_cast(1)); test.AddAttribute("weight_decay_mode", static_cast(0)); @@ -167,6 +169,8 @@ TEST(OptimizerTest, AdamWeightDecayMode0NoBiasCorrection) { test.AddOutput("W_Out", {3}, {-3.6210f, -2.8075f, -3.3723f}); test.AddOutput("G_Out", {3}, {-3.1576f, -3.1658f, -3.1601f}); + test.SetOutputTolerance(0.0001f); + test.AddAttribute("do_bias_correction", static_cast(0)); test.AddAttribute("lambda", 0.01f); test.AddAttribute("weight_decay_mode", static_cast(0)); @@ -191,6 +195,8 @@ TEST(OptimizerTest, AdamWeightDecayMode0WithBiasCorrection) { test.AddOutput("W_Out", {3}, {-1.4587f, -0.6452f, -1.2099f}); test.AddOutput("G_Out", {3}, {-0.9954f, -1.0036f, -0.9979f}); + test.SetOutputTolerance(0.0001f); + test.AddAttribute("do_bias_correction", static_cast(1)); test.AddAttribute("lambda", 0.01f); test.AddAttribute("weight_decay_mode", static_cast(0)); @@ -214,6 +220,8 @@ TEST(OptimizerTest, AdamWeightDecayMode1NoBiasCorrection) { test.AddOutput("Moment_2_Out", {3}, {1.7400e-04f, 8.9966e-04f, 1.5102e-03f}); test.AddOutput("W_Out", {3}, {-3.5894f, -2.7758f, -3.3406f}); + test.SetOutputTolerance(0.0001f); + test.AddAttribute("do_bias_correction", static_cast(0)); test.AddAttribute("lambda", 0.01f); test.AddAttribute("weight_decay_mode", static_cast(1)); @@ -237,6 +245,8 @@ TEST(OptimizerTest, AdamWeightDecayMode1WithBiasCorrection) { test.AddOutput("Moment_2_Out", {3}, {1.7400e-04f, 8.9966e-04f, 1.5102e-03f}); test.AddOutput("W_Out", {3}, {-1.4488f, -0.6352f, -1.1999f}); + test.SetOutputTolerance(0.0001f); + test.AddAttribute("do_bias_correction", static_cast(1)); test.AddAttribute("lambda", 0.01f); test.AddAttribute("weight_decay_mode", static_cast(1)); @@ -368,6 +378,11 @@ TEST(OptimizerTest, AdamOptimizerMixPrecision_FP16Weight_ClipNorm_Test) { test.AddOptionalOutputEdge(); test.AddOutput("FP16_W_Out", {3}, w_new_half); + test.SetOutputAbsErr("Moment_1_Out", 0.005f); + test.SetOutputAbsErr("Moment_2_Out", 0.005f); + test.SetOutputAbsErr("W_Out", 0.001f); + test.SetOutputAbsErr("FP16_W_Out", 0.005f); + test.AddAttribute("do_bias_correction", static_cast(0)); test.AddAttribute("weight_decay_mode", static_cast(0)); test.AddAttribute("max_norm_clip", 0.001f); @@ -617,6 +632,8 @@ void run_lamb_test_with_baseline( test.AddOptionalOutputEdge(); } + test.SetOutputTolerance(0.005f); + test.Run(); } @@ -737,6 +754,8 @@ void run_multi_tensor_lamb_test_with_baseline( test.AddAttribute("ratio_min", ratio_min); test.AddAttribute("ratio_max", ratio_max); + test.SetOutputTolerance(0.005f); + test.Run(); } diff --git a/orttraining/orttraining/test/training_ops/cpu/nn/batchnorm_internal_test.cc b/orttraining/orttraining/test/training_ops/cpu/nn/batchnorm_internal_test.cc index e9795a24681cb..e89883bfd4d94 100644 --- a/orttraining/orttraining/test/training_ops/cpu/nn/batchnorm_internal_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/nn/batchnorm_internal_test.cc @@ -37,6 +37,8 @@ TEST(BatchNormInternalTest, ForwardTrainingTest) { test.AddOutput("saved_mean", channel_dims, {-0.306f, 0.114562f}); test.AddOutput("saved_inv_std", channel_dims, {1.2288f, 0.861317f}); + test.SetOutputTolerance(0.0001f); + std::vector> execution_providers; execution_providers.emplace_back(DefaultCpuExecutionProvider()); diff --git a/orttraining/orttraining/test/training_ops/cuda/batch_norm_internal_test.cc b/orttraining/orttraining/test/training_ops/cuda/batch_norm_internal_test.cc index 6335a666e0381..d842d4f1ea736 100644 --- a/orttraining/orttraining/test/training_ops/cuda/batch_norm_internal_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/batch_norm_internal_test.cc @@ -68,6 +68,7 @@ static void TestBatchNormInternal(bool test_double = false, bool T_is_half = fal test.AddOutput("running_var", channel_dims, running_var_double); test.AddOutput("saved_mean", channel_dims, saved_mean_double); test.AddOutput("saved_inv_std", channel_dims, saved_inv_std_double); + test.SetOutputTolerance(0.0001f); } else { if (T_is_half) { std::vector X_half(X.size());