From 22307a1427e897bac844f9a9c8dd554c0fbf1665 Mon Sep 17 00:00:00 2001 From: Sean Silva Date: Fri, 18 Nov 2022 12:56:04 +0000 Subject: [PATCH] Clean up some parts of the test suite The purpose of the test suite is to accelerate the development of the compiler. However, we had various tests there that were not expected to work, had no in-progress work being tested by the test, and nobody was actively working on them. Having such tests in our test suite just adds clutter and slows down development on the compiler. --- e2e_testing/xfail_sets.py | 11 +- .../test_suite/__init__.py | 10 -- .../test_suite/backprop.py | 69 ------------ python/torch_mlir_e2e_test/test_suite/conv.py | 103 ------------------ .../torch_mlir_e2e_test/test_suite/pooling.py | 47 -------- .../test_suite/table_batch_embedding.py | 60 ---------- 6 files changed, 4 insertions(+), 296 deletions(-) delete mode 100644 python/torch_mlir_e2e_test/test_suite/table_batch_embedding.py diff --git a/e2e_testing/xfail_sets.py b/e2e_testing/xfail_sets.py index cf79a4298f47..5ab4ab96526d 100644 --- a/e2e_testing/xfail_sets.py +++ b/e2e_testing/xfail_sets.py @@ -14,10 +14,10 @@ REFBACKEND_XFAIL_SET = COMMON_TORCH_MLIR_LOWERING_XFAILS -EAGER_MODE_XFAIL_SET = { - # RefBackend fails - "TableBatchEmbeddingModule_basic", - "QuantizedMLP_basic", +EAGER_MODE_XFAIL_SET = COMMON_TORCH_MLIR_LOWERING_XFAILS | { + # RefBackend fails for some reason. + # These tests pass in the regular RefBackend flow, so it's unclear + # why they fail here. "Matmul_vecmat", "BatchMlpLayerModule_basic", "UpSampleNearest2dDynamicFactor_basic", @@ -619,7 +619,6 @@ "StdUnbiasedModule_basic", "SubFloatModule_basic", "SubIntModule_basic", - "TableBatchEmbeddingModule_basic", "TensorsConcatNegativeDimModule_basic", "TensorToBoolZeroRank_basic", "TensorToBool_basic", @@ -645,10 +644,8 @@ "Fill_TensorFloat32WithInt64_basic", "UpSampleNearest2dBackwardVec_basic", "UpSampleNearest2dBackwardOutputSizeNone_basic", - "ConvolutionBackwardModule1D_basic", "ConvolutionBackwardModule2D_basic", "ConvolutionBackwardModule2DPadded_basic", - "ConvolutionBackwardModule3D_basic", "VarMeanCorrectionModule_basic", "VarMeanCorrectionNoneModule_basic" } diff --git a/python/torch_mlir_e2e_test/test_suite/__init__.py b/python/torch_mlir_e2e_test/test_suite/__init__.py index d35151ccd53b..c107ceb37561 100644 --- a/python/torch_mlir_e2e_test/test_suite/__init__.py +++ b/python/torch_mlir_e2e_test/test_suite/__init__.py @@ -8,15 +8,6 @@ # to the backend contract. COMMON_TORCH_MLIR_LOWERING_XFAILS = { "QuantizedMLP_basic", - "TableBatchEmbeddingModule_basic", - "Convolution3DModule_basic", - "Convolution1DModule_basic", - "Conv_Transpose3dModule_basic", - "Conv_Transpose1dModule_basic", - "MaxPool2dWith3dInputModule_basic", - "MaxPool2dWithIndicesWith3dInputModule_basic", - "ConvolutionBackwardModule1D_basic", - "ConvolutionBackwardModule3D_basic", } def register_all_tests(): @@ -47,7 +38,6 @@ def register_all_tests(): from . import constant_alloc from . import threshold from . import histogram_binning_calibration - from . import table_batch_embedding from . import rng from . import cast from . import index_put diff --git a/python/torch_mlir_e2e_test/test_suite/backprop.py b/python/torch_mlir_e2e_test/test_suite/backprop.py index 01e4aef9ead1..354881c2feb4 100644 --- a/python/torch_mlir_e2e_test/test_suite/backprop.py +++ b/python/torch_mlir_e2e_test/test_suite/backprop.py @@ -58,41 +58,6 @@ def TanhBackward_basic(module, tu: TestUtils): # ============================================================================== - -class ConvolutionBackwardModule1D(torch.nn.Module): - - def __init__(self): - super().__init__() - - @export - @annotate_args([ - None, - ([-1, -1, -1], torch.float32, True), - ([-1, -1, -1], torch.float32, True), - ([-1, -1, -1], torch.float32, True), - ]) - def forward(self, grad_out, input_vec, weight): - return torch.ops.aten.convolution_backward( - grad_out, - input_vec, - weight, - bias_sizes=None, - stride=[1], - padding=[0], - dilation=[1], - transposed=False, - output_padding=[0], - groups=1, - output_mask=[True, True, True]) - - -@register_test_case(module_factory=lambda: ConvolutionBackwardModule1D()) -def ConvolutionBackwardModule1D_basic(module, tu: TestUtils): - with torch.backends.mkldnn.flags(enabled=False): - module.forward(tu.rand(3, 3, 3), tu.rand(3, 3, 3), - tu.rand(3, 3, 1)) - - class ConvolutionBackwardModule2D(torch.nn.Module): def __init__(self): @@ -161,40 +126,6 @@ def ConvolutionBackwardModule2DPadded_basic(module, tu: TestUtils): tu.rand(2, 2, 3, 3)) -class ConvolutionBackwardModule3D(torch.nn.Module): - - def __init__(self): - super().__init__() - - @export - @annotate_args([ - None, - ([-1, -1, -1, -1, -1], torch.float32, True), - ([-1, -1, -1, -1, -1], torch.float32, True), - ([-1, -1, -1, -1, -1], torch.float32, True), - ]) - def forward(self, grad_out, input_vec, weight): - return torch.ops.aten.convolution_backward( - grad_out, - input_vec, - weight, - bias_sizes=None, - stride=[1, 1, 1], - padding=[0], - dilation=[1, 1, 1], - transposed=False, - output_padding=[0], - groups=1, - output_mask=[True, True, True]) - - -@register_test_case(module_factory=lambda: ConvolutionBackwardModule3D()) -def ConvolutionBackwardModule3D_basic(module, tu: TestUtils): - with torch.backends.mkldnn.flags(enabled=False): - module.forward(tu.rand(3, 3, 3, 3, 3), tu.rand(3, 3, 3, 3, 3), - tu.rand(3, 3, 1, 1, 1)) - - # ============================================================================== diff --git a/python/torch_mlir_e2e_test/test_suite/conv.py b/python/torch_mlir_e2e_test/test_suite/conv.py index c1c122f49671..a2236f3ef7bc 100644 --- a/python/torch_mlir_e2e_test/test_suite/conv.py +++ b/python/torch_mlir_e2e_test/test_suite/conv.py @@ -142,32 +142,6 @@ def Conv2dWithPaddingDilationStrideStaticModule_basic(module, tu: TestUtils): # ============================================================================== -class Convolution1DModule(torch.nn.Module): - def __init__(self): - super().__init__() - - @export - @annotate_args([ - None, - ([-1, -1, -1], torch.float32, True), - ([-1, -1, -1], torch.float32, True), - ]) - def forward(self, inputVec, weight): - return torch.ops.aten.convolution(inputVec, - weight, - bias=None, - stride=[1], - padding=[0], - dilation=[1], - transposed=False, - output_padding=[0], - groups=1) - - -@register_test_case(module_factory=lambda: Convolution1DModule()) -def Convolution1DModule_basic(module, tu: TestUtils): - module.forward(torch.randn(3, 3, 10), torch.randn(3, 3, 2)) - class Convolution2DModule(torch.nn.Module): def __init__(self): super().__init__() @@ -193,31 +167,6 @@ def forward(self, inputVec, weight): def Convolution2DModule_basic(module, tu: TestUtils): module.forward(torch.randn(3, 3, 10, 10), torch.randn(3, 3, 2, 2)) -class Convolution3DModule(torch.nn.Module): - def __init__(self): - super().__init__() - - @export - @annotate_args([ - None, - ([-1, -1, -1, -1, -1], torch.float32, True), - ([-1, -1, -1, -1, -1], torch.float32, True), - ]) - def forward(self, inputVec, weight): - return torch.ops.aten.convolution(inputVec, - weight, - bias=None, - stride=[1, 1, 1], - padding=[0, 0, 0], - dilation=[1, 1, 1], - transposed=False, - output_padding=[0, 0, 0], - groups=1) - - -@register_test_case(module_factory=lambda: Convolution3DModule()) -def Convolution3DModule_basic(module, tu: TestUtils): - module.forward(torch.randn(3, 3, 10, 10, 10), torch.randn(3, 3, 2, 2, 2)) class Convolution2DStaticModule(torch.nn.Module): def __init__(self): @@ -635,33 +584,6 @@ def ConvolutionModule2DTransposeStridedStatic_basic(module, tu: TestUtils): module.forward(torch.randn(5, 2, 5, 6), torch.randn(2, 5, 2, 2)) -class Conv_Transpose1dModule(torch.nn.Module): - - def __init__(self): - super().__init__() - - @export - @annotate_args([ - None, - ([-1, -1, -1], torch.float32, True), - ([-1, -1, -1], torch.float32, True), - ]) - def forward(self, inputVec, weight): - return torch.ops.aten.conv_transpose1d(inputVec, - weight, - bias=None, - stride=[2], - padding=[1], - dilation=[1], - output_padding=[0], - groups=1) - - -@register_test_case(module_factory=lambda: Conv_Transpose1dModule()) -def Conv_Transpose1dModule_basic(module, tu: TestUtils): - module.forward(torch.randn(5, 2, 5), torch.randn(2, 5, 2)) - - class Conv_Transpose2dModule(torch.nn.Module): def __init__(self): @@ -688,31 +610,6 @@ def forward(self, inputVec, weight): def Conv_Transpose2dModule_basic(module, tu: TestUtils): module.forward(torch.randn(5, 2, 5, 6), torch.randn(2, 5, 2, 2)) -class Conv_Transpose3dModule(torch.nn.Module): - - def __init__(self): - super().__init__() - - @export - @annotate_args([ - None, - ([-1, -1, -1, -1, -1], torch.float32, True), - ([-1, -1, -1, -1, -1], torch.float32, True), - ]) - def forward(self, inputVec, weight): - return torch.ops.aten.conv_transpose3d(inputVec, - weight, - bias=None, - stride=[2, 2, 2], - padding=[1, 1, 1], - dilation=[1, 1, 1], - output_padding=[0, 0, 0], - groups=1) - - -@register_test_case(module_factory=lambda: Conv_Transpose3dModule()) -def Conv_Transpose3dModule_basic(module, tu: TestUtils): - module.forward(torch.randn(5, 2, 5, 6, 4), torch.randn(2, 5, 2, 2, 2)) class UpSampleNearest2d(torch.nn.Module): diff --git a/python/torch_mlir_e2e_test/test_suite/pooling.py b/python/torch_mlir_e2e_test/test_suite/pooling.py index 6c416f388611..bce8850eafb3 100644 --- a/python/torch_mlir_e2e_test/test_suite/pooling.py +++ b/python/torch_mlir_e2e_test/test_suite/pooling.py @@ -171,29 +171,6 @@ def MaxPool2dCeilModeTrueModule_basic(module, tu: TestUtils): module.forward(tu.rand(1, 1, 20, 20, low=0.5, high=1.0)) -class MaxPool2dWith3dInputModule(torch.nn.Module): - - def __init__(self): - super().__init__() - self.mp2d = torch.nn.MaxPool2d(kernel_size=[6, 8], - stride=[2, 2], - padding=[3, 4], - dilation=2) - - @export - @annotate_args([ - None, - ([-1, -1, -1], torch.float32, True), - ]) - def forward(self, x): - return self.mp2d(x) - - -@register_test_case(module_factory=lambda: MaxPool2dWith3dInputModule()) -def MaxPool2dWith3dInputModule_basic(module, tu: TestUtils): - module.forward(tu.rand(1, 20, 20, low=-1)) - - # ============================================================================== @@ -435,30 +412,6 @@ def MaxPool2dWithIndicesCeilModeTrueModule_basic(module, tu: TestUtils): module.forward(tu.rand(1, 1, 8, 8, low=0.5, high=1.0)) -class MaxPool2dWithIndicesWith3dInputModule(torch.nn.Module): - - def __init__(self): - super().__init__() - - @export - @annotate_args([ - None, - ([-1, -1, -1], torch.float32, True), - ]) - def forward(self, x): - return torch.ops.aten.max_pool2d_with_indices(x, - kernel_size=[2, 2], - stride=[1, 1], - padding=[0, 0], - dilation=[1, 1]) - - -@register_test_case( - module_factory=lambda: MaxPool2dWithIndicesWith3dInputModule()) -def MaxPool2dWithIndicesWith3dInputModule_basic(module, tu: TestUtils): - module.forward(tu.rand(1, 8, 8, low=0.5, high=1.0)) - - # ============================================================================== diff --git a/python/torch_mlir_e2e_test/test_suite/table_batch_embedding.py b/python/torch_mlir_e2e_test/test_suite/table_batch_embedding.py deleted file mode 100644 index 1ed41ffc165a..000000000000 --- a/python/torch_mlir_e2e_test/test_suite/table_batch_embedding.py +++ /dev/null @@ -1,60 +0,0 @@ -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https://llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# Also available under a BSD-style license. See LICENSE. - -import torch - -from torch_mlir_e2e_test.framework import TestUtils -from torch_mlir_e2e_test.registry import register_test_case -from torch_mlir_e2e_test.annotations import annotate_args, export - -# ============================================================================== - -# Reference: https://github.com/pytorch/FBGEMM/blob/main/fbgemm_gpu/bench/split_table_batched_embeddings_benchmark.py#L270 - -# Global parameters. -NUM_TABLES = 2 -NUM_EMBEDDINGS = 10 -EMBEDDING_DIM = 4 -BATCH_SIZE = 4 -BAG_SIZE = 3 - - -class TableBatchEmbeddingModule(torch.nn.Module): - def __init__(self): - super(TableBatchEmbeddingModule, self).__init__() - self.num_tables = NUM_TABLES - self.num_embeddings = NUM_EMBEDDINGS - self.embedding_dim = EMBEDDING_DIM - self.batch_size = BATCH_SIZE - self.bag_size = BAG_SIZE - # Currently, pooling_mode is fixed to 'sum'. - self.nn_embedding_list = torch.nn.ModuleList([ - torch.nn.EmbeddingBag( - self.num_embeddings, self.embedding_dim, mode="sum", sparse=False) - for i in range(self.num_tables) - ]) - - @export - @annotate_args([ - None, - ([-1], torch.int64, True), - ([-1], torch.int64, True), - ]) - def forward(self, indices, offsets): - indices_list = indices.view(self.num_tables, self.batch_size, self.bag_size) - final_output = torch.tensor([]) - for i, nn_embedding in enumerate(self.nn_embedding_list): - indices = indices_list[i].view(-1) - output = nn_embedding(indices, offsets).view(self.batch_size, -1) - final_output = torch.cat((final_output, output), dim=1) - return final_output - - -@register_test_case(module_factory=lambda: TableBatchEmbeddingModule()) -def TableBatchEmbeddingModule_basic(module, tu: TestUtils): - indices = tu.randint(NUM_TABLES * BATCH_SIZE * BAG_SIZE, high=NUM_EMBEDDINGS) - offsets = torch.cumsum( - torch.tensor([0] + [BAG_SIZE for _ in range(BATCH_SIZE - 1)], dtype=torch.int64), 0) - module.forward(indices, offsets)