From d31ac746ffe219d5f3612aec9268f0d2dd146359 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Tue, 11 Jul 2023 20:13:32 +0200 Subject: [PATCH 1/5] Allow gelu approximations. --- coremltools/converters/mil/frontend/torch/ops.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/coremltools/converters/mil/frontend/torch/ops.py b/coremltools/converters/mil/frontend/torch/ops.py index 89cfb7ace..9bdd965be 100644 --- a/coremltools/converters/mil/frontend/torch/ops.py +++ b/coremltools/converters/mil/frontend/torch/ops.py @@ -3898,8 +3898,15 @@ def gelu(context, node): assert len(inputs) in (1, 2) if len(inputs) == 2: approximate = inputs[1].val - assert approximate == 'none' - res = mb.gelu(x=inputs[0], name=node.name) + if approximate == "tanh": + approximate = "TANH_APPROXIMATION" + elif approximate == "sigmoid": + approximate = "SIGMOID_APPROXIMATION" + elif approximate == "none": + approximate = "EXACT" + else: + approximate = None + res = mb.gelu(x=inputs[0], mode=approximate, name=node.name) context.add(res) From 3741477c12862d34198b3d37ab114d0ca06cb90a Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Wed, 12 Jul 2023 15:16:46 +0200 Subject: [PATCH 2/5] Add test for `tanh` gelu approximation. --- .../converters/mil/frontend/torch/test/test_torch_ops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py b/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py index 07eb00ac7..5983ea0ee 100644 --- a/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py +++ b/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py @@ -4982,11 +4982,11 @@ def test_hardswish(self, compute_unit, backend, shape, minimum_deployment_target ) @pytest.mark.parametrize( - "compute_unit, backend, shape", - itertools.product(compute_units, backends, COMMON_SHAPES_ALL), + "compute_unit, backend, shape, approximate", + itertools.product(compute_units, backends, COMMON_SHAPES_ALL, ["none", "tanh"]), ) - def test_gelu(self, compute_unit, backend, shape): - model = nn.GELU().eval() + def test_gelu(self, compute_unit, backend, shape, approximate): + model = nn.GELU(approximate=approximate).eval() self.run_compare_torch(shape, model, backend=backend, compute_unit=compute_unit) @pytest.mark.parametrize( From dd9a028f6ed463faf734e0d0f0994234fbf938f9 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Wed, 12 Jul 2023 15:17:14 +0200 Subject: [PATCH 3/5] Remove `sigmoid` GELU in frontend (not yet supported in PyTorch) --- coremltools/converters/mil/frontend/torch/ops.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/coremltools/converters/mil/frontend/torch/ops.py b/coremltools/converters/mil/frontend/torch/ops.py index 9bdd965be..0611eaffa 100644 --- a/coremltools/converters/mil/frontend/torch/ops.py +++ b/coremltools/converters/mil/frontend/torch/ops.py @@ -3900,8 +3900,6 @@ def gelu(context, node): approximate = inputs[1].val if approximate == "tanh": approximate = "TANH_APPROXIMATION" - elif approximate == "sigmoid": - approximate = "SIGMOID_APPROXIMATION" elif approximate == "none": approximate = "EXACT" else: From 044a0961bac5e4cc62f4ac00760b3c4f7987333e Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Wed, 12 Jul 2023 20:53:43 +0200 Subject: [PATCH 4/5] Assert gelu approximation is supported. --- coremltools/converters/mil/frontend/torch/ops.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/coremltools/converters/mil/frontend/torch/ops.py b/coremltools/converters/mil/frontend/torch/ops.py index 0611eaffa..e319c315a 100644 --- a/coremltools/converters/mil/frontend/torch/ops.py +++ b/coremltools/converters/mil/frontend/torch/ops.py @@ -3896,15 +3896,14 @@ def hardsigmoid(context, node): def gelu(context, node): inputs = _get_inputs(context, node) assert len(inputs) in (1, 2) + mode = None if len(inputs) == 2: approximate = inputs[1].val if approximate == "tanh": - approximate = "TANH_APPROXIMATION" - elif approximate == "none": - approximate = "EXACT" - else: - approximate = None - res = mb.gelu(x=inputs[0], mode=approximate, name=node.name) + mode = "TANH_APPROXIMATION" + else: + assert approximate == "none" + res = mb.gelu(x=inputs[0], mode=mode, name=node.name) context.add(res) From f13ffec0bfa42c1eb5abaa6885199917bc8f86cf Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Wed, 12 Jul 2023 20:55:16 +0200 Subject: [PATCH 5/5] gelu tests cover the default initialization. --- .../converters/mil/frontend/torch/test/test_torch_ops.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py b/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py index 5983ea0ee..02a8eaf77 100644 --- a/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py +++ b/coremltools/converters/mil/frontend/torch/test/test_torch_ops.py @@ -4983,10 +4983,11 @@ def test_hardswish(self, compute_unit, backend, shape, minimum_deployment_target @pytest.mark.parametrize( "compute_unit, backend, shape, approximate", - itertools.product(compute_units, backends, COMMON_SHAPES_ALL, ["none", "tanh"]), + itertools.product(compute_units, backends, COMMON_SHAPES_ALL, ["none", "tanh", None]), ) def test_gelu(self, compute_unit, backend, shape, approximate): - model = nn.GELU(approximate=approximate).eval() + model = nn.GELU() if approximate is None else nn.GELU(approximate=approximate) + model = model.eval() self.run_compare_torch(shape, model, backend=backend, compute_unit=compute_unit) @pytest.mark.parametrize(