Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

【Hackathon 5th No.95】add paddle unique op #20077

Merged
merged 7 commits into from
Dec 15, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions src/frontends/paddle/src/op/unique.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
// Copyright (C) 2018-2023 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "default_opset.hpp"
#include "openvino/frontend/paddle/node_context.hpp"
#include "openvino/opsets/opset10.hpp"

namespace ov {
namespace frontend {
namespace paddle {
namespace op {
NamedOutputs unique(const NodeContext& node) {
auto x = node.get_input("X");

std::vector<Output<Node>> outputs;
NamedOutputs named_outputs;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As far as I know, the NamedOutputs is a std::map.
In this case, you'd better use constructor with initailization list insteading of inserting k-v one by one, I think.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done


auto axis = node.get_attribute<std::vector<int32_t>>("axis");

auto dtype_str = node.get_attribute<std::string>("dtype");
auto dtype = dtype_str == "int32" ? element::i32 : element::i64;

if (axis.size() != 0) {
auto axis_node = std::make_shared<default_opset::Constant>(element::i32, Shape{}, axis);
outputs = std::make_shared<ov::opset10::Unique>(x, axis_node, true, dtype, dtype)->outputs();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we use default_opset other than opset10 ?

Copy link
Contributor Author

@AndPuQing AndPuQing Nov 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

To the best of my knowledge, the unique operator is defined in opset10

But the default_opset is 9

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is okay here. I will upgrate the default_opset later.

} else {
outputs = std::make_shared<ov::opset10::Unique>(x, true, dtype, dtype)->outputs();
}

named_outputs["Index"] = {outputs[2]};
named_outputs["Indices"] = {outputs[1]};
named_outputs["Counts"] = {outputs[3]};
Copy link
Contributor

@xczhai xczhai Nov 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In paddle, the count_element_type and index_element_type both be controlled by attr_dtype

named_outputs["Out"] = {outputs[0]};
return named_outputs;
}

} // namespace op
} // namespace paddle
} // namespace frontend
} // namespace ov
2 changes: 2 additions & 0 deletions src/frontends/paddle/src/op_table.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ OP_CONVERTER(top_k_v2);
OP_CONVERTER(transpose2);
OP_CONVERTER(trilinear_interp_v2);
OP_CONVERTER(unsqueeze);
OP_CONVERTER(unique);
OP_CONVERTER(where);
OP_CONVERTER(while_);
OP_CONVERTER(write_to_array);
Expand Down Expand Up @@ -249,6 +250,7 @@ std::map<std::string, CreatorFunction> get_supported_ops() {
{"transpose2", op::transpose2},
{"trilinear_interp_v2", op::trilinear_interp_v2},
{"unsqueeze2", op::unsqueeze},
{"unique", op::unique},
{"where", op::where},
{"while", op::while_},
{"write_to_array", op::write_to_array},
Expand Down
12 changes: 11 additions & 1 deletion src/frontends/paddle/tests/op_fuzzy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@

#include <fstream>

#include "common_test_utils/test_control.hpp"
#include "ngraph/ngraph.hpp"
#include "paddle_utils.hpp"
#include "common_test_utils/test_control.hpp"

using namespace ngraph;
using namespace InferenceEngine;
Expand Down Expand Up @@ -565,6 +565,16 @@ static const std::vector<std::string> models{
std::string("trilinear_upsample_scales2/trilinear_upsample_scales2.pdmodel"),
std::string("trilinear_upsample_true_0/trilinear_upsample_true_0.pdmodel"),
std::string("unsqueeze"),
std::string("unique"),
std::string("unique_ret_index"),
std::string("unique_ret_inverse"),
std::string("unique_ret_counts"),
std::string("unique_ret_index_inverse"),
std::string("unique_ret_index_counts"),
std::string("unique_ret_inverse_counts"),
std::string("unique_ret_index_inverse_counts"),
std::string("unique_ret_index_axis"),
std::string("unique_ret_index_i32"),
std::string("where_1"),
std::string("where_2"),
std::string("where_3"),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

#
# unique paddle model generator
#
import numpy as np
from save_model import saveModel
import paddle
import sys

data_type = "float32"


def unique(
name: str,
x,
reture_index=False,
reture_inverse=False,
reture_counts=False,
dtype="int64",
axes=None,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we leave these parameter's name as origin one, which can be loaded to paddle.unique directly

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Already fixed.

):
paddle.enable_static()

with paddle.static.program_guard(paddle.static.Program(), paddle.static.Program()):
node_x = paddle.static.data(name="x", shape=x.shape, dtype=data_type)

unique_outs = paddle.unique(
node_x,
return_index=reture_index,
return_inverse=reture_inverse,
return_counts=reture_counts,
dtype=dtype,
axis=axes,
)
if reture_counts or reture_inverse or reture_index:
outputs = []
for out in unique_outs:
if out is not None:
if out.dtype == paddle.int64 or out.dtype == paddle.int32:
out = paddle.cast(out, "float32")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can't figure out why cast operation is needed. In my understanding, we should keep the original unique output for verification.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AFAIK, the openvino test compare_results only support fp32 & int32, so for the int64 index we should convert to float32/int32

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@AndPuQing Thanks for your very detailed analysis. Your finding makes sense.
Maybe the test case framework is not perfect enough to call. Don't worry at all. If you think it is not reasonable, just pull PR to fix or optimize it, which is much appreciated. We should NOT skip it using a work-around.
If you have no time for this case, let me know.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, it seems I may not have the time to rectify the test case framework.

outputs.append(out)
else:
outputs = [unique_outs]

cpu = paddle.static.cpu_places(1)
exe = paddle.static.Executor(cpu[0])
# startup program will call initializer to initialize the parameters.
exe.run(paddle.static.default_startup_program())

fetch_vars = [x for x in outputs if x is not None]

outs = exe.run(feed={"x": x}, fetch_list=fetch_vars)

saveModel(
name,
exe,
feedkeys=["x"],
fetchlist=fetch_vars,
inputs=[x],
outputs=outs,
target_dir=sys.argv[1],
)


def main():
data = np.array([2, 3, 3, 1, 5, 3]).astype(data_type)
unique("unique", data)
unique("unique_ret_index", data, reture_index=True)
unique("unique_ret_inverse", data, reture_inverse=True)
unique("unique_ret_counts", data, reture_counts=True)
unique("unique_ret_index_inverse", data, reture_index=True, reture_inverse=True)
unique("unique_ret_index_counts", data, reture_index=True, reture_counts=True)
unique("unique_ret_inverse_counts", data, reture_inverse=True, reture_counts=True)
unique(
"unique_ret_index_inverse_counts",
data,
reture_index=True,
reture_inverse=True,
reture_counts=True,
)

data = np.array([[2, 1, 3], [3, 0, 1], [2, 1, 3]]).astype(data_type)
unique("unique_ret_index_axis", data, reture_index=True, axes=0)
unique("unique_ret_index_i32", data, reture_index=True, dtype="int32")


if __name__ == "__main__":
main()