From 46a7f39330c8eeb8f6917bf2b8102483d60864d1 Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Wed, 9 May 2018 14:54:11 +0800 Subject: [PATCH 1/9] Implemented has_data_op --- paddle/fluid/operators/has_data_op.cc | 77 +++++++++++++++++++ paddle/fluid/operators/has_data_op.h | 44 +++++++++++ .../fluid/tests/unittests/test_has_data_op.py | 26 +++++++ 3 files changed, 147 insertions(+) create mode 100644 paddle/fluid/operators/has_data_op.cc create mode 100644 paddle/fluid/operators/has_data_op.h create mode 100644 python/paddle/fluid/tests/unittests/test_has_data_op.py diff --git a/paddle/fluid/operators/has_data_op.cc b/paddle/fluid/operators/has_data_op.cc new file mode 100644 index 0000000000000..39ff3f50d9acb --- /dev/null +++ b/paddle/fluid/operators/has_data_op.cc @@ -0,0 +1,77 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/fluid/operators/has_data_op.h" +#include +#include "paddle/fluid/framework/op_registry.h" + +#include + +namespace paddle { +namespace operators { + +class HasDataOpMaker : public framework::OpProtoAndCheckerMaker { + public: + HasDataOpMaker(OpProto *proto, OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + // inputs and outputs stored in proto + AddInput("X", "(LoDTensor) the LoDTensor to check"); + AddOutput("Out", "(LoDTensor) the ouput of has_data_op"); + AddComment(R"DOC( +Has Data Operator. + +This operator tests whether the input tensor has data or not. +Out is a boolean scalar. + )DOC"); + } +}; + +class HasDataOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of HasDataOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of HasDataOp should not be null."); + std::cout << "Before set\n"; + ctx->SetOutputDim("Out", {1}); + std::cout << "After set\n"; + ctx->ShareLoD("X", "Out"); + } + + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + framework::OpKernelType kt = framework::OpKernelType( + framework::ToDataType( + ctx.Input("X")->type()), + platform::CPUPlace()); + return kt; + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OPERATOR(has_data, ops::HasDataOp, ops::HasDataOpMaker); +REGISTER_OP_CPU_KERNEL( + has_data, + ops::HasDataOpKernel, + ops::HasDataOpKernel, + ops::HasDataOpKernel, + ops::HasDataOpKernel); diff --git a/paddle/fluid/operators/has_data_op.h b/paddle/fluid/operators/has_data_op.h new file mode 100644 index 0000000000000..d1465dcba402c --- /dev/null +++ b/paddle/fluid/operators/has_data_op.h @@ -0,0 +1,44 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include +#include +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/operators/elementwise_op_function.h" +#include "paddle/fluid/platform/transform.h" + +namespace paddle { +namespace operators { + +template +class HasDataOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* input = context.Input("X"); + auto* output = context.Output("Out"); + size_t mem_size = input->memory_size(); + + auto* output_data = + output->mutable_data(platform::CPUPlace()); + if (mem_size > 0) { + output_data[0] = true; + } else { + output_data[0] = false; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle/fluid/tests/unittests/test_has_data_op.py b/python/paddle/fluid/tests/unittests/test_has_data_op.py new file mode 100644 index 0000000000000..c5318b632dcfe --- /dev/null +++ b/python/paddle/fluid/tests/unittests/test_has_data_op.py @@ -0,0 +1,26 @@ +from paddle.fluid.op import Operator +import paddle.fluid.core as core +import unittest +import numpy as np + + +class BeamSearchOpTester(unittest.TestCase): + def setUp(self): + self.scope = core.Scope() + self.scope.var('X') + self.scope.var('Out') + self.place = core.CUDAPlace(0) + x_data = np.array([]) + x_tensor = self.scope.var('X').get_tensor() + x_tensor.set(x_data, self.place) + out_tensor = self.scope.var('Out').get_tensor() + + def test_run(self): + op = Operator('has_data', X='X', Out='Out') + op.run(self.scope, self.place) + out_tensor = self.scope.find_var('Out').get_tensor() + print 'output: ', np.array(out_tensor) + + +if __name__ == '__main__': + unittest.main() From 2de44cc04a513388d5f03a1431293a2e89fe7c0d Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Thu, 10 May 2018 17:28:17 +0800 Subject: [PATCH 2/9] Added copyright --- paddle/fluid/operators/has_data_op.h | 1 - .../fluid/tests/unittests/test_has_data_op.py | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/operators/has_data_op.h b/paddle/fluid/operators/has_data_op.h index d1465dcba402c..1f00ede97664c 100644 --- a/paddle/fluid/operators/has_data_op.h +++ b/paddle/fluid/operators/has_data_op.h @@ -16,7 +16,6 @@ limitations under the License. */ #include #include #include "paddle/fluid/framework/op_registry.h" -#include "paddle/fluid/operators/elementwise_op_function.h" #include "paddle/fluid/platform/transform.h" namespace paddle { diff --git a/python/paddle/fluid/tests/unittests/test_has_data_op.py b/python/paddle/fluid/tests/unittests/test_has_data_op.py index c5318b632dcfe..218a5c6a3dba7 100644 --- a/python/paddle/fluid/tests/unittests/test_has_data_op.py +++ b/python/paddle/fluid/tests/unittests/test_has_data_op.py @@ -1,3 +1,17 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from paddle.fluid.op import Operator import paddle.fluid.core as core import unittest From bc8f059679e4d269ad6355f5670d8ec9285bf138 Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Thu, 10 May 2018 18:36:13 +0800 Subject: [PATCH 3/9] Added a test case --- paddle/fluid/operators/has_data_op.cc | 4 ---- .../fluid/tests/unittests/test_has_data_op.py | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/operators/has_data_op.cc b/paddle/fluid/operators/has_data_op.cc index 39ff3f50d9acb..b55d3247fa851 100644 --- a/paddle/fluid/operators/has_data_op.cc +++ b/paddle/fluid/operators/has_data_op.cc @@ -16,8 +16,6 @@ limitations under the License. */ #include #include "paddle/fluid/framework/op_registry.h" -#include - namespace paddle { namespace operators { @@ -47,9 +45,7 @@ class HasDataOp : public framework::OperatorWithKernel { "Input(X) of HasDataOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of HasDataOp should not be null."); - std::cout << "Before set\n"; ctx->SetOutputDim("Out", {1}); - std::cout << "After set\n"; ctx->ShareLoD("X", "Out"); } diff --git a/python/paddle/fluid/tests/unittests/test_has_data_op.py b/python/paddle/fluid/tests/unittests/test_has_data_op.py index 218a5c6a3dba7..005fb76f85d46 100644 --- a/python/paddle/fluid/tests/unittests/test_has_data_op.py +++ b/python/paddle/fluid/tests/unittests/test_has_data_op.py @@ -18,13 +18,13 @@ import numpy as np -class BeamSearchOpTester(unittest.TestCase): +class HasDataOpTester(unittest.TestCase): def setUp(self): self.scope = core.Scope() self.scope.var('X') self.scope.var('Out') self.place = core.CUDAPlace(0) - x_data = np.array([]) + x_data = np.array([1]) x_tensor = self.scope.var('X').get_tensor() x_tensor.set(x_data, self.place) out_tensor = self.scope.var('Out').get_tensor() @@ -36,5 +36,17 @@ def test_run(self): print 'output: ', np.array(out_tensor) +class HasDataOpFalseTester(HasDataOpTester): + def setUp(self): + self.scope = core.Scope() + self.scope.var('X') + self.scope.var('Out') + self.place = core.CUDAPlace(0) + x_data = np.array([]) + x_tensor = self.scope.var('X').get_tensor() + x_tensor.set(x_data, self.place) + out_tensor = self.scope.var('Out').get_tensor() + + if __name__ == '__main__': unittest.main() From d0c74944e8741cc34bf31c6cdd79145ed4e3ea8b Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Thu, 10 May 2018 18:40:54 +0800 Subject: [PATCH 4/9] Added a test case --- python/paddle/fluid/tests/unittests/test_has_data_op.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_has_data_op.py b/python/paddle/fluid/tests/unittests/test_has_data_op.py index 005fb76f85d46..e6d0d86ca99ba 100644 --- a/python/paddle/fluid/tests/unittests/test_has_data_op.py +++ b/python/paddle/fluid/tests/unittests/test_has_data_op.py @@ -23,7 +23,7 @@ def setUp(self): self.scope = core.Scope() self.scope.var('X') self.scope.var('Out') - self.place = core.CUDAPlace(0) + self.place = core.CPUPlace() x_data = np.array([1]) x_tensor = self.scope.var('X').get_tensor() x_tensor.set(x_data, self.place) @@ -36,7 +36,7 @@ def test_run(self): print 'output: ', np.array(out_tensor) -class HasDataOpFalseTester(HasDataOpTester): +class HasDataOpGPUTester(HasDataOpTester): def setUp(self): self.scope = core.Scope() self.scope.var('X') From 2d498b1af14fa19f73f372e54e2e54dc90911cb7 Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Thu, 10 May 2018 19:10:45 +0800 Subject: [PATCH 5/9] Implemented the python API for has_data_op --- python/paddle/fluid/layers/control_flow.py | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 4b707973e2739..a2e8fe3b7b34a 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -49,6 +49,7 @@ 'reorder_lod_tensor_by_rank', 'ParallelDo', 'Print', + 'has_data', ] @@ -1562,3 +1563,33 @@ def reorder_lod_tensor_by_rank(x, rank_table): 'RankTable': [rank_table]}, outputs={'Out': [out]}) return out + + +def has_data(x, cond=None, **ignored): + """ + **Less than** + + This layer returns the truth value of whether the variable contains data. + + Args: + x(Variable): Operand of *has_data* + cond(Variable|None): Optional output variable to store the result of *has_data* + + Returns: + Variable: The tensor variable storing the output of *has_data*. + + Examples: + .. code-block:: python + + less = fluid.layers.has_data(x=label) + """ + helper = LayerHelper("has_data", **locals()) + if cond is None: + cond = helper.create_tmp_variable(dtype='bool') + cond.stop_gradient = True + + helper.append_op( + type='has_data', + inputs={'X': [x]}, + outputs={'Out': [cond]}) + return cond From 36e7845096ef9e35241573fb34e1d02b349fbb70 Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Tue, 15 May 2018 06:30:12 +0000 Subject: [PATCH 6/9] style fix --- paddle/fluid/operators/has_data_op.cc | 29 +++++++++++++-------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/paddle/fluid/operators/has_data_op.cc b/paddle/fluid/operators/has_data_op.cc index b55d3247fa851..ccf30c76a31e9 100644 --- a/paddle/fluid/operators/has_data_op.cc +++ b/paddle/fluid/operators/has_data_op.cc @@ -40,23 +40,23 @@ class HasDataOp : public framework::OperatorWithKernel { using framework::OperatorWithKernel::OperatorWithKernel; protected: - void InferShape(framework::InferShapeContext *ctx) const override { - PADDLE_ENFORCE(ctx->HasInput("X"), - "Input(X) of HasDataOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("Out"), - "Output(Out) of HasDataOp should not be null."); - ctx->SetOutputDim("Out", {1}); - ctx->ShareLoD("X", "Out"); - } + void InferShape(framework::InferShapeContext *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("X"), + "Input(X) of HasDataOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("Out"), + "Output(Out) of HasDataOp should not be null."); + ctx->SetOutputDim("Out", {1}); + ctx->ShareLoD("X", "Out"); + } - framework::OpKernelType GetExpectedKernelType( - const framework::ExecutionContext &ctx) const override { - framework::OpKernelType kt = framework::OpKernelType( + framework::OpKernelType GetExpectedKernelType( + const framework::ExecutionContext &ctx) const override { + framework::OpKernelType kt = framework::OpKernelType( framework::ToDataType( ctx.Input("X")->type()), platform::CPUPlace()); - return kt; - } + return kt; + } }; } // namespace operators @@ -66,8 +66,7 @@ namespace ops = paddle::operators; REGISTER_OPERATOR(has_data, ops::HasDataOp, ops::HasDataOpMaker); REGISTER_OP_CPU_KERNEL( - has_data, - ops::HasDataOpKernel, + has_data, ops::HasDataOpKernel, ops::HasDataOpKernel, ops::HasDataOpKernel, ops::HasDataOpKernel); From 08a1708778298f7ff1d5f075cfaa55db75d07e73 Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Tue, 15 May 2018 07:00:12 +0000 Subject: [PATCH 7/9] style ifx --- paddle/fluid/operators/has_data_op.cc | 3 +-- paddle/fluid/operators/has_data_op.h | 3 +-- python/paddle/fluid/layers/control_flow.py | 5 +---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/paddle/fluid/operators/has_data_op.cc b/paddle/fluid/operators/has_data_op.cc index ccf30c76a31e9..4a3ae27e8352d 100644 --- a/paddle/fluid/operators/has_data_op.cc +++ b/paddle/fluid/operators/has_data_op.cc @@ -52,8 +52,7 @@ class HasDataOp : public framework::OperatorWithKernel { framework::OpKernelType GetExpectedKernelType( const framework::ExecutionContext &ctx) const override { framework::OpKernelType kt = framework::OpKernelType( - framework::ToDataType( - ctx.Input("X")->type()), + framework::ToDataType(ctx.Input("X")->type()), platform::CPUPlace()); return kt; } diff --git a/paddle/fluid/operators/has_data_op.h b/paddle/fluid/operators/has_data_op.h index 1f00ede97664c..0bad7a7f4acf2 100644 --- a/paddle/fluid/operators/has_data_op.h +++ b/paddle/fluid/operators/has_data_op.h @@ -29,8 +29,7 @@ class HasDataOpKernel : public framework::OpKernel { auto* output = context.Output("Out"); size_t mem_size = input->memory_size(); - auto* output_data = - output->mutable_data(platform::CPUPlace()); + auto* output_data = output->mutable_data(platform::CPUPlace()); if (mem_size > 0) { output_data[0] = true; } else { diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index a2e8fe3b7b34a..76dd2ebc1d694 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1588,8 +1588,5 @@ def has_data(x, cond=None, **ignored): cond = helper.create_tmp_variable(dtype='bool') cond.stop_gradient = True - helper.append_op( - type='has_data', - inputs={'X': [x]}, - outputs={'Out': [cond]}) + helper.append_op(type='has_data', inputs={'X': [x]}, outputs={'Out': [cond]}) return cond From 38f31b2920536f59e49a31accff39b4710e7f511 Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Tue, 15 May 2018 07:18:40 +0000 Subject: [PATCH 8/9] style fix --- python/paddle/fluid/layers/control_flow.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/layers/control_flow.py b/python/paddle/fluid/layers/control_flow.py index 76dd2ebc1d694..ac34f36488efe 100644 --- a/python/paddle/fluid/layers/control_flow.py +++ b/python/paddle/fluid/layers/control_flow.py @@ -1588,5 +1588,6 @@ def has_data(x, cond=None, **ignored): cond = helper.create_tmp_variable(dtype='bool') cond.stop_gradient = True - helper.append_op(type='has_data', inputs={'X': [x]}, outputs={'Out': [cond]}) + helper.append_op( + type='has_data', inputs={'X': [x]}, outputs={'Out': [cond]}) return cond From a7ae9b5f752b467216935aa18c872089e25956a8 Mon Sep 17 00:00:00 2001 From: ktlichkid Date: Tue, 15 May 2018 16:30:05 +0800 Subject: [PATCH 9/9] Style fix --- paddle/fluid/operators/has_data_op.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/paddle/fluid/operators/has_data_op.cc b/paddle/fluid/operators/has_data_op.cc index 4a3ae27e8352d..48ffbed0c2aa7 100644 --- a/paddle/fluid/operators/has_data_op.cc +++ b/paddle/fluid/operators/has_data_op.cc @@ -21,8 +21,7 @@ namespace operators { class HasDataOpMaker : public framework::OpProtoAndCheckerMaker { public: - HasDataOpMaker(OpProto *proto, OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { + void Make() override { // inputs and outputs stored in proto AddInput("X", "(LoDTensor) the LoDTensor to check"); AddOutput("Out", "(LoDTensor) the ouput of has_data_op");