From 911501769b7fbf9f5dca0ccd10185dbbf8db301a Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 31 Jan 2018 15:43:09 +0800 Subject: [PATCH 01/32] init polynomial_decay --- python/paddle/v2/fluid/layers/control_flow.py | 30 ++++++++ python/paddle/v2/fluid/learning_rate_decay.py | 45 +++++++++++- .../fluid/tests/test_learning_rate_decay.py | 73 ++++++++++++------- 3 files changed, 120 insertions(+), 28 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index 0fcbfe0e2f2f9..ef026f26382b5 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -973,6 +973,36 @@ def less_than(x, y, cond=None, **ignored): return cond +def equal(x, y, cond=None, **ignored): + """ + **equal** + + This layer returns the truth value of :math:`x == y` elementwise. + + Args: + x(Variable): First operand of *equal* + y(Variable): Second operand of *equal* + cond(Variable|None): Optional output variable to store the result of *equal* + + Returns: + Variable: The tensor variable storing the output of *equal*. + + Examples: + .. code-block:: python + + less = fluid.layers.equal(x=label, y=limit) + """ + helper = LayerHelper("equal", **locals()) + if cond is None: + cond = helper.create_tmp_variable(dtype='bool') + cond.stop_gradient = True + + helper.append_op( + type='equal', inputs={'X': [x], + 'Y': [y]}, outputs={'Out': [cond]}) + return cond + + def array_read(array, i): """This function performs the operation to read the data in as an LOD_TENSOR_ARRAY. diff --git a/python/paddle/v2/fluid/learning_rate_decay.py b/python/paddle/v2/fluid/learning_rate_decay.py index 96b3e9a0d73ce..846d2f482451b 100644 --- a/python/paddle/v2/fluid/learning_rate_decay.py +++ b/python/paddle/v2/fluid/learning_rate_decay.py @@ -101,7 +101,7 @@ def inverse_time_decay(learning_rate, ```python if staircase: decayed_learning_rate = learning_rate / (1 + decay_rate * floor(global_step / decay_step)) - else + else: decayed_learning_rate = learning_rate / (1 + decay_rate * global_step / decay_step) ``` Args: @@ -123,3 +123,46 @@ def inverse_time_decay(learning_rate, div_res = layers.floor(x=div_res) return learning_rate / (1 + decay_rate * div_res) + + +def polynomial_decay(learning_rate, + global_step, + decay_steps, + end_learning_rate=0.0001, + power=1.0, + cycle=False): + """Applies inverse time decay to the initial learning rate. + + ```python + if cycle: + decay_steps = decay_steps * ceil(global_step / decay_steps) + else: + global_step = min(global_step, decay_steps) + decayed_learning_rate = (learning_rate - end_learning_rate) * + (1 - global_step / decay_steps) ^ power + + end_learning_rate + ``` + Args: + learning_rate: A scalar float32 value or a Variable. This + will be the initial learning rate during training + global_step: A Variable that record the training step. + decay_steps: A Python `int32` number. + end_learning_rate: A Python `float` number. + power: A Python `float` number + cycle: Boolean. If set true, decay the learning rate every decay_steps. + + Returns: + The decayed learning rate + """ + if not isinstance(global_step, Variable): + raise ValueError("global_step is required for inverse_time_decay.") + + if cycle: + pass + else: + decay_steps_var = layers.fill_constant( + shape=[1], dtype='float32', value=float(decay_steps)) + global_step = layers.elementwise_min(x=global_step, y=decay_steps_var) + + return (learning_rate - end_learning_rate) * \ + ((1 - global_step / decay_steps) ** power) + end_learning_rate diff --git a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py index dc348cf2d2169..804ef4711d38e 100644 --- a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py +++ b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py @@ -15,6 +15,8 @@ import unittest import math +import copy + import paddle.v2.fluid.framework as framework import paddle.v2.fluid as fluid import paddle.v2.fluid.layers as layers @@ -54,21 +56,26 @@ def inverse_time_decay(learning_rate, return learning_rate / (1 + decay_rate * temp) -class TestLearningRateDecay(unittest.TestCase): - def check_decay(self, python_decay_fn, fluid_decay_fn, staircase): - init_lr = 1.0 - decay_steps = 5 - decay_rate = 0.5 +def polynomial_decay(learning_rate, + global_step, + decay_steps, + end_learning_rate=0.0001, + power=1.0, + cycle=False): + if cycle: + decay_steps = decay_steps * math.ceil(global_step / decay_steps) + else: + global_step = min(global_step, decay_steps) + return (learning_rate - end_learning_rate) * \ + (1 - float(global_step) / float(decay_steps)) ** power + end_learning_rate + +class TestLearningRateDecay(unittest.TestCase): + def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): global_step = layers.create_global_var( shape=[1], value=0.0, dtype='float32', persistable=True) - decayed_lr = fluid_decay_fn( - learning_rate=init_lr, - global_step=global_step, - decay_steps=decay_steps, - decay_rate=decay_rate, - staircase=staircase) + decayed_lr = fluid_decay_fn(global_step=global_step, **kwargs) layers.increment(global_step, 1.0) place = fluid.CPUPlace() @@ -79,31 +86,43 @@ def check_decay(self, python_decay_fn, fluid_decay_fn, staircase): step_val, lr_val = exe.run(fluid.default_main_program(), feed=[], fetch_list=[global_step, decayed_lr]) - python_decayed_lr = python_decay_fn( - learning_rate=init_lr, - global_step=step, - decay_steps=decay_steps, - decay_rate=decay_rate, - staircase=staircase) + python_decayed_lr = python_decay_fn(global_step=step, **kwargs) self.assertAlmostEqual(python_decayed_lr, lr_val[0]) def test_decay(self): + common_kwargs_true = { + "learning_rate": 1.0, + "decay_steps": 5, + "decay_rate": 0.5, + "staircase": True + } + common_kwargs_false = copy.deepcopy(common_kwargs_true) + common_kwargs_false["staircase"] = False + decay_fns = [ - (exponential_decay, lr_decay.exponential_decay, True), - (exponential_decay, lr_decay.exponential_decay, False), - (natural_exp_decay, lr_decay.natural_exp_decay, True), - (natural_exp_decay, lr_decay.natural_exp_decay, False), - (inverse_time_decay, lr_decay.inverse_time_decay, True), - (inverse_time_decay, lr_decay.inverse_time_decay, False), + (exponential_decay, lr_decay.exponential_decay, common_kwargs_true), + (exponential_decay, lr_decay.exponential_decay, + common_kwargs_false), + (natural_exp_decay, lr_decay.natural_exp_decay, common_kwargs_true), + (natural_exp_decay, lr_decay.natural_exp_decay, + common_kwargs_false), + (inverse_time_decay, lr_decay.inverse_time_decay, + common_kwargs_true), + (inverse_time_decay, lr_decay.inverse_time_decay, + common_kwargs_false), + (polynomial_decay, lr_decay.polynomial_decay, { + "learning_rate": 1.0, + "decay_steps": 5, + "cycle": False + }), ] - for py_decay_fn, fluid_decay_fn, staircase in decay_fns: - print("decay_fn=" + str(py_decay_fn) + " staircase=" + str( - staircase)) + for py_decay_fn, fluid_decay_fn, kwargs in decay_fns: + print("decay_fn=" + py_decay_fn.__name__ + " kwargs=" + str(kwargs)) main_program = framework.Program() startup_program = framework.Program() with framework.program_guard(main_program, startup_program): - self.check_decay(py_decay_fn, fluid_decay_fn, staircase) + self.check_decay(py_decay_fn, fluid_decay_fn, kwargs) if __name__ == '__main__': From b591ac7c1d49ee630bc7e0fd9846af2a541c1f1a Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 31 Jan 2018 22:45:46 +0800 Subject: [PATCH 02/32] test polynomial_decay --- paddle/operators/reshape_op.cc | 6 ++- paddle/operators/reshape_op.cu | 6 ++- python/paddle/v2/fluid/layers/control_flow.py | 1 + python/paddle/v2/fluid/learning_rate_decay.py | 12 +++++- .../fluid/tests/test_learning_rate_decay.py | 42 ++++++++++++------- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index b9743a5df1092..710d5997bd25b 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -125,6 +125,8 @@ namespace ops = paddle::operators; REGISTER_OP(reshape, ops::ReshapeOp, ops::ReshapeOpMaker, reshape_grad, ops::ReshapeGradOp); REGISTER_OP_CPU_KERNEL(reshape, - ops::ReshapeKernel); + ops::ReshapeKernel, + ops::ReshapeKernel); REGISTER_OP_CPU_KERNEL( - reshape_grad, ops::ReshapeGradKernel); + reshape_grad, ops::ReshapeGradKernel, + ops::ReshapeGradKernel); diff --git a/paddle/operators/reshape_op.cu b/paddle/operators/reshape_op.cu index f487e43b99d5b..dc2583864155c 100644 --- a/paddle/operators/reshape_op.cu +++ b/paddle/operators/reshape_op.cu @@ -16,7 +16,9 @@ limitations under the License. */ REGISTER_OP_CUDA_KERNEL( reshape, - paddle::operators::ReshapeKernel); + paddle::operators::ReshapeKernel, + paddle::operators::ReshapeKernel); REGISTER_OP_CUDA_KERNEL( reshape_grad, - paddle::operators::ReshapeGradKernel); + paddle::operators::ReshapeGradKernel, + paddle::operators::ReshapeGradKernel); diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index ef026f26382b5..55497d31bc54f 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -36,6 +36,7 @@ 'array_write', 'create_array', 'less_than', + 'equal', 'array_read', 'shrink_memory', 'array_length', diff --git a/python/paddle/v2/fluid/learning_rate_decay.py b/python/paddle/v2/fluid/learning_rate_decay.py index 846d2f482451b..c48c450b6b00d 100644 --- a/python/paddle/v2/fluid/learning_rate_decay.py +++ b/python/paddle/v2/fluid/learning_rate_decay.py @@ -158,7 +158,17 @@ def polynomial_decay(learning_rate, raise ValueError("global_step is required for inverse_time_decay.") if cycle: - pass + div_res = layers.ceil(x=(global_step / decay_steps)) + zero_var = layers.fill_constant( + shape=[1, 1], dtype='float32', value=0.0) + one_var = layers.fill_constant(shape=[1, 1], dtype='float32', value=1.0) + + cond = layers.equal(x=zero_var, y=global_step) + true_cond = layers.ConditionalBlock([cond]) + with true_cond.block(): + layers.assign(input=one_var, output=div_res) + return div_res + decay_steps = decay_steps * div_res else: decay_steps_var = layers.fill_constant( shape=[1], dtype='float32', value=float(decay_steps)) diff --git a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py index 804ef4711d38e..53defbbaabdaa 100644 --- a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py +++ b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py @@ -63,11 +63,17 @@ def polynomial_decay(learning_rate, power=1.0, cycle=False): if cycle: - decay_steps = decay_steps * math.ceil(global_step / decay_steps) + div = math.ceil(global_step / float(decay_steps)) + if div == 0: + div = 1 + decay_steps = decay_steps * div else: global_step = min(global_step, decay_steps) - return (learning_rate - end_learning_rate) * \ - (1 - float(global_step) / float(decay_steps)) ** power + end_learning_rate + res = (learning_rate - end_learning_rate) * \ + ((1 - float(global_step) / float(decay_steps)) ** power) + end_learning_rate + print("global_step=" + str(global_step) + " decay_steps=" + str(decay_steps) + + " res=" + str(res)) + return res class TestLearningRateDecay(unittest.TestCase): @@ -87,7 +93,8 @@ def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): feed=[], fetch_list=[global_step, decayed_lr]) python_decayed_lr = python_decay_fn(global_step=step, **kwargs) - self.assertAlmostEqual(python_decayed_lr, lr_val[0]) + print(str(python_decayed_lr) + ":" + str(lr_val[0])) + # self.assertAlmostEqual(python_decayed_lr, lr_val[0]) def test_decay(self): common_kwargs_true = { @@ -100,21 +107,26 @@ def test_decay(self): common_kwargs_false["staircase"] = False decay_fns = [ - (exponential_decay, lr_decay.exponential_decay, common_kwargs_true), - (exponential_decay, lr_decay.exponential_decay, - common_kwargs_false), - (natural_exp_decay, lr_decay.natural_exp_decay, common_kwargs_true), - (natural_exp_decay, lr_decay.natural_exp_decay, - common_kwargs_false), - (inverse_time_decay, lr_decay.inverse_time_decay, - common_kwargs_true), - (inverse_time_decay, lr_decay.inverse_time_decay, - common_kwargs_false), + # (exponential_decay, lr_decay.exponential_decay, common_kwargs_true), + # (exponential_decay, lr_decay.exponential_decay, + # common_kwargs_false), + # (natural_exp_decay, lr_decay.natural_exp_decay, common_kwargs_true), + # (natural_exp_decay, lr_decay.natural_exp_decay, + # common_kwargs_false), + # (inverse_time_decay, lr_decay.inverse_time_decay, + # common_kwargs_true), + # (inverse_time_decay, lr_decay.inverse_time_decay, + # common_kwargs_false), (polynomial_decay, lr_decay.polynomial_decay, { "learning_rate": 1.0, "decay_steps": 5, - "cycle": False + "cycle": True }), + # (polynomial_decay, lr_decay.polynomial_decay, { + # "learning_rate": 1.0, + # "decay_steps": 5, + # "cycle": False + # }), ] for py_decay_fn, fluid_decay_fn, kwargs in decay_fns: From 7d09fe640d00e77ceed1ee83609526326dad4b30 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 31 Jan 2018 23:32:09 +0800 Subject: [PATCH 03/32] complete polynomial_decay --- paddle/operators/conditional_block_op.cc | 19 +++++++++ python/paddle/v2/fluid/learning_rate_decay.py | 8 ++-- .../fluid/tests/test_learning_rate_decay.py | 40 +++++++++---------- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index 3cae61a438431..56260e3273e0c 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -12,6 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include + #include "paddle/framework/executor.h" #include "paddle/framework/op_registry.h" @@ -41,6 +42,16 @@ class ConditionalOp : public framework::OperatorBase { }); return retv; } + + bool IsScalarTrue( + const std::vector &ips) const { + if (ips.size() == 1UL && ips[0]->numel() == 1 && + ips[0]->type().hash_code() == typeid(bool).hash_code()) { + return ips[0]->data()[0]; + } else { + return false; + } + } }; class ConditionalBlockOp : public ConditionalOp { @@ -57,6 +68,10 @@ class ConditionalBlockOp : public ConditionalOp { xs.begin(), xs.end(), [](const framework::LoDTensor *t) { return t->numel() != 0; }); + if (!IsScalarTrue(xs)) { + need_run = false; + } + if (need_run) { auto *scope_var = scope.FindVar(Output("Scope")); PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); @@ -110,6 +125,10 @@ class ConditionalBlockGradOp : public ConditionalOp { xs.begin(), xs.end(), [](const framework::LoDTensor *t) { return t->numel() != 0; }); + if (!IsScalarTrue(xs)) { + need_run = false; + } + if (need_run) { auto *scope_var = scope.FindVar(Input("Scope")); PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); diff --git a/python/paddle/v2/fluid/learning_rate_decay.py b/python/paddle/v2/fluid/learning_rate_decay.py index c48c450b6b00d..0c8ea3873f663 100644 --- a/python/paddle/v2/fluid/learning_rate_decay.py +++ b/python/paddle/v2/fluid/learning_rate_decay.py @@ -159,15 +159,13 @@ def polynomial_decay(learning_rate, if cycle: div_res = layers.ceil(x=(global_step / decay_steps)) - zero_var = layers.fill_constant( - shape=[1, 1], dtype='float32', value=0.0) - one_var = layers.fill_constant(shape=[1, 1], dtype='float32', value=1.0) + zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) + one_var = layers.fill_constant(shape=[1], dtype='float32', value=1.0) - cond = layers.equal(x=zero_var, y=global_step) + cond = layers.equal(x=global_step, y=zero_var) true_cond = layers.ConditionalBlock([cond]) with true_cond.block(): layers.assign(input=one_var, output=div_res) - return div_res decay_steps = decay_steps * div_res else: decay_steps_var = layers.fill_constant( diff --git a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py index 53defbbaabdaa..32df92dc78e5a 100644 --- a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py +++ b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py @@ -69,11 +69,8 @@ def polynomial_decay(learning_rate, decay_steps = decay_steps * div else: global_step = min(global_step, decay_steps) - res = (learning_rate - end_learning_rate) * \ - ((1 - float(global_step) / float(decay_steps)) ** power) + end_learning_rate - print("global_step=" + str(global_step) + " decay_steps=" + str(decay_steps) - + " res=" + str(res)) - return res + return (learning_rate - end_learning_rate) * \ + ((1 - float(global_step) / float(decay_steps)) ** power) + end_learning_rate class TestLearningRateDecay(unittest.TestCase): @@ -93,8 +90,7 @@ def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): feed=[], fetch_list=[global_step, decayed_lr]) python_decayed_lr = python_decay_fn(global_step=step, **kwargs) - print(str(python_decayed_lr) + ":" + str(lr_val[0])) - # self.assertAlmostEqual(python_decayed_lr, lr_val[0]) + self.assertAlmostEqual(python_decayed_lr, lr_val[0]) def test_decay(self): common_kwargs_true = { @@ -107,26 +103,26 @@ def test_decay(self): common_kwargs_false["staircase"] = False decay_fns = [ - # (exponential_decay, lr_decay.exponential_decay, common_kwargs_true), - # (exponential_decay, lr_decay.exponential_decay, - # common_kwargs_false), - # (natural_exp_decay, lr_decay.natural_exp_decay, common_kwargs_true), - # (natural_exp_decay, lr_decay.natural_exp_decay, - # common_kwargs_false), - # (inverse_time_decay, lr_decay.inverse_time_decay, - # common_kwargs_true), - # (inverse_time_decay, lr_decay.inverse_time_decay, - # common_kwargs_false), + (exponential_decay, lr_decay.exponential_decay, common_kwargs_true), + (exponential_decay, lr_decay.exponential_decay, + common_kwargs_false), + (natural_exp_decay, lr_decay.natural_exp_decay, common_kwargs_true), + (natural_exp_decay, lr_decay.natural_exp_decay, + common_kwargs_false), + (inverse_time_decay, lr_decay.inverse_time_decay, + common_kwargs_true), + (inverse_time_decay, lr_decay.inverse_time_decay, + common_kwargs_false), (polynomial_decay, lr_decay.polynomial_decay, { "learning_rate": 1.0, "decay_steps": 5, "cycle": True }), - # (polynomial_decay, lr_decay.polynomial_decay, { - # "learning_rate": 1.0, - # "decay_steps": 5, - # "cycle": False - # }), + (polynomial_decay, lr_decay.polynomial_decay, { + "learning_rate": 1.0, + "decay_steps": 5, + "cycle": False + }), ] for py_decay_fn, fluid_decay_fn, kwargs in decay_fns: From e804f066fbe072eba6b5a98294e738ea2eca5f61 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 1 Feb 2018 10:41:26 +0800 Subject: [PATCH 04/32] fix conditional block op --- paddle/operators/conditional_block_op.cc | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index 56260e3273e0c..d86a4825b7318 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -43,14 +43,17 @@ class ConditionalOp : public framework::OperatorBase { return retv; } - bool IsScalarTrue( + bool IsScalarFalse( const std::vector &ips) const { - if (ips.size() == 1UL && ips[0]->numel() == 1 && - ips[0]->type().hash_code() == typeid(bool).hash_code()) { - return ips[0]->data()[0]; - } else { - return false; + if (ips.size() == 1UL && ips[0]->IsInitialized()) { + if (ips[0]->type().hash_code() == typeid(bool).hash_code() && + ips[0]->numel() == 1) { + if (ips[0]->data()[0] == false) { + return true; + } + } } + return false; } }; @@ -68,7 +71,7 @@ class ConditionalBlockOp : public ConditionalOp { xs.begin(), xs.end(), [](const framework::LoDTensor *t) { return t->numel() != 0; }); - if (!IsScalarTrue(xs)) { + if (IsScalarFalse(xs)) { need_run = false; } @@ -125,7 +128,7 @@ class ConditionalBlockGradOp : public ConditionalOp { xs.begin(), xs.end(), [](const framework::LoDTensor *t) { return t->numel() != 0; }); - if (!IsScalarTrue(xs)) { + if (IsScalarFalse(xs)) { need_run = false; } From 9ee8f776770753bd68e7675c03519d1c5f5e73eb Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 1 Feb 2018 20:38:51 +0800 Subject: [PATCH 05/32] init scalar-switch-case-op --- paddle/framework/framework.proto | 2 ++ paddle/framework/op_desc.cc | 21 +++++++++-- paddle/framework/type_defs.h | 3 +- paddle/framework/var_desc.cc | 1 + paddle/operators/scalar_switch_case_op.cc | 43 +++++++++++++++++++++++ 5 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 paddle/operators/scalar_switch_case_op.cc diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index 5b6ef03f61092..06e65bfdfe85f 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -27,6 +27,7 @@ enum AttrType { BOOLEANS = 7; BLOCK = 8; LONG = 9; + BLOCKS = 10; } // OpDesc describes an instance of a C++ framework::OperatorBase @@ -46,6 +47,7 @@ message OpDesc { repeated bool bools = 11; optional int32 block_idx = 12; optional int64 l = 13; + repeated int32 block_idxs = 14; }; message Var { diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index f8df2cf97ad53..750ea7e692a7d 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -124,11 +124,18 @@ OpDesc::OpDesc(const proto::OpDesc &desc, ProgramDesc *prog, BlockDesc *block) // restore attrs_ for (const proto::OpDesc::Attr &attr : desc_.attrs()) { std::string attr_name = attr.name(); - if (attr.type() != proto::AttrType::BLOCK) { - attrs_[attr_name] = GetAttrValue(attr); - } else { + if (attr.type() == proto::AttrType::BLOCK) { auto bid = attr.block_idx(); attrs_[attr_name] = prog->MutableBlock(bid); + } else if (attr.type() == proto::AttrType::BLOCKS) { + auto block_idxs = attr.block_idxs(); + std::vector ret; + for (auto idx : block_idxs) { + ret.emplace_back(prog->MutableBlock(idx)); + } + attrs_[attr_name] = ret; + } else { + attrs_[attr_name] = GetAttrValue(attr); } } this->block_ = block; @@ -284,6 +291,14 @@ struct SetAttrDescVisitor : public boost::static_visitor { } void operator()(BlockDesc *desc) const { attr_->set_block_idx(desc->ID()); } void operator()(int64_t v) const { attr_->set_l(v); } + void operator()(const std::vector &v) const { + auto *repeated_field = attr_->mutable_block_idxs(); + repeated_field->Clear(); + repeated_field->Reserve(v.size()); + for (auto elem : v) { + *repeated_field->Add() = elem->ID(); + } + } void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } }; diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h index 1eedbbc419ab6..4e1e70cb494f7 100644 --- a/paddle/framework/type_defs.h +++ b/paddle/framework/type_defs.h @@ -35,7 +35,8 @@ using VariableNameMap = std::map>; using Attribute = boost::variant, std::vector, std::vector, bool, - std::vector, BlockDesc*, int64_t>; + std::vector, BlockDesc*, int64_t, + std::vector>; using AttributeMap = std::unordered_map; diff --git a/paddle/framework/var_desc.cc b/paddle/framework/var_desc.cc index 62ab6593ef23c..63ff57b7711bb 100644 --- a/paddle/framework/var_desc.cc +++ b/paddle/framework/var_desc.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/framework/var_desc.h" +#include "paddle/framework/block_desc.h" #include "paddle/platform/enforce.h" namespace paddle { diff --git a/paddle/operators/scalar_switch_case_op.cc b/paddle/operators/scalar_switch_case_op.cc new file mode 100644 index 0000000000000..1d85f177e18ab --- /dev/null +++ b/paddle/operators/scalar_switch_case_op.cc @@ -0,0 +1,43 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/scalar_switch_case_op.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +class ScalarSwitchCaseOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + ScalarSwitchCaseOpProtoMaker(OpProto *proto, OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The conditional variable of this operator.").AsDuplicable(); + AddOutput("Scope", + "(std::vector) The step scope of conditional block. To " + "unify the conditional block, rnn and while op, the type of " + "scope is std::vector"); + AddAttr>( + "sub_blocks", + "The step block of conditional " + "block operator, the length should be the same as X"); + AddComment(R"DOC(Conditional block operator + +Run the sub-block if X is not empty. Params is the other inputs and Out is the +outputs of the sub-block. +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle \ No newline at end of file From 9ae65c4ed44f340ed1dcdf9f3652f5f0eb6b947c Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 2 Feb 2018 14:42:14 +0800 Subject: [PATCH 06/32] switch op can compile --- paddle/framework/op_desc.cc | 20 +++ paddle/framework/op_desc.h | 4 + paddle/operators/scalar_switch_case_op.cc | 43 ----- paddle/operators/switch_op.cc | 186 ++++++++++++++++++++++ 4 files changed, 210 insertions(+), 43 deletions(-) delete mode 100644 paddle/operators/scalar_switch_case_op.cc create mode 100644 paddle/operators/switch_op.cc diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 750ea7e692a7d..4435af7818960 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -213,6 +213,12 @@ void OpDesc::SetBlockAttr(const std::string &name, BlockDesc &block) { need_update_ = true; } +void OpDesc::SetBlocksAttr(const std::string &name, + std::vector &blocks) { + this->attrs_[name] = blocks; + need_update_ = true; +} + void OpDesc::SetAttrMap( const std::unordered_map &attr_map) { attrs_ = attr_map; @@ -231,6 +237,20 @@ int OpDesc::GetBlockAttr(const std::string &name) const { return boost::get(it->second)->ID(); } +std::vector OpDesc::GetBlocksAttr(const std::string &name) const { + auto it = attrs_.find(name); + PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); + auto blocks = boost::get>(it->second); + std::vector retv; + std::transform(blocks.begin(), blocks.end(), retv.begin(), + [](const BlockDesc *block_desc) -> int { + PADDLE_ENFORCE(block_desc != nullptr, + "block desc should not be null"); + return block_desc->ID(); + }); + return retv; +} + const std::unordered_map &OpDesc::GetAttrMap() const { return attrs_; } diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 13695cff59f0b..668cf77418815 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -75,10 +75,14 @@ class OpDesc { void SetBlockAttr(const std::string &name, BlockDesc &block); + void SetBlocksAttr(const std::string &name, std::vector &blocks); + Attribute GetAttr(const std::string &name) const; int GetBlockAttr(const std::string &name) const; + std::vector GetBlocksAttr(const std::string &name) const; + void Rename(const std::string &old_name, const std::string &new_name); void RenameOutput(const std::string &old_name, const std::string &new_name); diff --git a/paddle/operators/scalar_switch_case_op.cc b/paddle/operators/scalar_switch_case_op.cc deleted file mode 100644 index 1d85f177e18ab..0000000000000 --- a/paddle/operators/scalar_switch_case_op.cc +++ /dev/null @@ -1,43 +0,0 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/operators/scalar_switch_case_op.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -class ScalarSwitchCaseOpProtoMaker : public framework::OpProtoAndCheckerMaker { - public: - ScalarSwitchCaseOpProtoMaker(OpProto *proto, OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The conditional variable of this operator.").AsDuplicable(); - AddOutput("Scope", - "(std::vector) The step scope of conditional block. To " - "unify the conditional block, rnn and while op, the type of " - "scope is std::vector"); - AddAttr>( - "sub_blocks", - "The step block of conditional " - "block operator, the length should be the same as X"); - AddComment(R"DOC(Conditional block operator - -Run the sub-block if X is not empty. Params is the other inputs and Out is the -outputs of the sub-block. -)DOC"); - } -}; - -} // namespace operators -} // namespace paddle \ No newline at end of file diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc new file mode 100644 index 0000000000000..698289c224c14 --- /dev/null +++ b/paddle/operators/switch_op.cc @@ -0,0 +1,186 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/executor.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +class SwitchOpProtoMaker : public framework::OpProtoAndCheckerMaker { + public: + SwitchOpProtoMaker(OpProto *proto, OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "The conditional variable of this operator.").AsDuplicable(); + AddOutput("Scope", + "(std::vector) The step scope of conditional block. To " + "unify the conditional block, rnn and while op, " + "the type of scope is std::vector"); + AddAttr>( + "sub_blocks", + "The step block of conditional " + "block operator, the length should be the same as X"); + AddComment(R"DOC(switch operator + +Run one sub block according to condition list, +)DOC"); + } +}; + +class SwitchOpGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto grad_op = new framework::OpDesc(); + grad_op->SetType("switch_grad"); + grad_op->SetInput("X", Input("X")); + grad_op->SetInput("Scope", Output("Scope")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X", false)); + grad_op->SetBlockAttr("sub_blocks", *this->grad_block_[0]); + return std::unique_ptr(grad_op); + } +}; + +class SwitchOpBase : public framework::OperatorBase { + public: + SwitchOpBase(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : framework::OperatorBase(type, inputs, outputs, attrs) {} + + protected: + std::vector InputTensors( + const framework::Scope &scope) const { + std::vector retv; + auto xs = Inputs("X"); + retv.resize(xs.size(), nullptr); + std::transform( + xs.begin(), xs.end(), retv.begin(), + [&scope](const std::string &var_name) -> const framework::LoDTensor * { + auto *var = scope.FindVar(var_name); + PADDLE_ENFORCE(var != nullptr, "Cannot find variable %s", var_name); + return &var->Get(); + }); + return retv; + } +}; + +class SwitchOp : public SwitchOpBase { + public: + SwitchOp(const std::string &type, const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : SwitchOpBase(type, inputs, outputs, attrs) {} + + void Run(const framework::Scope &scope, + const platform::Place &dev_place) const override { + auto xs = InputTensors(scope); + bool need_run = std::all_of( + xs.begin(), xs.end(), + [](const framework::LoDTensor *t) { return t->numel() != 0; }); + + if (need_run) { + auto *scope_var = scope.FindVar(Output("Scope")); + PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); + auto *scopes = scope_var->GetMutable>(); + scopes->resize(1); + scopes->front() = &scope.NewScope(); + auto &cur_scope = *scopes->front(); + + framework::Executor exec(dev_place); + auto *block = Attr("sub_blocks"); + exec.Run(*block->Program(), &cur_scope, block->ID(), false); + } + } +}; + +class SwitchGradOp : public SwitchOpBase { + public: + SwitchGradOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : SwitchOpBase(type, inputs, outputs, attrs) {} + void Run(const framework::Scope &scope, + const platform::Place &dev_place) const override { + auto xs = this->InputTensors(scope); + bool need_run = std::all_of( + xs.begin(), xs.end(), + [](const framework::LoDTensor *t) { return t->numel() != 0; }); + + if (need_run) { + auto *scope_var = scope.FindVar(Input("Scope")); + PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); + auto &scopes = scope_var->Get>(); + framework::Scope &cur_scope = *scopes[0]; + + framework::Executor exec(dev_place); + auto *block = Attr("sub_blocks"); + exec.Run(*block->Program(), &cur_scope, block->ID(), false); + + AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("Params"), + Outputs(framework::GradVarName("Params"))); + + AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("X"), + Outputs(framework::GradVarName("X"))); + } + } + + private: + void AssignLocalGradientToGlobal( + const platform::Place &place, const framework::Scope &cur_scope, + const std::vector &p_names, + const std::vector &pg_names) const { + for (size_t i = 0; i < p_names.size(); ++i) { + auto out_grad_name = pg_names[i]; + auto in_grad_name = framework::GradVarName(p_names[i]); + auto *in_var = cur_scope.FindVar(in_grad_name); + if (in_var == nullptr) { + continue; + } + auto new_in_grad_name = cur_scope.Rename(in_grad_name); + auto assign = framework::OpRegistry::CreateOp( + "assign", {{"X", {new_in_grad_name}}}, {{"Out", {out_grad_name}}}, + framework::AttributeMap{}); + assign->Run(cur_scope, place); + cur_scope.Rename(new_in_grad_name, in_grad_name); + } + } +}; + +class SwitchGradInferShape : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *context) const override { + PADDLE_ENFORCE(context->HasInputs("X")); + if (context->HasInputs("Params")) { + PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("Params"))); + context->SetOutputsDim(framework::GradVarName("Params"), + context->GetInputsDim("Params")); + } + PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("X"))); + context->SetOutputsDim(framework::GradVarName("X"), + context->GetInputsDim("X")); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OPERATOR(switch, ops::SwitchOp, ops::SwitchOpProtoMaker, + ops::SwitchOpGradMaker); +REGISTER_OPERATOR(switch_grad, ops::SwitchGradOp, ops::SwitchGradInferShape); From 3d5b807e263f6ce05fcdd3c43aa7bf5613cd73ad Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 2 Feb 2018 16:19:35 +0800 Subject: [PATCH 07/32] complete forward switch_op --- paddle/framework/op_desc.cc | 2 +- paddle/framework/op_desc.h | 3 ++- paddle/operators/switch_op.cc | 40 +++++++++++++++++++++++++++++------ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 4435af7818960..a9ebdc8806906 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -214,7 +214,7 @@ void OpDesc::SetBlockAttr(const std::string &name, BlockDesc &block) { } void OpDesc::SetBlocksAttr(const std::string &name, - std::vector &blocks) { + const std::vector &blocks) { this->attrs_[name] = blocks; need_update_ = true; } diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 668cf77418815..8589bdc8e67d6 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -75,7 +75,8 @@ class OpDesc { void SetBlockAttr(const std::string &name, BlockDesc &block); - void SetBlocksAttr(const std::string &name, std::vector &blocks); + void SetBlocksAttr(const std::string &name, + const std::vector &blocks); Attribute GetAttr(const std::string &name) const; diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index 698289c224c14..ca6319ac68b89 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -48,8 +48,7 @@ class SwitchOpGradMaker : public framework::SingleGradOpDescMaker { grad_op->SetType("switch_grad"); grad_op->SetInput("X", Input("X")); grad_op->SetInput("Scope", Output("Scope")); - grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X", false)); - grad_op->SetBlockAttr("sub_blocks", *this->grad_block_[0]); + grad_op->SetBlocksAttr("sub_blocks", this->grad_block_); return std::unique_ptr(grad_op); } }; @@ -89,11 +88,38 @@ class SwitchOp : public SwitchOpBase { void Run(const framework::Scope &scope, const platform::Place &dev_place) const override { auto xs = InputTensors(scope); - bool need_run = std::all_of( - xs.begin(), xs.end(), - [](const framework::LoDTensor *t) { return t->numel() != 0; }); + auto blocks = Attr>("sub_blocks"); + + size_t cond_num = xs.size(); + size_t case_num = blocks.size(); + PADDLE_ENFORCE(cond_num == case_num || cond_num + 1 == case_num, + "cond_num %d and case_num %d does not meet requirement", + cond_num, case_num); + + int64_t match_cond_id = -1; + int64_t match_case_id = -1; + + for (size_t i = 0; i < xs.size(); ++i) { + auto cond = xs[i]; + PADDLE_ENFORCE(cond->IsInitialized() && + cond->dims() == framework::make_ddim({1}) && + cond->type().hash_code() == typeid(bool).hash_code(), + "cond should be a scalar bool tensor"); + if (cond->data()[0]) { + match_cond_id = static_cast(i); + break; + } + } + + if (match_cond_id >= 0) { + match_case_id = match_cond_id; + } else if (cond_num + 1 == case_num) { + match_case_id = static_cast(case_num - 1); + } + + if (match_case_id >= 0) { + auto block = blocks[match_case_id]; - if (need_run) { auto *scope_var = scope.FindVar(Output("Scope")); PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); auto *scopes = scope_var->GetMutable>(); @@ -102,7 +128,7 @@ class SwitchOp : public SwitchOpBase { auto &cur_scope = *scopes->front(); framework::Executor exec(dev_place); - auto *block = Attr("sub_blocks"); + exec.Run(*block->Program(), &cur_scope, block->ID(), false); } } From 0b4e4c9697301e6dfad95fe7c2e8a7e4cc539b98 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 2 Feb 2018 16:43:09 +0800 Subject: [PATCH 08/32] add GetMatchCaseIndex --- paddle/operators/switch_op.cc | 55 +++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index ca6319ac68b89..0215ab026eab5 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -76,6 +76,35 @@ class SwitchOpBase : public framework::OperatorBase { }); return retv; } + + int GetMatchCaseIndex( + const std::vector &conditions, + const std::vector &case_blocks) const { + size_t cond_num = conditions.size(); + size_t case_num = case_blocks.size(); + + int match_cond_id = -1; + + for (size_t i = 0; i < conditions.size(); ++i) { + auto cond = conditions[i]; + PADDLE_ENFORCE(cond->IsInitialized() && + cond->dims() == framework::make_ddim({1}) && + cond->type().hash_code() == typeid(bool).hash_code(), + "cond should be a scalar bool tensor"); + if (cond->data()[0]) { + match_cond_id = static_cast(i); + break; + } + } + + if (match_cond_id >= 0) { + return match_cond_id; + } else if (cond_num + 1 == case_num) { + return case_num - 1; + } else { + return -1; + } + } }; class SwitchOp : public SwitchOpBase { @@ -95,29 +124,9 @@ class SwitchOp : public SwitchOpBase { PADDLE_ENFORCE(cond_num == case_num || cond_num + 1 == case_num, "cond_num %d and case_num %d does not meet requirement", cond_num, case_num); - - int64_t match_cond_id = -1; - int64_t match_case_id = -1; - - for (size_t i = 0; i < xs.size(); ++i) { - auto cond = xs[i]; - PADDLE_ENFORCE(cond->IsInitialized() && - cond->dims() == framework::make_ddim({1}) && - cond->type().hash_code() == typeid(bool).hash_code(), - "cond should be a scalar bool tensor"); - if (cond->data()[0]) { - match_cond_id = static_cast(i); - break; - } - } - - if (match_cond_id >= 0) { - match_case_id = match_cond_id; - } else if (cond_num + 1 == case_num) { - match_case_id = static_cast(case_num - 1); - } - + int match_case_id = GetMatchCaseIndex(xs, blocks); if (match_case_id >= 0) { + VLOG(3) << "match case " << match_case_id; auto block = blocks[match_case_id]; auto *scope_var = scope.FindVar(Output("Scope")); @@ -130,6 +139,8 @@ class SwitchOp : public SwitchOpBase { framework::Executor exec(dev_place); exec.Run(*block->Program(), &cur_scope, block->ID(), false); + } else { + VLOG(3) << "no case is matched, do nothing"; } } }; From 7af4dda39d51538756e96f44f3d6ba536a5eacb2 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 2 Feb 2018 17:32:42 +0800 Subject: [PATCH 09/32] add switch_grad_op --- paddle/operators/switch_op.cc | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index 0215ab026eab5..3b2439248d01c 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -155,22 +155,18 @@ class SwitchGradOp : public SwitchOpBase { void Run(const framework::Scope &scope, const platform::Place &dev_place) const override { auto xs = this->InputTensors(scope); - bool need_run = std::all_of( - xs.begin(), xs.end(), - [](const framework::LoDTensor *t) { return t->numel() != 0; }); + auto blocks = Attr>("sub_blocks"); - if (need_run) { + int match_case_id = GetMatchCaseIndex(xs, blocks); + if (match_case_id >= 0) { auto *scope_var = scope.FindVar(Input("Scope")); PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); auto &scopes = scope_var->Get>(); framework::Scope &cur_scope = *scopes[0]; framework::Executor exec(dev_place); - auto *block = Attr("sub_blocks"); - exec.Run(*block->Program(), &cur_scope, block->ID(), false); - - AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("Params"), - Outputs(framework::GradVarName("Params"))); + exec.Run(*blocks[match_case_id]->Program(), &cur_scope, + blocks[match_case_id]->ID(), false); AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("X"), Outputs(framework::GradVarName("X"))); @@ -203,14 +199,6 @@ class SwitchGradInferShape : public framework::InferShapeBase { public: void operator()(framework::InferShapeContext *context) const override { PADDLE_ENFORCE(context->HasInputs("X")); - if (context->HasInputs("Params")) { - PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("Params"))); - context->SetOutputsDim(framework::GradVarName("Params"), - context->GetInputsDim("Params")); - } - PADDLE_ENFORCE(context->HasOutputs(framework::GradVarName("X"))); - context->SetOutputsDim(framework::GradVarName("X"), - context->GetInputsDim("X")); } }; From bdfb8354b9621592ea162a7e652c889597e29224 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 13:02:49 +0800 Subject: [PATCH 10/32] init switch Python API --- paddle/operators/switch_op.cc | 7 +- python/paddle/v2/fluid/layers/control_flow.py | 101 ++++++++++++++++++ 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index 3b2439248d01c..0c6899918f609 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -26,7 +26,8 @@ class SwitchOpProtoMaker : public framework::OpProtoAndCheckerMaker { AddOutput("Scope", "(std::vector) The step scope of conditional block. To " "unify the conditional block, rnn and while op, " - "the type of scope is std::vector"); + "the type of scope is std::vector") + .AsIntermediate(); AddAttr>( "sub_blocks", "The step block of conditional " @@ -168,8 +169,8 @@ class SwitchGradOp : public SwitchOpBase { exec.Run(*blocks[match_case_id]->Program(), &cur_scope, blocks[match_case_id]->ID(), false); - AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("X"), - Outputs(framework::GradVarName("X"))); + // AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("X"), + // Outputs(framework::GradVarName("X"))); } } diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index 0fcbfe0e2f2f9..752aea0ad4b2b 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1115,6 +1115,107 @@ def complete(self): attrs={'sub_block': inside_block}) +class SwitchScopeGuard(object): + """ + 1. only switch.case can be called inside a switch.scope + 2. switch.case can only be called inside a switch.scope + """ + + def __init__(self, op): + if not isinstance(op, SwitchOp): + raise TypeError("op should be switch") + self.switch_op = op + + def __enter__(self): + """ + set flag that now is inside switch.block {} + :return: + """ + self.switch_op.inside_scope = True + + def __exit__(self, exc_type, exc_val, exc_tb): + self.switch_op.inside_scope = False + if exc_type is not None: + return False # re-raise exception + + cond_num = len(self.switch_op.conditions) + case_block_num = len(self.switch_op.case_blocks) + + tmp = case_block_num - cond_num + if tmp != 0 and tmp != 1: + raise ValueError("case_num=%d, cond_num=%d not match", + case_block_num, cond_num) + + self.switch_op.helper.append_op( + type='switch', + inputs={'X': self.switch_op.conditions}, + attrs={'level': 0}) + + return True + + +class CaseBlockGuard(BlockGuard): + def __init__(self, op, condition): + if not isinstance(op, SwitchOp): + raise TypeError("op should be switch") + if not op.inside_scope: + raise ValueError( + "switch.case can only be called inside switch.block") + self.switch_op = op + super(CaseBlockGuard, self).__init__(self.switch_op.helper.main_program) + self.switch_op.conditions.append(condition) + + def __enter__(self): + return super(CaseBlockGuard, self).__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.switch_op.case_blocks.append(self.main_program.current_block()) + return super(CaseBlockGuard, self).__exit__(exc_type, exc_val, exc_tb) + + +class DefaultCaseBlockGuard(BlockGuard): + def __init__(self, op): + if not isinstance(op, SwitchOp): + raise TypeError("op should be switch") + if not op.inside_scope: + raise ValueError( + "switch.case can only be called inside switch.block") + self.switch_op = op + super(DefaultCaseBlockGuard, + self).__init__(self.switch_op.helper.main_program) + + def __enter__(self): + return super(DefaultCaseBlockGuard, self).__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + self.switch_op.case_blocks.append(self.main_program.current_block()) + return super(DefaultCaseBlockGuard, self).__exit__(exc_type, exc_val, + exc_tb) + + +class SwitchOp(object): + def __init__(self, name=None): + self.helper = LayerHelper('switch', name=name) + self.inside_scope = False + self.conditions = [] + self.case_blocks = [] + + def scope(self): + """the scope for this switch op, works like `{}` + """ + return SwitchScopeGuard(self) + + def case(self, condition): + """create a new block for this condition + """ + return CaseBlockGuard(self, condition) + + def default(self): + """create a default case for this switch + """ + return DefaultCaseBlockGuard(self) + + class IfElseBlockGuard(object): def __init__(self, is_true, ifelse): if not isinstance(ifelse, IfElse): From 33fcaed1a84ea74b1dec42aceecd88c8a904ce72 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 14:04:27 +0800 Subject: [PATCH 11/32] add test_switch --- doc/design/switch.md | 3 +- paddle/operators/switch_op.cc | 8 +-- paddle/pybind/protobuf.cc | 1 + python/paddle/v2/fluid/layers/control_flow.py | 37 +++++++++-- python/paddle/v2/fluid/tests/test_switch.py | 64 +++++++++++++++++++ 5 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 python/paddle/v2/fluid/tests/test_switch.py diff --git a/doc/design/switch.md b/doc/design/switch.md index 9db1b2782a521..827d0601c621e 100644 --- a/doc/design/switch.md +++ b/doc/design/switch.md @@ -10,8 +10,7 @@ The following example shows the usage of `fluid.switch`. a = fluid.Var(10) b = fluid.Var(0) -switch = fluid.switch() -with switch.block(): +with switch() as switch: with switch.case(fluid.less_equal(a, 10)): fluid.print("Case 1") with switch.case(fluid.larger(a, 0)): diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index 0c6899918f609..863c5e5efcf17 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -29,7 +29,7 @@ class SwitchOpProtoMaker : public framework::OpProtoAndCheckerMaker { "the type of scope is std::vector") .AsIntermediate(); AddAttr>( - "sub_blocks", + "case_blocks", "The step block of conditional " "block operator, the length should be the same as X"); AddComment(R"DOC(switch operator @@ -49,7 +49,7 @@ class SwitchOpGradMaker : public framework::SingleGradOpDescMaker { grad_op->SetType("switch_grad"); grad_op->SetInput("X", Input("X")); grad_op->SetInput("Scope", Output("Scope")); - grad_op->SetBlocksAttr("sub_blocks", this->grad_block_); + grad_op->SetBlocksAttr("case_blocks", this->grad_block_); return std::unique_ptr(grad_op); } }; @@ -118,7 +118,7 @@ class SwitchOp : public SwitchOpBase { void Run(const framework::Scope &scope, const platform::Place &dev_place) const override { auto xs = InputTensors(scope); - auto blocks = Attr>("sub_blocks"); + auto blocks = Attr>("case_blocks"); size_t cond_num = xs.size(); size_t case_num = blocks.size(); @@ -156,7 +156,7 @@ class SwitchGradOp : public SwitchOpBase { void Run(const framework::Scope &scope, const platform::Place &dev_place) const override { auto xs = this->InputTensors(scope); - auto blocks = Attr>("sub_blocks"); + auto blocks = Attr>("case_blocks"); int match_case_id = GetMatchCaseIndex(xs, blocks); if (match_case_id >= 0) { diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index 371d6119d4ab7..af5291681fc82 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -271,6 +271,7 @@ void BindOpDesc(py::module &m) { .def("set_attr", &OpDesc::SetAttr) .def("attr", &OpDesc::GetAttr) .def("set_block_attr", &OpDesc::SetBlockAttr) + .def("set_blocks_attr", &OpDesc::SetBlocksAttr) .def("set_serialized_attr", [](OpDesc &self, const std::string &name, const py::bytes &seriralized) { diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index 752aea0ad4b2b..b33b76f19e066 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -27,6 +27,7 @@ 'StaticRNNMemoryLink', 'WhileGuard', 'While', + 'Switch', 'lod_rank_table', 'max_sequence_len', 'topk', @@ -1122,7 +1123,7 @@ class SwitchScopeGuard(object): """ def __init__(self, op): - if not isinstance(op, SwitchOp): + if not isinstance(op, Switch): raise TypeError("op should be switch") self.switch_op = op @@ -1156,7 +1157,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class CaseBlockGuard(BlockGuard): def __init__(self, op, condition): - if not isinstance(op, SwitchOp): + if not isinstance(op, Switch): raise TypeError("op should be switch") if not op.inside_scope: raise ValueError( @@ -1175,7 +1176,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class DefaultCaseBlockGuard(BlockGuard): def __init__(self, op): - if not isinstance(op, SwitchOp): + if not isinstance(op, Switch): raise TypeError("op should be switch") if not op.inside_scope: raise ValueError( @@ -1193,7 +1194,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): exc_tb) -class SwitchOp(object): +class Switch(object): def __init__(self, name=None): self.helper = LayerHelper('switch', name=name) self.inside_scope = False @@ -1215,6 +1216,34 @@ def default(self): """ return DefaultCaseBlockGuard(self) + def __enter__(self): + """ + set flag that now is inside switch.block {} + :return: + """ + self.inside_scope = True + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.inside_scope = False + if exc_type is not None: + return False # re-raise exception + + cond_num = len(self.conditions) + case_block_num = len(self.case_blocks) + + tmp = case_block_num - cond_num + if tmp != 0 and tmp != 1: + raise ValueError("case_num=%d, cond_num=%d not match", + case_block_num, cond_num) + + self.helper.append_op( + type='switch', + inputs={'X': self.conditions}, + attrs={'case_blocks': self.case_blocks}) + + return True + class IfElseBlockGuard(object): def __init__(self, is_true, ifelse): diff --git a/python/paddle/v2/fluid/tests/test_switch.py b/python/paddle/v2/fluid/tests/test_switch.py new file mode 100644 index 0000000000000..023e7cef65ef5 --- /dev/null +++ b/python/paddle/v2/fluid/tests/test_switch.py @@ -0,0 +1,64 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import paddle.v2.fluid.core as core +import paddle.v2.fluid.layers as layers +import paddle.v2.fluid.framework as framework +from paddle.v2.fluid.executor import Executor +from paddle.v2.fluid.framework import default_startup_program + + +class TestSwitch(unittest.TestCase): + def check_switch(self, value): + + x = layers.fill_constant(shape=[1], dtype='float32', value=value) + + zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) + one_var = layers.fill_constant(shape=[1], dtype='float32', value=1.0) + two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0) + + cond1 = layers.less_than(x, zero_var) + cond2 = layers.less_than(x, one_var) + cond3 = layers.less_than(x, two_var) + + result = layers.create_global_var( + shape=[1], value=-1.0, dtype='float32', persistable=True) + + with layers.Switch() as switch: + with switch.case(cond1): + layers.assign(result, zero_var) + with switch.case(cond2): + layers.assign(result, one_var) + with switch.case(cond3): + layers.assign(result, two_var) + + cpu = core.CPUPlace() + exe = Executor(cpu) + exe.run(default_startup_program()) + + out = exe.run(feed={}, fetch_list=[result])[0] + return out + + def test_switch(self): + main_program = framework.Program() + startup_program = framework.Program() + with framework.program_guard(main_program, startup_program): + result = self.check_switch(0.0) + self.assertEqual(result, 1) + + +if __name__ == '__main__': + unittest.main() From 83e1bc9a4963ab7a3706a31d31635fb06f0b3433 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 14:29:09 +0800 Subject: [PATCH 12/32] support set block list in python --- python/paddle/v2/fluid/framework.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 69cbebe41e573..035f55e88347f 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -468,6 +468,16 @@ def find_name(var_list, name): arg.op = self self.desc.set_output(out_proto.name, out_arg_names) + def __is_block_list__(attr): + if not isinstance(attr, list): + return False + if len(attr) == 0: + return False + for item in attr: + if not isinstance(item, Block): + return False + return True + if attrs is not None: if not isinstance(attrs, dict): raise TypeError("'attrs' should be a dict.") @@ -481,6 +491,9 @@ def find_name(var_list, name): isinstance(attrs[attr_name], core.ProgramDesc): self.desc.set_serialized_attr( attr_name, attrs[attr_name].serialize_to_string()) + elif __is_block_list__(attrs[attr_name]): + self.desc.set_blocks_attr( + attr_name, [item.desc for item in attrs[attr_name]]) else: self.desc.set_attr(attr_name, attrs[attr_name]) From 5fe59364219f84cfe63a12c926bbea5bd785a47f Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 15:58:02 +0800 Subject: [PATCH 13/32] fix scope problem --- paddle/operators/switch_op.cc | 4 ++ python/paddle/v2/fluid/framework.py | 2 +- python/paddle/v2/fluid/layers/control_flow.py | 52 +++---------------- 3 files changed, 12 insertions(+), 46 deletions(-) diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index 863c5e5efcf17..6a9e4d8f4e176 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -23,6 +23,10 @@ class SwitchOpProtoMaker : public framework::OpProtoAndCheckerMaker { SwitchOpProtoMaker(OpProto *proto, OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The conditional variable of this operator.").AsDuplicable(); + AddInput("ScopeIn", + "(std::vector) The step scope of conditional block. To " + "unify the conditional block, rnn and while op, " + "the type of scope is std::vector"); AddOutput("Scope", "(std::vector) The step scope of conditional block. To " "unify the conditional block, rnn and while op, " diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 035f55e88347f..3c6c95437d325 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -502,7 +502,7 @@ def __is_block_list__(attr): 'feed', 'fetch', 'save', 'load', 'recurrent', 'rnn_memory_helper_grad', 'conditional_block', 'while', 'send', 'recv', 'listen_and_serv', 'parallel_do', 'save_combine', - 'load_combine' + 'load_combine', 'switch' } if type not in no_kernel_op_set: self.desc.infer_var_type(self.block.desc) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index b33b76f19e066..26f8d5c90afe5 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1116,45 +1116,6 @@ def complete(self): attrs={'sub_block': inside_block}) -class SwitchScopeGuard(object): - """ - 1. only switch.case can be called inside a switch.scope - 2. switch.case can only be called inside a switch.scope - """ - - def __init__(self, op): - if not isinstance(op, Switch): - raise TypeError("op should be switch") - self.switch_op = op - - def __enter__(self): - """ - set flag that now is inside switch.block {} - :return: - """ - self.switch_op.inside_scope = True - - def __exit__(self, exc_type, exc_val, exc_tb): - self.switch_op.inside_scope = False - if exc_type is not None: - return False # re-raise exception - - cond_num = len(self.switch_op.conditions) - case_block_num = len(self.switch_op.case_blocks) - - tmp = case_block_num - cond_num - if tmp != 0 and tmp != 1: - raise ValueError("case_num=%d, cond_num=%d not match", - case_block_num, cond_num) - - self.switch_op.helper.append_op( - type='switch', - inputs={'X': self.switch_op.conditions}, - attrs={'level': 0}) - - return True - - class CaseBlockGuard(BlockGuard): def __init__(self, op, condition): if not isinstance(op, Switch): @@ -1201,11 +1162,6 @@ def __init__(self, name=None): self.conditions = [] self.case_blocks = [] - def scope(self): - """the scope for this switch op, works like `{}` - """ - return SwitchScopeGuard(self) - def case(self, condition): """create a new block for this condition """ @@ -1229,6 +1185,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: return False # re-raise exception + current_block = self.helper.main_program.current_block() + cond_num = len(self.conditions) case_block_num = len(self.case_blocks) @@ -1237,9 +1195,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): raise ValueError("case_num=%d, cond_num=%d not match", case_block_num, cond_num) + step_scope = current_block.create_var( + type=core.VarDesc.VarType.STEP_SCOPES) self.helper.append_op( type='switch', - inputs={'X': self.conditions}, + inputs={'X': self.conditions, + 'ScopeIn': [step_scope]}, + outputs={'Scope': [step_scope]}, attrs={'case_blocks': self.case_blocks}) return True From 942bdcb1fab4de15fff85b25b15581ebaec172a5 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 16:04:34 +0800 Subject: [PATCH 14/32] complete test --- python/paddle/v2/fluid/tests/test_switch.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/python/paddle/v2/fluid/tests/test_switch.py b/python/paddle/v2/fluid/tests/test_switch.py index 023e7cef65ef5..cc1a486eedf73 100644 --- a/python/paddle/v2/fluid/tests/test_switch.py +++ b/python/paddle/v2/fluid/tests/test_switch.py @@ -23,7 +23,6 @@ class TestSwitch(unittest.TestCase): def check_switch(self, value): - x = layers.fill_constant(shape=[1], dtype='float32', value=value) zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) @@ -39,25 +38,27 @@ def check_switch(self, value): with layers.Switch() as switch: with switch.case(cond1): - layers.assign(result, zero_var) + layers.assign(zero_var, result) with switch.case(cond2): - layers.assign(result, one_var) + layers.assign(one_var, result) with switch.case(cond3): - layers.assign(result, two_var) + layers.assign(two_var, result) cpu = core.CPUPlace() exe = Executor(cpu) exe.run(default_startup_program()) - out = exe.run(feed={}, fetch_list=[result])[0] + out = exe.run(feed={}, fetch_list=[result])[0][0] return out def test_switch(self): - main_program = framework.Program() - startup_program = framework.Program() - with framework.program_guard(main_program, startup_program): - result = self.check_switch(0.0) - self.assertEqual(result, 1) + test_data = {(-0.1, 0), (0.1, 1), (1.1, 2)} + for x, expected_result in test_data: + main_program = framework.Program() + startup_program = framework.Program() + with framework.program_guard(main_program, startup_program): + result = self.check_switch(x) + self.assertEqual(result, expected_result) if __name__ == '__main__': From 9d1385bb310df74e9b0706ff3f43e49f7d01a26d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 16:32:43 +0800 Subject: [PATCH 15/32] optimize test --- paddle/operators/switch_op.cc | 6 +++++- python/paddle/v2/fluid/tests/test_switch.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index 6a9e4d8f4e176..afd46b1bc3326 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -131,7 +131,11 @@ class SwitchOp : public SwitchOpBase { cond_num, case_num); int match_case_id = GetMatchCaseIndex(xs, blocks); if (match_case_id >= 0) { - VLOG(3) << "match case " << match_case_id; + if (match_case_id == case_num - 1) { + VLOG(3) << "match default case " << match_case_id; + } else { + VLOG(3) << "match case " << match_case_id; + } auto block = blocks[match_case_id]; auto *scope_var = scope.FindVar(Output("Scope")); diff --git a/python/paddle/v2/fluid/tests/test_switch.py b/python/paddle/v2/fluid/tests/test_switch.py index cc1a486eedf73..d2f5f72d734b0 100644 --- a/python/paddle/v2/fluid/tests/test_switch.py +++ b/python/paddle/v2/fluid/tests/test_switch.py @@ -28,6 +28,7 @@ def check_switch(self, value): zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) one_var = layers.fill_constant(shape=[1], dtype='float32', value=1.0) two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0) + three_var = layers.fill_constant(shape=[1], dtype='float32', value=3.0) cond1 = layers.less_than(x, zero_var) cond2 = layers.less_than(x, one_var) @@ -43,6 +44,8 @@ def check_switch(self, value): layers.assign(one_var, result) with switch.case(cond3): layers.assign(two_var, result) + with switch.default(): + layers.assign(three_var, result) cpu = core.CPUPlace() exe = Executor(cpu) @@ -52,7 +55,7 @@ def check_switch(self, value): return out def test_switch(self): - test_data = {(-0.1, 0), (0.1, 1), (1.1, 2)} + test_data = {(-0.1, 0), (0.1, 1), (1.1, 2), (2.1, 3)} for x, expected_result in test_data: main_program = framework.Program() startup_program = framework.Program() From 410db57cd45378bd7dbc27d5de99e4db61df16ee Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 16:37:22 +0800 Subject: [PATCH 16/32] optimize test --- python/paddle/v2/fluid/tests/test_switch.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/python/paddle/v2/fluid/tests/test_switch.py b/python/paddle/v2/fluid/tests/test_switch.py index d2f5f72d734b0..52ebf773ec722 100644 --- a/python/paddle/v2/fluid/tests/test_switch.py +++ b/python/paddle/v2/fluid/tests/test_switch.py @@ -30,19 +30,15 @@ def check_switch(self, value): two_var = layers.fill_constant(shape=[1], dtype='float32', value=2.0) three_var = layers.fill_constant(shape=[1], dtype='float32', value=3.0) - cond1 = layers.less_than(x, zero_var) - cond2 = layers.less_than(x, one_var) - cond3 = layers.less_than(x, two_var) - result = layers.create_global_var( shape=[1], value=-1.0, dtype='float32', persistable=True) with layers.Switch() as switch: - with switch.case(cond1): + with switch.case(layers.less_than(x, zero_var)): layers.assign(zero_var, result) - with switch.case(cond2): + with switch.case(layers.less_than(x, one_var)): layers.assign(one_var, result) - with switch.case(cond3): + with switch.case(layers.less_than(x, two_var)): layers.assign(two_var, result) with switch.default(): layers.assign(three_var, result) From 511cb49f100cae39b3544857815d7223b7bf8fe6 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 16:44:48 +0800 Subject: [PATCH 17/32] rm backward part --- paddle/operators/switch_op.cc | 58 ----------------------------------- 1 file changed, 58 deletions(-) diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index afd46b1bc3326..a82cc1b7e485e 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -154,67 +154,9 @@ class SwitchOp : public SwitchOpBase { } }; -class SwitchGradOp : public SwitchOpBase { - public: - SwitchGradOp(const std::string &type, - const framework::VariableNameMap &inputs, - const framework::VariableNameMap &outputs, - const framework::AttributeMap &attrs) - : SwitchOpBase(type, inputs, outputs, attrs) {} - void Run(const framework::Scope &scope, - const platform::Place &dev_place) const override { - auto xs = this->InputTensors(scope); - auto blocks = Attr>("case_blocks"); - - int match_case_id = GetMatchCaseIndex(xs, blocks); - if (match_case_id >= 0) { - auto *scope_var = scope.FindVar(Input("Scope")); - PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); - auto &scopes = scope_var->Get>(); - framework::Scope &cur_scope = *scopes[0]; - - framework::Executor exec(dev_place); - exec.Run(*blocks[match_case_id]->Program(), &cur_scope, - blocks[match_case_id]->ID(), false); - - // AssignLocalGradientToGlobal(dev_place, cur_scope, Inputs("X"), - // Outputs(framework::GradVarName("X"))); - } - } - - private: - void AssignLocalGradientToGlobal( - const platform::Place &place, const framework::Scope &cur_scope, - const std::vector &p_names, - const std::vector &pg_names) const { - for (size_t i = 0; i < p_names.size(); ++i) { - auto out_grad_name = pg_names[i]; - auto in_grad_name = framework::GradVarName(p_names[i]); - auto *in_var = cur_scope.FindVar(in_grad_name); - if (in_var == nullptr) { - continue; - } - auto new_in_grad_name = cur_scope.Rename(in_grad_name); - auto assign = framework::OpRegistry::CreateOp( - "assign", {{"X", {new_in_grad_name}}}, {{"Out", {out_grad_name}}}, - framework::AttributeMap{}); - assign->Run(cur_scope, place); - cur_scope.Rename(new_in_grad_name, in_grad_name); - } - } -}; - -class SwitchGradInferShape : public framework::InferShapeBase { - public: - void operator()(framework::InferShapeContext *context) const override { - PADDLE_ENFORCE(context->HasInputs("X")); - } -}; - } // namespace operators } // namespace paddle namespace ops = paddle::operators; REGISTER_OPERATOR(switch, ops::SwitchOp, ops::SwitchOpProtoMaker, ops::SwitchOpGradMaker); -REGISTER_OPERATOR(switch_grad, ops::SwitchGradOp, ops::SwitchGradInferShape); From 2af2a18b30e5bf660bedc4bb3a7b8ce49e5fe31d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 16:54:43 +0800 Subject: [PATCH 18/32] clear grad op --- paddle/operators/switch_op.cc | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc index a82cc1b7e485e..9965f869e1fe2 100644 --- a/paddle/operators/switch_op.cc +++ b/paddle/operators/switch_op.cc @@ -43,21 +43,6 @@ Run one sub block according to condition list, } }; -class SwitchOpGradMaker : public framework::SingleGradOpDescMaker { - public: - using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; - - protected: - std::unique_ptr Apply() const override { - auto grad_op = new framework::OpDesc(); - grad_op->SetType("switch_grad"); - grad_op->SetInput("X", Input("X")); - grad_op->SetInput("Scope", Output("Scope")); - grad_op->SetBlocksAttr("case_blocks", this->grad_block_); - return std::unique_ptr(grad_op); - } -}; - class SwitchOpBase : public framework::OperatorBase { public: SwitchOpBase(const std::string &type, @@ -158,5 +143,4 @@ class SwitchOp : public SwitchOpBase { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OPERATOR(switch, ops::SwitchOp, ops::SwitchOpProtoMaker, - ops::SwitchOpGradMaker); +REGISTER_OPERATOR(switch, ops::SwitchOp, ops::SwitchOpProtoMaker); From 1e6f229cb47016b2eb71cb595c1b2be156541a1a Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 17:24:11 +0800 Subject: [PATCH 19/32] polynomial_decay use switch op --- python/paddle/v2/fluid/learning_rate_decay.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/fluid/learning_rate_decay.py b/python/paddle/v2/fluid/learning_rate_decay.py index 0c8ea3873f663..30eaf393470a0 100644 --- a/python/paddle/v2/fluid/learning_rate_decay.py +++ b/python/paddle/v2/fluid/learning_rate_decay.py @@ -131,7 +131,7 @@ def polynomial_decay(learning_rate, end_learning_rate=0.0001, power=1.0, cycle=False): - """Applies inverse time decay to the initial learning rate. + """Applies polynomial decay to the initial learning rate. ```python if cycle: @@ -162,10 +162,9 @@ def polynomial_decay(learning_rate, zero_var = layers.fill_constant(shape=[1], dtype='float32', value=0.0) one_var = layers.fill_constant(shape=[1], dtype='float32', value=1.0) - cond = layers.equal(x=global_step, y=zero_var) - true_cond = layers.ConditionalBlock([cond]) - with true_cond.block(): - layers.assign(input=one_var, output=div_res) + with layers.Switch() as switch: + with switch.case(layers.equal(x=global_step, y=zero_var)): + layers.assign(input=one_var, output=div_res) decay_steps = decay_steps * div_res else: decay_steps_var = layers.fill_constant( From 33079d9b183d4ebc2e940f83ecfcfe9d3e7c233c Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 5 Feb 2018 17:30:05 +0800 Subject: [PATCH 20/32] revert conditional_block_op and reshape_op --- paddle/operators/conditional_block_op.cc | 22 ---------------------- paddle/operators/reshape_op.cc | 6 ++---- paddle/operators/reshape_op.cu | 6 ++---- 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index d86a4825b7318..3cae61a438431 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include - #include "paddle/framework/executor.h" #include "paddle/framework/op_registry.h" @@ -42,19 +41,6 @@ class ConditionalOp : public framework::OperatorBase { }); return retv; } - - bool IsScalarFalse( - const std::vector &ips) const { - if (ips.size() == 1UL && ips[0]->IsInitialized()) { - if (ips[0]->type().hash_code() == typeid(bool).hash_code() && - ips[0]->numel() == 1) { - if (ips[0]->data()[0] == false) { - return true; - } - } - } - return false; - } }; class ConditionalBlockOp : public ConditionalOp { @@ -71,10 +57,6 @@ class ConditionalBlockOp : public ConditionalOp { xs.begin(), xs.end(), [](const framework::LoDTensor *t) { return t->numel() != 0; }); - if (IsScalarFalse(xs)) { - need_run = false; - } - if (need_run) { auto *scope_var = scope.FindVar(Output("Scope")); PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); @@ -128,10 +110,6 @@ class ConditionalBlockGradOp : public ConditionalOp { xs.begin(), xs.end(), [](const framework::LoDTensor *t) { return t->numel() != 0; }); - if (IsScalarFalse(xs)) { - need_run = false; - } - if (need_run) { auto *scope_var = scope.FindVar(Input("Scope")); PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); diff --git a/paddle/operators/reshape_op.cc b/paddle/operators/reshape_op.cc index 710d5997bd25b..b9743a5df1092 100644 --- a/paddle/operators/reshape_op.cc +++ b/paddle/operators/reshape_op.cc @@ -125,8 +125,6 @@ namespace ops = paddle::operators; REGISTER_OP(reshape, ops::ReshapeOp, ops::ReshapeOpMaker, reshape_grad, ops::ReshapeGradOp); REGISTER_OP_CPU_KERNEL(reshape, - ops::ReshapeKernel, - ops::ReshapeKernel); + ops::ReshapeKernel); REGISTER_OP_CPU_KERNEL( - reshape_grad, ops::ReshapeGradKernel, - ops::ReshapeGradKernel); + reshape_grad, ops::ReshapeGradKernel); diff --git a/paddle/operators/reshape_op.cu b/paddle/operators/reshape_op.cu index dc2583864155c..f487e43b99d5b 100644 --- a/paddle/operators/reshape_op.cu +++ b/paddle/operators/reshape_op.cu @@ -16,9 +16,7 @@ limitations under the License. */ REGISTER_OP_CUDA_KERNEL( reshape, - paddle::operators::ReshapeKernel, - paddle::operators::ReshapeKernel); + paddle::operators::ReshapeKernel); REGISTER_OP_CUDA_KERNEL( reshape_grad, - paddle::operators::ReshapeGradKernel, - paddle::operators::ReshapeGradKernel); + paddle::operators::ReshapeGradKernel); From 04e8a236ecd2736b1ab680e384436f4781682814 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 09:53:35 +0800 Subject: [PATCH 21/32] add piecewise_decay and test --- python/paddle/v2/fluid/learning_rate_decay.py | 45 +++++++++++++++++++ .../fluid/tests/test_learning_rate_decay.py | 14 ++++++ 2 files changed, 59 insertions(+) diff --git a/python/paddle/v2/fluid/learning_rate_decay.py b/python/paddle/v2/fluid/learning_rate_decay.py index 30eaf393470a0..936a5a0aea1f9 100644 --- a/python/paddle/v2/fluid/learning_rate_decay.py +++ b/python/paddle/v2/fluid/learning_rate_decay.py @@ -173,3 +173,48 @@ def polynomial_decay(learning_rate, return (learning_rate - end_learning_rate) * \ ((1 - global_step / decay_steps) ** power) + end_learning_rate + + +def piecewise_decay(global_step, boundaries, values): + """Applies piecewise decay to the initial learning rate. + + ```python + boundaries = [10000, 20000] + values = [1.0, 0.5, 0.1] + + if step < 10000: + learning_rate = 1.0 + elif step >= 10000 and step < 20000: + learning_rate = 0.5 + else: + learning_rate = 0.1 + ``` + """ + + if len(values) - len(boundaries) != 1: + raise ValueError("len(values) - len(boundaries) should be 1") + + if not isinstance(global_step, Variable): + raise ValueError("global_step is required for inverse_time_decay.") + + lr = layers.create_global_var( + shape=[1], + value=0.0, + dtype='float32', + persistable=True, + name="learning_rate") + + with layers.Switch() as switch: + for i in range(len(boundaries)): + boundary_val = layers.fill_constant( + shape=[1], dtype='float32', value=float(boundaries[i])) + value_var = layers.fill_constant( + shape=[1], dtype='float32', value=float(values[i])) + with switch.case(layers.less_than(global_step, boundary_val)): + layers.assign(lr, value_var) + last_value_var = layers.fill_constant( + shape=[1], dtype='float32', value=float(values[len(values) - 1])) + with switch.default(): + layers.assign(lr, last_value_var) + + return lr diff --git a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py index 32df92dc78e5a..5e22fb70c8f88 100644 --- a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py +++ b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py @@ -21,6 +21,7 @@ import paddle.v2.fluid as fluid import paddle.v2.fluid.layers as layers import paddle.v2.fluid.learning_rate_decay as lr_decay +import paddle.v2.fluid.debuger as debugger def exponential_decay(learning_rate, @@ -73,6 +74,14 @@ def polynomial_decay(learning_rate, ((1 - float(global_step) / float(decay_steps)) ** power) + end_learning_rate +def piecewise_decay(global_step, boundaries, values): + assert len(boundaries) + 1 == len(values) + for i in range(len(boundaries)): + if global_step < boundaries[i]: + return values[i] + return values[len(values) - 1] + + class TestLearningRateDecay(unittest.TestCase): def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): global_step = layers.create_global_var( @@ -90,6 +99,7 @@ def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): feed=[], fetch_list=[global_step, decayed_lr]) python_decayed_lr = python_decay_fn(global_step=step, **kwargs) + print(str(python_decayed_lr) + ":" + str(lr_val[0])) self.assertAlmostEqual(python_decayed_lr, lr_val[0]) def test_decay(self): @@ -123,6 +133,10 @@ def test_decay(self): "decay_steps": 5, "cycle": False }), + (piecewise_decay, lr_decay.piecewise_decay, { + "boundaries": [3, 6, 9], + "values": [0.1, 0.2, 0.3, 0.4] + }), ] for py_decay_fn, fluid_decay_fn, kwargs in decay_fns: From 7d86a0c8366a0f89c7aaf2f3b6499dc2e0d28352 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 10:14:17 +0800 Subject: [PATCH 22/32] fix piecewise_decay --- python/paddle/v2/fluid/learning_rate_decay.py | 4 ++-- python/paddle/v2/fluid/tests/test_learning_rate_decay.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/paddle/v2/fluid/learning_rate_decay.py b/python/paddle/v2/fluid/learning_rate_decay.py index 936a5a0aea1f9..4648c0a6a623a 100644 --- a/python/paddle/v2/fluid/learning_rate_decay.py +++ b/python/paddle/v2/fluid/learning_rate_decay.py @@ -211,10 +211,10 @@ def piecewise_decay(global_step, boundaries, values): value_var = layers.fill_constant( shape=[1], dtype='float32', value=float(values[i])) with switch.case(layers.less_than(global_step, boundary_val)): - layers.assign(lr, value_var) + layers.assign(value_var, lr) last_value_var = layers.fill_constant( shape=[1], dtype='float32', value=float(values[len(values) - 1])) with switch.default(): - layers.assign(lr, last_value_var) + layers.assign(last_value_var, lr) return lr diff --git a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py index 5e22fb70c8f88..1d6bab3d6c44b 100644 --- a/python/paddle/v2/fluid/tests/test_learning_rate_decay.py +++ b/python/paddle/v2/fluid/tests/test_learning_rate_decay.py @@ -21,7 +21,6 @@ import paddle.v2.fluid as fluid import paddle.v2.fluid.layers as layers import paddle.v2.fluid.learning_rate_decay as lr_decay -import paddle.v2.fluid.debuger as debugger def exponential_decay(learning_rate, @@ -99,7 +98,6 @@ def check_decay(self, python_decay_fn, fluid_decay_fn, kwargs): feed=[], fetch_list=[global_step, decayed_lr]) python_decayed_lr = python_decay_fn(global_step=step, **kwargs) - print(str(python_decayed_lr) + ":" + str(lr_val[0])) self.assertAlmostEqual(python_decayed_lr, lr_val[0]) def test_decay(self): From 835dc2f3592fb6535ba1186ad2f284e306aa4d47 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 13:52:49 +0800 Subject: [PATCH 23/32] try to use condition op for switch --- paddle/operators/conditional_block_op.cc | 21 +++++++ python/paddle/v2/fluid/layers/control_flow.py | 56 ++++++++++++++++++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index 3cae61a438431..9e4569dad9b5e 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -41,6 +41,19 @@ class ConditionalOp : public framework::OperatorBase { }); return retv; } + + bool IsScalarFalse( + const std::vector &ips) const { + if (ips.size() == 1UL && ips[0]->IsInitialized()) { + if (ips[0]->type().hash_code() == typeid(bool).hash_code() && + ips[0]->numel() == 1) { + if (ips[0]->data()[0] == false) { + return true; + } + } + } + return false; + } }; class ConditionalBlockOp : public ConditionalOp { @@ -57,6 +70,10 @@ class ConditionalBlockOp : public ConditionalOp { xs.begin(), xs.end(), [](const framework::LoDTensor *t) { return t->numel() != 0; }); + if (IsScalarFalse(xs)) { + need_run = false; + } + if (need_run) { auto *scope_var = scope.FindVar(Output("Scope")); PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); @@ -110,6 +127,10 @@ class ConditionalBlockGradOp : public ConditionalOp { xs.begin(), xs.end(), [](const framework::LoDTensor *t) { return t->numel() != 0; }); + if (IsScalarFalse(xs)) { + need_run = false; + } + if (need_run) { auto *scope_var = scope.FindVar(Input("Scope")); PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index 26f8d5c90afe5..c878b8141ccc4 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1155,6 +1155,58 @@ def __exit__(self, exc_type, exc_val, exc_tb): exc_tb) +# class Switch(object): +# def __init__(self, name=None): +# self.helper = LayerHelper('switch', name=name) +# self.inside_scope = False +# self.conditions = [] +# self.case_blocks = [] +# +# def case(self, condition): +# """create a new block for this condition +# """ +# return CaseBlockGuard(self, condition) +# +# def default(self): +# """create a default case for this switch +# """ +# return DefaultCaseBlockGuard(self) +# +# def __enter__(self): +# """ +# set flag that now is inside switch.block {} +# :return: +# """ +# self.inside_scope = True +# return self +# +# def __exit__(self, exc_type, exc_val, exc_tb): +# self.inside_scope = False +# if exc_type is not None: +# return False # re-raise exception +# +# current_block = self.helper.main_program.current_block() +# +# cond_num = len(self.conditions) +# case_block_num = len(self.case_blocks) +# +# tmp = case_block_num - cond_num +# if tmp != 0 and tmp != 1: +# raise ValueError("case_num=%d, cond_num=%d not match", +# case_block_num, cond_num) +# +# step_scope = current_block.create_var( +# type=core.VarDesc.VarType.STEP_SCOPES) +# self.helper.append_op( +# type='switch', +# inputs={'X': self.conditions, +# 'ScopeIn': [step_scope]}, +# outputs={'Scope': [step_scope]}, +# attrs={'case_blocks': self.case_blocks}) +# +# return True + + class Switch(object): def __init__(self, name=None): self.helper = LayerHelper('switch', name=name) @@ -1165,7 +1217,9 @@ def __init__(self, name=None): def case(self, condition): """create a new block for this condition """ - return CaseBlockGuard(self, condition) + self.conditions.append(condition) + cond_block = ConditionalBlock([condition]) + return ConditionalBlockGuard(cond_block) def default(self): """create a default case for this switch From d3e148f73f2d44b30636cbaf449669bbcb7de119 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 16:32:55 +0800 Subject: [PATCH 24/32] can work --- python/paddle/v2/fluid/layers/control_flow.py | 52 +++++++++++-------- python/paddle/v2/fluid/layers/ops.py | 4 ++ 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index c878b8141ccc4..957c5979a015f 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -18,6 +18,7 @@ from .. import core from ..framework import Program, Variable, Operator from ..layer_helper import LayerHelper, unique_name +from ops import logical_and, logical_not, logical_or __all__ = [ 'split_lod_tensor', @@ -1212,19 +1213,45 @@ def __init__(self, name=None): self.helper = LayerHelper('switch', name=name) self.inside_scope = False self.conditions = [] - self.case_blocks = [] + self.pre_conditions = [] def case(self, condition): """create a new block for this condition """ + if not self.inside_scope: + raise ValueError("case should be called inside with") + + if len(self.conditions) == 0: + cond_block = ConditionalBlock([condition]) + not_cond = logical_not(x=condition) + self.pre_conditions.append(not_cond) + else: + case_num = len(self.conditions) + pre_cond_num = len(self.pre_conditions) + if case_num != pre_cond_num: + raise ValueError("case_num=" + str(case_num) + " pre_cond_num=" + + str(pre_cond_num)) + assert case_num == len(self.pre_conditions) + pre_not_cond = self.pre_conditions[case_num - 1] + not_cond = logical_and(x=pre_not_cond, y=logical_not(x=condition)) + self.pre_conditions.append(not_cond) + cond_block = ConditionalBlock( + [logical_and( + x=pre_not_cond, y=condition)]) self.conditions.append(condition) - cond_block = ConditionalBlock([condition]) + return ConditionalBlockGuard(cond_block) def default(self): """create a default case for this switch """ - return DefaultCaseBlockGuard(self) + if not self.inside_scope: + raise ValueError("default should be called inside with") + cond_num = len(self.conditions) + if cond_num == 0: + raise ValueError("there should be at least one condition") + cond_block = ConditionalBlock([self.pre_conditions[cond_num - 1]]) + return ConditionalBlockGuard(cond_block) def __enter__(self): """ @@ -1239,25 +1266,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: return False # re-raise exception - current_block = self.helper.main_program.current_block() - - cond_num = len(self.conditions) - case_block_num = len(self.case_blocks) - - tmp = case_block_num - cond_num - if tmp != 0 and tmp != 1: - raise ValueError("case_num=%d, cond_num=%d not match", - case_block_num, cond_num) - - step_scope = current_block.create_var( - type=core.VarDesc.VarType.STEP_SCOPES) - self.helper.append_op( - type='switch', - inputs={'X': self.conditions, - 'ScopeIn': [step_scope]}, - outputs={'Scope': [step_scope]}, - attrs={'case_blocks': self.case_blocks}) - return True diff --git a/python/paddle/v2/fluid/layers/ops.py b/python/paddle/v2/fluid/layers/ops.py index c701e79ad266d..38dea2892fc18 100644 --- a/python/paddle/v2/fluid/layers/ops.py +++ b/python/paddle/v2/fluid/layers/ops.py @@ -61,6 +61,10 @@ 'clip_by_norm', 'softmax', 'sequence_softmax', + 'logical_and', + 'logical_or', + 'logical_xor', + 'logical_not', ] + __activations__ for _OP in set(__all__): From 60a45f8115613de78f33f00f173ab684101ccfe2 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 16:36:40 +0800 Subject: [PATCH 25/32] clean old code --- python/paddle/v2/fluid/layers/control_flow.py | 91 ------------------- 1 file changed, 91 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index 957c5979a015f..ae9aa4ae05c30 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1117,97 +1117,6 @@ def complete(self): attrs={'sub_block': inside_block}) -class CaseBlockGuard(BlockGuard): - def __init__(self, op, condition): - if not isinstance(op, Switch): - raise TypeError("op should be switch") - if not op.inside_scope: - raise ValueError( - "switch.case can only be called inside switch.block") - self.switch_op = op - super(CaseBlockGuard, self).__init__(self.switch_op.helper.main_program) - self.switch_op.conditions.append(condition) - - def __enter__(self): - return super(CaseBlockGuard, self).__enter__() - - def __exit__(self, exc_type, exc_val, exc_tb): - self.switch_op.case_blocks.append(self.main_program.current_block()) - return super(CaseBlockGuard, self).__exit__(exc_type, exc_val, exc_tb) - - -class DefaultCaseBlockGuard(BlockGuard): - def __init__(self, op): - if not isinstance(op, Switch): - raise TypeError("op should be switch") - if not op.inside_scope: - raise ValueError( - "switch.case can only be called inside switch.block") - self.switch_op = op - super(DefaultCaseBlockGuard, - self).__init__(self.switch_op.helper.main_program) - - def __enter__(self): - return super(DefaultCaseBlockGuard, self).__enter__() - - def __exit__(self, exc_type, exc_val, exc_tb): - self.switch_op.case_blocks.append(self.main_program.current_block()) - return super(DefaultCaseBlockGuard, self).__exit__(exc_type, exc_val, - exc_tb) - - -# class Switch(object): -# def __init__(self, name=None): -# self.helper = LayerHelper('switch', name=name) -# self.inside_scope = False -# self.conditions = [] -# self.case_blocks = [] -# -# def case(self, condition): -# """create a new block for this condition -# """ -# return CaseBlockGuard(self, condition) -# -# def default(self): -# """create a default case for this switch -# """ -# return DefaultCaseBlockGuard(self) -# -# def __enter__(self): -# """ -# set flag that now is inside switch.block {} -# :return: -# """ -# self.inside_scope = True -# return self -# -# def __exit__(self, exc_type, exc_val, exc_tb): -# self.inside_scope = False -# if exc_type is not None: -# return False # re-raise exception -# -# current_block = self.helper.main_program.current_block() -# -# cond_num = len(self.conditions) -# case_block_num = len(self.case_blocks) -# -# tmp = case_block_num - cond_num -# if tmp != 0 and tmp != 1: -# raise ValueError("case_num=%d, cond_num=%d not match", -# case_block_num, cond_num) -# -# step_scope = current_block.create_var( -# type=core.VarDesc.VarType.STEP_SCOPES) -# self.helper.append_op( -# type='switch', -# inputs={'X': self.conditions, -# 'ScopeIn': [step_scope]}, -# outputs={'Scope': [step_scope]}, -# attrs={'case_blocks': self.case_blocks}) -# -# return True - - class Switch(object): def __init__(self, name=None): self.helper = LayerHelper('switch', name=name) From 0217065c21c55fc0909c7e085dab3e4f93f19bdb Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 16:52:56 +0800 Subject: [PATCH 26/32] revert --- paddle/framework/framework.proto | 2 -- paddle/framework/op_desc.cc | 41 +++-------------------------- paddle/framework/op_desc.h | 5 ---- paddle/framework/type_defs.h | 3 +-- paddle/framework/var_desc.cc | 1 - paddle/pybind/protobuf.cc | 1 - python/paddle/v2/fluid/framework.py | 15 +---------- 7 files changed, 5 insertions(+), 63 deletions(-) diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index 466f499614215..f65ccae6e6a4d 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -27,7 +27,6 @@ enum AttrType { BOOLEANS = 7; BLOCK = 8; LONG = 9; - BLOCKS = 10; } // OpDesc describes an instance of a C++ framework::OperatorBase @@ -47,7 +46,6 @@ message OpDesc { repeated bool bools = 11; optional int32 block_idx = 12; optional int64 l = 13; - repeated int32 block_idxs = 14; }; message Var { diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 0cbeab50cb9bb..ad361852ec9f2 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -120,18 +120,11 @@ OpDesc::OpDesc(const proto::OpDesc &desc, ProgramDesc *prog, BlockDesc *block) // restore attrs_ for (const proto::OpDesc::Attr &attr : desc_.attrs()) { std::string attr_name = attr.name(); - if (attr.type() == proto::AttrType::BLOCK) { + if (attr.type() != proto::AttrType::BLOCK) { + attrs_[attr_name] = GetAttrValue(attr); + } else { auto bid = attr.block_idx(); attrs_[attr_name] = prog->MutableBlock(bid); - } else if (attr.type() == proto::AttrType::BLOCKS) { - auto block_idxs = attr.block_idxs(); - std::vector ret; - for (auto idx : block_idxs) { - ret.emplace_back(prog->MutableBlock(idx)); - } - attrs_[attr_name] = ret; - } else { - attrs_[attr_name] = GetAttrValue(attr); } } this->block_ = block; @@ -209,12 +202,6 @@ void OpDesc::SetBlockAttr(const std::string &name, BlockDesc &block) { need_update_ = true; } -void OpDesc::SetBlocksAttr(const std::string &name, - const std::vector &blocks) { - this->attrs_[name] = blocks; - need_update_ = true; -} - void OpDesc::SetAttrMap( const std::unordered_map &attr_map) { attrs_ = attr_map; @@ -233,20 +220,6 @@ int OpDesc::GetBlockAttr(const std::string &name) const { return boost::get(it->second)->ID(); } -std::vector OpDesc::GetBlocksAttr(const std::string &name) const { - auto it = attrs_.find(name); - PADDLE_ENFORCE(it != attrs_.end(), "Attribute %s is not found", name); - auto blocks = boost::get>(it->second); - std::vector retv; - std::transform(blocks.begin(), blocks.end(), retv.begin(), - [](const BlockDesc *block_desc) -> int { - PADDLE_ENFORCE(block_desc != nullptr, - "block desc should not be null"); - return block_desc->ID(); - }); - return retv; -} - const std::unordered_map &OpDesc::GetAttrMap() const { return attrs_; } @@ -307,14 +280,6 @@ struct SetAttrDescVisitor : public boost::static_visitor { } void operator()(BlockDesc *desc) const { attr_->set_block_idx(desc->ID()); } void operator()(int64_t v) const { attr_->set_l(v); } - void operator()(const std::vector &v) const { - auto *repeated_field = attr_->mutable_block_idxs(); - repeated_field->Clear(); - repeated_field->Reserve(v.size()); - for (auto elem : v) { - *repeated_field->Add() = elem->ID(); - } - } void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } }; diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 8589bdc8e67d6..13695cff59f0b 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -75,15 +75,10 @@ class OpDesc { void SetBlockAttr(const std::string &name, BlockDesc &block); - void SetBlocksAttr(const std::string &name, - const std::vector &blocks); - Attribute GetAttr(const std::string &name) const; int GetBlockAttr(const std::string &name) const; - std::vector GetBlocksAttr(const std::string &name) const; - void Rename(const std::string &old_name, const std::string &new_name); void RenameOutput(const std::string &old_name, const std::string &new_name); diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h index 4e1e70cb494f7..1eedbbc419ab6 100644 --- a/paddle/framework/type_defs.h +++ b/paddle/framework/type_defs.h @@ -35,8 +35,7 @@ using VariableNameMap = std::map>; using Attribute = boost::variant, std::vector, std::vector, bool, - std::vector, BlockDesc*, int64_t, - std::vector>; + std::vector, BlockDesc*, int64_t>; using AttributeMap = std::unordered_map; diff --git a/paddle/framework/var_desc.cc b/paddle/framework/var_desc.cc index 6b425e36de723..6d83e2e41126d 100644 --- a/paddle/framework/var_desc.cc +++ b/paddle/framework/var_desc.cc @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/framework/var_desc.h" -#include "paddle/framework/block_desc.h" #include "paddle/platform/enforce.h" namespace paddle { diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc index a835c2140f1b4..0f1953abe0864 100644 --- a/paddle/pybind/protobuf.cc +++ b/paddle/pybind/protobuf.cc @@ -281,7 +281,6 @@ void BindOpDesc(py::module &m) { .def("set_attr", &OpDesc::SetAttr) .def("attr", &OpDesc::GetAttr) .def("set_block_attr", &OpDesc::SetBlockAttr) - .def("set_blocks_attr", &OpDesc::SetBlocksAttr) .def("set_serialized_attr", [](OpDesc &self, const std::string &name, const py::bytes &seriralized) { diff --git a/python/paddle/v2/fluid/framework.py b/python/paddle/v2/fluid/framework.py index 0551fec943419..a12427258e9d3 100644 --- a/python/paddle/v2/fluid/framework.py +++ b/python/paddle/v2/fluid/framework.py @@ -469,16 +469,6 @@ def find_name(var_list, name): arg.op = self self.desc.set_output(out_proto.name, out_arg_names) - def __is_block_list__(attr): - if not isinstance(attr, list): - return False - if len(attr) == 0: - return False - for item in attr: - if not isinstance(item, Block): - return False - return True - if attrs is not None: if not isinstance(attrs, dict): raise TypeError("'attrs' should be a dict.") @@ -492,9 +482,6 @@ def __is_block_list__(attr): isinstance(attrs[attr_name], core.ProgramDesc): self.desc.set_serialized_attr( attr_name, attrs[attr_name].serialize_to_string()) - elif __is_block_list__(attrs[attr_name]): - self.desc.set_blocks_attr( - attr_name, [item.desc for item in attrs[attr_name]]) else: self.desc.set_attr(attr_name, attrs[attr_name]) @@ -503,7 +490,7 @@ def __is_block_list__(attr): 'feed', 'fetch', 'save', 'load', 'recurrent', 'rnn_memory_helper_grad', 'conditional_block', 'while', 'send', 'recv', 'listen_and_serv', 'parallel_do', 'save_combine', - 'load_combine', 'switch' + 'load_combine' } if type not in no_kernel_op_set: self.desc.infer_var_type(self.block.desc) From 8f02fdf024b10fe95e6960b01e2fa13c96a20fd9 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 16:55:29 +0800 Subject: [PATCH 27/32] rm switch_op.cc --- paddle/operators/switch_op.cc | 146 ---------------------------------- 1 file changed, 146 deletions(-) delete mode 100644 paddle/operators/switch_op.cc diff --git a/paddle/operators/switch_op.cc b/paddle/operators/switch_op.cc deleted file mode 100644 index 9965f869e1fe2..0000000000000 --- a/paddle/operators/switch_op.cc +++ /dev/null @@ -1,146 +0,0 @@ -/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/framework/executor.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -class SwitchOpProtoMaker : public framework::OpProtoAndCheckerMaker { - public: - SwitchOpProtoMaker(OpProto *proto, OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The conditional variable of this operator.").AsDuplicable(); - AddInput("ScopeIn", - "(std::vector) The step scope of conditional block. To " - "unify the conditional block, rnn and while op, " - "the type of scope is std::vector"); - AddOutput("Scope", - "(std::vector) The step scope of conditional block. To " - "unify the conditional block, rnn and while op, " - "the type of scope is std::vector") - .AsIntermediate(); - AddAttr>( - "case_blocks", - "The step block of conditional " - "block operator, the length should be the same as X"); - AddComment(R"DOC(switch operator - -Run one sub block according to condition list, -)DOC"); - } -}; - -class SwitchOpBase : public framework::OperatorBase { - public: - SwitchOpBase(const std::string &type, - const framework::VariableNameMap &inputs, - const framework::VariableNameMap &outputs, - const framework::AttributeMap &attrs) - : framework::OperatorBase(type, inputs, outputs, attrs) {} - - protected: - std::vector InputTensors( - const framework::Scope &scope) const { - std::vector retv; - auto xs = Inputs("X"); - retv.resize(xs.size(), nullptr); - std::transform( - xs.begin(), xs.end(), retv.begin(), - [&scope](const std::string &var_name) -> const framework::LoDTensor * { - auto *var = scope.FindVar(var_name); - PADDLE_ENFORCE(var != nullptr, "Cannot find variable %s", var_name); - return &var->Get(); - }); - return retv; - } - - int GetMatchCaseIndex( - const std::vector &conditions, - const std::vector &case_blocks) const { - size_t cond_num = conditions.size(); - size_t case_num = case_blocks.size(); - - int match_cond_id = -1; - - for (size_t i = 0; i < conditions.size(); ++i) { - auto cond = conditions[i]; - PADDLE_ENFORCE(cond->IsInitialized() && - cond->dims() == framework::make_ddim({1}) && - cond->type().hash_code() == typeid(bool).hash_code(), - "cond should be a scalar bool tensor"); - if (cond->data()[0]) { - match_cond_id = static_cast(i); - break; - } - } - - if (match_cond_id >= 0) { - return match_cond_id; - } else if (cond_num + 1 == case_num) { - return case_num - 1; - } else { - return -1; - } - } -}; - -class SwitchOp : public SwitchOpBase { - public: - SwitchOp(const std::string &type, const framework::VariableNameMap &inputs, - const framework::VariableNameMap &outputs, - const framework::AttributeMap &attrs) - : SwitchOpBase(type, inputs, outputs, attrs) {} - - void Run(const framework::Scope &scope, - const platform::Place &dev_place) const override { - auto xs = InputTensors(scope); - auto blocks = Attr>("case_blocks"); - - size_t cond_num = xs.size(); - size_t case_num = blocks.size(); - PADDLE_ENFORCE(cond_num == case_num || cond_num + 1 == case_num, - "cond_num %d and case_num %d does not meet requirement", - cond_num, case_num); - int match_case_id = GetMatchCaseIndex(xs, blocks); - if (match_case_id >= 0) { - if (match_case_id == case_num - 1) { - VLOG(3) << "match default case " << match_case_id; - } else { - VLOG(3) << "match case " << match_case_id; - } - auto block = blocks[match_case_id]; - - auto *scope_var = scope.FindVar(Output("Scope")); - PADDLE_ENFORCE(scope_var != nullptr, "Must set scope"); - auto *scopes = scope_var->GetMutable>(); - scopes->resize(1); - scopes->front() = &scope.NewScope(); - auto &cur_scope = *scopes->front(); - - framework::Executor exec(dev_place); - - exec.Run(*block->Program(), &cur_scope, block->ID(), false); - } else { - VLOG(3) << "no case is matched, do nothing"; - } - } -}; - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OPERATOR(switch, ops::SwitchOp, ops::SwitchOpProtoMaker); From 06d87f90e45c483664a59422585485a2108bb49f Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 17:06:55 +0800 Subject: [PATCH 28/32] optimize code --- python/paddle/v2/fluid/layers/control_flow.py | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index ae9aa4ae05c30..b264f0a1357bb 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1121,8 +1121,7 @@ class Switch(object): def __init__(self, name=None): self.helper = LayerHelper('switch', name=name) self.inside_scope = False - self.conditions = [] - self.pre_conditions = [] + self.pre_not_conditions = [] def case(self, condition): """create a new block for this condition @@ -1130,36 +1129,30 @@ def case(self, condition): if not self.inside_scope: raise ValueError("case should be called inside with") - if len(self.conditions) == 0: + if len(self.pre_not_conditions) == 0: cond_block = ConditionalBlock([condition]) not_cond = logical_not(x=condition) - self.pre_conditions.append(not_cond) + self.pre_not_conditions.append(not_cond) else: - case_num = len(self.conditions) - pre_cond_num = len(self.pre_conditions) - if case_num != pre_cond_num: - raise ValueError("case_num=" + str(case_num) + " pre_cond_num=" - + str(pre_cond_num)) - assert case_num == len(self.pre_conditions) - pre_not_cond = self.pre_conditions[case_num - 1] - not_cond = logical_and(x=pre_not_cond, y=logical_not(x=condition)) - self.pre_conditions.append(not_cond) + pre_cond_num = len(self.pre_not_conditions) + pre_not_cond = self.pre_not_conditions[pre_cond_num - 1] + new_not_cond = logical_and( + x=pre_not_cond, y=logical_not(x=condition)) + self.pre_not_conditions.append(new_not_cond) cond_block = ConditionalBlock( [logical_and( x=pre_not_cond, y=condition)]) - self.conditions.append(condition) return ConditionalBlockGuard(cond_block) def default(self): """create a default case for this switch """ - if not self.inside_scope: - raise ValueError("default should be called inside with") - cond_num = len(self.conditions) - if cond_num == 0: + pre_cond_num = len(self.pre_not_conditions) + if pre_cond_num == 0: raise ValueError("there should be at least one condition") - cond_block = ConditionalBlock([self.pre_conditions[cond_num - 1]]) + cond_block = ConditionalBlock( + [self.pre_not_conditions[pre_cond_num - 1]]) return ConditionalBlockGuard(cond_block) def __enter__(self): From edacca8662f8b4fde64ca9e6fc38d66f130ce67a Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 18:59:16 +0800 Subject: [PATCH 29/32] add attr is_scalar_condition for condition_block_op --- paddle/operators/conditional_block_op.cc | 49 ++++++++++++------- python/paddle/v2/fluid/layers/control_flow.py | 16 ++++-- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index 9e4569dad9b5e..8f5bdfedee328 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -42,17 +42,19 @@ class ConditionalOp : public framework::OperatorBase { return retv; } - bool IsScalarFalse( + bool ScalarCondition( const std::vector &ips) const { - if (ips.size() == 1UL && ips[0]->IsInitialized()) { - if (ips[0]->type().hash_code() == typeid(bool).hash_code() && - ips[0]->numel() == 1) { - if (ips[0]->data()[0] == false) { - return true; - } - } + if (!(ips.size() == 1UL && ips[0]->IsInitialized())) { + PADDLE_THROW("should have one initialized input as condition"); + } + if (!(ips[0]->type().hash_code() == typeid(bool).hash_code() && + ips[0]->numel() == 1)) { + PADDLE_THROW( + "condition input's data type should be bool, " + "number should be 1, actual numel is %d", + ips[0]->numel()); } - return false; + return ips[0]->data()[0]; } }; @@ -66,12 +68,14 @@ class ConditionalBlockOp : public ConditionalOp { void Run(const framework::Scope &scope, const platform::Place &dev_place) const override { auto xs = InputTensors(scope); - bool need_run = std::all_of( - xs.begin(), xs.end(), - [](const framework::LoDTensor *t) { return t->numel() != 0; }); - if (IsScalarFalse(xs)) { - need_run = false; + bool need_run; + if (Attr("is_scalar_condition")) { + need_run = ScalarCondition(xs); + } else { + need_run = std::all_of( + xs.begin(), xs.end(), + [](const framework::LoDTensor *t) { return t->numel() != 0; }); } if (need_run) { @@ -105,6 +109,10 @@ class ConditionalBlockOpProtoMaker : public framework::OpProtoAndCheckerMaker { "scope is std::vector"); AddAttr( "sub_block", "The step block of conditional block operator"); + AddAttr("is_scalar_condition", + "the input X is used as scalar " + "condition") + .SetDefault(false); AddComment(R"DOC(Conditional block operator Run the sub-block if X is not empty. Params is the other inputs and Out is the @@ -123,12 +131,14 @@ class ConditionalBlockGradOp : public ConditionalOp { void Run(const framework::Scope &scope, const platform::Place &dev_place) const override { auto xs = this->InputTensors(scope); - bool need_run = std::all_of( - xs.begin(), xs.end(), - [](const framework::LoDTensor *t) { return t->numel() != 0; }); - if (IsScalarFalse(xs)) { - need_run = false; + bool need_run; + if (Attr("is_scalar_condition")) { + need_run = ScalarCondition(xs); + } else { + need_run = std::all_of( + xs.begin(), xs.end(), + [](const framework::LoDTensor *t) { return t->numel() != 0; }); } if (need_run) { @@ -203,6 +213,7 @@ class ConditionalBlockGradMaker : public framework::SingleGradOpDescMaker { grad_op->SetOutput(framework::GradVarName("Params"), InputGrad("Params", false)); grad_op->SetBlockAttr("sub_block", *this->grad_block_[0]); + grad_op->SetAttr("is_scalar_condition", GetAttr("is_scalar_condition")); return std::unique_ptr(grad_op); } }; diff --git a/python/paddle/v2/fluid/layers/control_flow.py b/python/paddle/v2/fluid/layers/control_flow.py index b264f0a1357bb..e71f3858b0a6d 100644 --- a/python/paddle/v2/fluid/layers/control_flow.py +++ b/python/paddle/v2/fluid/layers/control_flow.py @@ -1065,11 +1065,12 @@ def __exit__(self, exc_type, exc_val, exc_tb): class ConditionalBlock(object): - def __init__(self, inputs, name=None): + def __init__(self, inputs, is_scalar_condition=False, name=None): for each_input in inputs: if not isinstance(each_input, Variable): raise TypeError("Each input should be variable") self.inputs = inputs + self.is_scalar_condition = is_scalar_condition self.helper = LayerHelper('conditional_block', name=name) def block(self): @@ -1114,7 +1115,10 @@ def complete(self): }, outputs={'Out': out_list, 'Scope': [step_scope]}, - attrs={'sub_block': inside_block}) + attrs={ + 'sub_block': inside_block, + 'is_scalar_condition': self.is_scalar_condition + }) class Switch(object): @@ -1130,7 +1134,7 @@ def case(self, condition): raise ValueError("case should be called inside with") if len(self.pre_not_conditions) == 0: - cond_block = ConditionalBlock([condition]) + cond_block = ConditionalBlock([condition], is_scalar_condition=True) not_cond = logical_not(x=condition) self.pre_not_conditions.append(not_cond) else: @@ -1141,7 +1145,8 @@ def case(self, condition): self.pre_not_conditions.append(new_not_cond) cond_block = ConditionalBlock( [logical_and( - x=pre_not_cond, y=condition)]) + x=pre_not_cond, y=condition)], + is_scalar_condition=True) return ConditionalBlockGuard(cond_block) @@ -1152,7 +1157,8 @@ def default(self): if pre_cond_num == 0: raise ValueError("there should be at least one condition") cond_block = ConditionalBlock( - [self.pre_not_conditions[pre_cond_num - 1]]) + [self.pre_not_conditions[pre_cond_num - 1]], + is_scalar_condition=True) return ConditionalBlockGuard(cond_block) def __enter__(self): From c2d32071fd4fa7febe0a0c90b2480f093a40adf9 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 6 Feb 2018 19:04:31 +0800 Subject: [PATCH 30/32] fix comment --- paddle/operators/conditional_block_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/conditional_block_op.cc b/paddle/operators/conditional_block_op.cc index 8f5bdfedee328..bdcdb85be7a94 100644 --- a/paddle/operators/conditional_block_op.cc +++ b/paddle/operators/conditional_block_op.cc @@ -51,7 +51,7 @@ class ConditionalOp : public framework::OperatorBase { ips[0]->numel() == 1)) { PADDLE_THROW( "condition input's data type should be bool, " - "number should be 1, actual numel is %d", + "numel should be 1, actual numel is %d", ips[0]->numel()); } return ips[0]->data()[0]; From 31aa827c5f95a193458fab0dac4007e6e43153d7 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 7 Feb 2018 20:46:30 +0800 Subject: [PATCH 31/32] fix comment --- python/paddle/v2/fluid/learning_rate_decay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/learning_rate_decay.py b/python/paddle/v2/fluid/learning_rate_decay.py index 4648c0a6a623a..6b10cb0791562 100644 --- a/python/paddle/v2/fluid/learning_rate_decay.py +++ b/python/paddle/v2/fluid/learning_rate_decay.py @@ -195,7 +195,7 @@ def piecewise_decay(global_step, boundaries, values): raise ValueError("len(values) - len(boundaries) should be 1") if not isinstance(global_step, Variable): - raise ValueError("global_step is required for inverse_time_decay.") + raise ValueError("global_step is required for piecewise_decay.") lr = layers.create_global_var( shape=[1], From 003ed1a7824cb496f5b4fd6ae7df39677592ad84 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 7 Feb 2018 20:48:24 +0800 Subject: [PATCH 32/32] add export --- python/paddle/v2/fluid/learning_rate_decay.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/paddle/v2/fluid/learning_rate_decay.py b/python/paddle/v2/fluid/learning_rate_decay.py index 6b10cb0791562..13dc98075f7d3 100644 --- a/python/paddle/v2/fluid/learning_rate_decay.py +++ b/python/paddle/v2/fluid/learning_rate_decay.py @@ -15,7 +15,10 @@ import layers from framework import Variable -__all__ = ['exponential_decay', 'natural_exp_decay', 'inverse_time_decay'] +__all__ = [ + 'exponential_decay', 'natural_exp_decay', 'inverse_time_decay', + 'polynomial_decay', 'piecewise_decay' +] """ When training a model, it's often useful to decay the learning rate during training process, this is called