From 46d30ec680f494e4cc30a73330074497da064fbd Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Thu, 17 Aug 2017 20:34:02 -0700
Subject: [PATCH 01/14] init minst.py

---
 python/paddle/v2/framework/tests/mnist.py | 140 ++++++++++++++++++++++
 1 file changed, 140 insertions(+)
 create mode 100644 python/paddle/v2/framework/tests/mnist.py

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
new file mode 100644
index 00000000000000..32a088ac280866
--- /dev/null
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -0,0 +1,140 @@
+import paddle.v2.framework.core as core
+from paddle.v2.framework.op import Operator
+import numpy
+
+BATCH_SIZE = 100
+
+scope = core.Scope()
+place = core.CPUPlace()
+dev_ctx = core.DeviceContext.create(place)
+
+# init_net = core.Net.create()
+forward_network = core.Net.create()
+
+# should be init after forward_op is constructed
+# backward_net = core.Operator.backward(forward_net, set())
+backward_net = None
+optimize_net = core.Net.create()
+
+
+def atom_id():
+    id = 0
+    while True:
+        yield id
+        id += 1
+
+
+uniq_id = atom_id().next
+
+
+def data_layer(name, dims):
+    var = scope.new_var(name)
+    tensor = var.get_tensor()
+    tensor.set_dims(dims)  # 1 is batch size holder.
+    return name
+
+
+def feed_data(name, data):
+    assert isinstance(data, numpy.array)
+    tensor = scope.find_var(name).get_tensor()
+    tensor.set_dims(data.shape)
+    tensor.alloc_float(place)
+    tensor.set(data, place)
+
+
+def grad_var_name(var_name):
+    return var_name + "@GRAD"
+
+
+def sgd_optimizer(net, param_name, learning_rate=0.01):
+    grad_name = grad_var_name(param_name)
+    optimize_op = Operator(
+        "sgd", param=param_name, grad=grad_name, learning_rate=learning_rate)
+    net.add_op(optimize_op)
+
+
+# should use operator and add these to the init_network
+def init_param(param_name, dims):
+    print param_name
+    var = scope.new_var(param_name)
+    tensor = var.get_tensor()
+    tensor.set_dims(dims)
+    data = numpy.random.uniform(
+        low=0.0, high=1.0, size=tensor.shape()).astype("float32")
+    tensor.set(data, place)
+
+
+# fc_layer
+def fc_layer(net, input, size, act="sigmoid", bias=True, param=None, name=None):
+    """
+    Add a fc layer to net
+
+    :param input: input variable name.
+    :type input: str
+    :param size: fully connected layer size.
+    :param act: activation name
+    :param param: parameter attribute, used for initialize parameters.
+    :param bias: bias attribute. False will not have a bias.
+    :param name: the name of fc layer. If not set, model will generate a
+    readable name
+    :return: output variable name.
+    """
+    if name is None:
+        name = 'fc_%d' % uniq_id()
+    if not isinstance(name, str):
+        raise ValueError("name should be string")
+
+    input_dims = scope.find_var(input).get_tensor().get_dims()
+
+    w_name = param or name + ".w"
+    init_param(param_name=w_name, dims=[input_dims[1], size])
+    sgd_optimizer(net=optimize_net, param_name=w_name, learning_rate=0.01)
+
+    pre_activation = name + ".mul.out"
+    scope.new_var(pre_activation)
+    mul_op = Operator("mul", X=input, Y=w_name, Out=pre_activation)
+    net.add_op(mul_op)
+
+    # create bias variable if needed
+    if bias:
+        bias_name = name + ".b"
+        init_param(param_name=bias_name, dims=[size])
+        sgd_optimizer(
+            net=optimize_net, param_name=bias_name, learning_rate=0.01)
+        bias_out = name + ".rowwise_add.out"
+        scope.new_var(bias_out)
+        rowwise_add_op = Operator(
+            "rowwise_add", X=pre_activation, b=bias_name, Out=bias_out)
+        net.add_op(rowwise_add_op)
+        pre_activation = bias_out
+
+    activation_op = Operator(act, X=pre_activation, Y=name)
+    net.add_op(activation_op)
+    scope.new_var(name)
+    net.infer_shape(scope)
+    return name
+
+
+def cross_entropy_layer(net, input, label):
+    cost_name = 'cross_entropy_%d' % uniq_id()
+    cross_entropy_op = Operator(
+        "onehot_cross_entropy", X=input, label=label, Y=cost_name)
+    net.add_op(cross_entropy_op)
+    scope.new_var(cost_name)
+    net.infer_shape(scope)
+    return cost_name
+
+
+images = data_layer(name='pixel', dims=[BATCH_SIZE, 784])
+label = data_layer(name='label', dims=[BATCH_SIZE])
+fc = fc_layer(net=forward_network, input=images, size=10, act="softmax")
+cost = cross_entropy_layer(net=forward_network, input=fc, label=label)
+forward_network.complete_add_op(True)
+print(forward_network)
+backward_net = core.Operator.backward(forward_network, set())
+
+print(backward_net)
+
+PASS_NUM = 10
+for pass_id in range(PASS_NUM):
+    print pass_id

From 118dd1494fbe3654da8f71c2245523e27616d475 Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Mon, 21 Aug 2017 18:22:59 -0700
Subject: [PATCH 02/14] can run, for debug

---
 .../paddle/v2/framework/tests/CMakeLists.txt  |  1 +
 python/paddle/v2/framework/tests/mnist.py     | 73 +++++++++++++++++--
 2 files changed, 66 insertions(+), 8 deletions(-)

diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt
index ce57a071309272..41682c8350def7 100644
--- a/python/paddle/v2/framework/tests/CMakeLists.txt
+++ b/python/paddle/v2/framework/tests/CMakeLists.txt
@@ -27,3 +27,4 @@ py_test(test_uniform_random_op SRCS test_uniform_random_op.py)
 py_test(test_recurrent_op SRCS test_recurrent_op.py)
 py_test(test_sgd_op SRCS test_sgd_op.py)
 py_test(test_gradient_checker SRCS test_gradient_checker.py)
+py_test(mnist SRCS mnist.py)
diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index 32a088ac280866..d0c56c457d9646 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -2,7 +2,7 @@
 from paddle.v2.framework.op import Operator
 import numpy
 
-BATCH_SIZE = 100
+BATCH_SIZE = 2
 
 scope = core.Scope()
 place = core.CPUPlace()
@@ -35,10 +35,15 @@ def data_layer(name, dims):
 
 
 def feed_data(name, data):
-    assert isinstance(data, numpy.array)
+    assert isinstance(data, numpy.ndarray)
     tensor = scope.find_var(name).get_tensor()
     tensor.set_dims(data.shape)
-    tensor.alloc_float(place)
+    if data.dtype == numpy.dtype('int32'):
+        tensor.alloc_float(place)
+    elif data.dtype == numpy.dtype('float32'):
+        tensor.alloc_int(place)
+    else:
+        raise ValueError("data type not supported")
     tensor.set(data, place)
 
 
@@ -49,7 +54,11 @@ def grad_var_name(var_name):
 def sgd_optimizer(net, param_name, learning_rate=0.01):
     grad_name = grad_var_name(param_name)
     optimize_op = Operator(
-        "sgd", param=param_name, grad=grad_name, learning_rate=learning_rate)
+        "sgd",
+        param=param_name,
+        grad=grad_name,
+        param_out=param_name,
+        learning_rate=learning_rate)
     net.add_op(optimize_op)
 
 
@@ -65,7 +74,7 @@ def init_param(param_name, dims):
 
 
 # fc_layer
-def fc_layer(net, input, size, act="sigmoid", bias=True, param=None, name=None):
+def fc_layer(net, input, size, act="softmax", bias=True, param=None, name=None):
     """
     Add a fc layer to net
 
@@ -125,16 +134,64 @@ def cross_entropy_layer(net, input, label):
     return cost_name
 
 
+def get_backward_net(forward_net):
+    net = core.Operator.backward(forward_net, set())
+    for input in net.inputs()["all"]:
+        var = scope.new_var(input)
+        var.get_tensor()
+    for output in net.outputs()["all"]:
+        var = scope.new_var(output)
+        var.get_tensor()
+    return net
+
+
+def print_inputs_outputs(op):
+    print("===============" + op.type() + "==============")
+    print("***inputs:***")
+    for input in op.inputs()["all"]:
+        print input, scope.find_var(input).get_tensor().get_dims()
+    print("***outputs:***")
+    for output in op.outputs()["all"]:
+        print output, scope.find_var(output).get_tensor().get_dims()
+    print("")
+    print("")
+
+
 images = data_layer(name='pixel', dims=[BATCH_SIZE, 784])
 label = data_layer(name='label', dims=[BATCH_SIZE])
 fc = fc_layer(net=forward_network, input=images, size=10, act="softmax")
 cost = cross_entropy_layer(net=forward_network, input=fc, label=label)
 forward_network.complete_add_op(True)
 print(forward_network)
-backward_net = core.Operator.backward(forward_network, set())
-
+backward_net = get_backward_net(forward_network)
 print(backward_net)
+optimize_net.complete_add_op(True)
+print(optimize_net)
 
 PASS_NUM = 10
 for pass_id in range(PASS_NUM):
-    print pass_id
+    print("===========forward==========")
+    feed_data("pixel", numpy.random.random((BATCH_SIZE, 784)).astype('float32'))
+    feed_data("label", numpy.ones(BATCH_SIZE).astype("int32"))
+    forward_network.infer_shape(scope)
+    print_inputs_outputs(forward_network)
+
+    print(numpy.array(scope.find_var("label").get_tensor()))
+    forward_network.run(scope, dev_ctx)
+    # print(numpy.array(scope.find_var("fc_0").get_tensor()))
+
+    print("===========backward==========")
+    cost_data = numpy.array(scope.find_var("cross_entropy_1").get_tensor())
+    cost_grad = scope.find_var(grad_var_name("cross_entropy_1")).get_tensor()
+    cost_grad.set_dims(cost_data.shape)
+    cost_grad.alloc_float(place)
+    cost_grad.set(cost_data, place)
+
+    backward_net.infer_shape(scope)
+    print_inputs_outputs(backward_net)
+
+    backward_net.run(scope, dev_ctx)
+
+    print("===========optimize_net==========")
+    print_inputs_outputs(optimize_net)
+    optimize_net.run(scope, dev_ctx)

From 5a8fbb7d19e95f3be16bbee029e82e14f0a240df Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Tue, 22 Aug 2017 00:56:34 -0700
Subject: [PATCH 03/14] add data

---
 python/paddle/v2/framework/tests/mnist.py | 26 +++++++++++++++++------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index d0c56c457d9646..f75f196168c6d2 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -1,8 +1,9 @@
 import paddle.v2.framework.core as core
 from paddle.v2.framework.op import Operator
 import numpy
+import paddle.v2 as paddle
 
-BATCH_SIZE = 2
+BATCH_SIZE = 100
 
 scope = core.Scope()
 place = core.CPUPlace()
@@ -39,9 +40,9 @@ def feed_data(name, data):
     tensor = scope.find_var(name).get_tensor()
     tensor.set_dims(data.shape)
     if data.dtype == numpy.dtype('int32'):
-        tensor.alloc_float(place)
-    elif data.dtype == numpy.dtype('float32'):
         tensor.alloc_int(place)
+    elif data.dtype == numpy.dtype('float32'):
+        tensor.alloc_float(place)
     else:
         raise ValueError("data type not supported")
     tensor.set(data, place)
@@ -168,20 +169,31 @@ def print_inputs_outputs(op):
 optimize_net.complete_add_op(True)
 print(optimize_net)
 
-PASS_NUM = 10
+reader = paddle.batch(
+    paddle.reader.shuffle(
+        paddle.dataset.mnist.train(), buf_size=8192),
+    batch_size=BATCH_SIZE)
+
+PASS_NUM = 1000
 for pass_id in range(PASS_NUM):
     print("===========forward==========")
-    feed_data("pixel", numpy.random.random((BATCH_SIZE, 784)).astype('float32'))
-    feed_data("label", numpy.ones(BATCH_SIZE).astype("int32"))
+    # feed_data("pixel", numpy.random.random((BATCH_SIZE, 784)).astype('float32'))
+    # feed_data("label", numpy.ones(BATCH_SIZE).astype("int32"))
+    data = reader().next()
+    image = numpy.array(map(lambda x: x[0], data)).astype("float32")
+    label = numpy.array(map(lambda x: x[1], data)).astype("int32")
+    feed_data("pixel", image)
+    feed_data("label", label)
     forward_network.infer_shape(scope)
     print_inputs_outputs(forward_network)
 
-    print(numpy.array(scope.find_var("label").get_tensor()))
+    # print(numpy.array(scope.find_var("label").get_tensor()))
     forward_network.run(scope, dev_ctx)
     # print(numpy.array(scope.find_var("fc_0").get_tensor()))
 
     print("===========backward==========")
     cost_data = numpy.array(scope.find_var("cross_entropy_1").get_tensor())
+    print(cost_data.sum() / len(cost_data))
     cost_grad = scope.find_var(grad_var_name("cross_entropy_1")).get_tensor()
     cost_grad.set_dims(cost_data.shape)
     cost_grad.alloc_float(place)

From 51792022c9f7963321d77d7dac4143e566af9fdc Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Tue, 22 Aug 2017 12:54:44 -0700
Subject: [PATCH 04/14] refine code and add debug info

---
 python/paddle/v2/framework/tests/mnist.py | 47 +++++++++++------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index f75f196168c6d2..6a3ed0dce0ab5b 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -52,7 +52,7 @@ def grad_var_name(var_name):
     return var_name + "@GRAD"
 
 
-def sgd_optimizer(net, param_name, learning_rate=0.01):
+def sgd_optimizer(net, param_name, learning_rate=0.001):
     grad_name = grad_var_name(param_name)
     optimize_op = Operator(
         "sgd",
@@ -65,7 +65,6 @@ def sgd_optimizer(net, param_name, learning_rate=0.01):
 
 # should use operator and add these to the init_network
 def init_param(param_name, dims):
-    print param_name
     var = scope.new_var(param_name)
     tensor = var.get_tensor()
     tensor.set_dims(dims)
@@ -158,17 +157,34 @@ def print_inputs_outputs(op):
     print("")
 
 
+def set_cost():
+    cost_data = numpy.array(scope.find_var("cross_entropy_1").get_tensor())
+    # print(cost_data)
+    print(cost_data.sum() / len(cost_data))
+
+    cost_grad = scope.find_var(grad_var_name("cross_entropy_1")).get_tensor()
+    cost_grad.set_dims(cost_data.shape)
+    cost_grad.alloc_float(place)
+    cost_grad.set(cost_data, place)
+
+
 images = data_layer(name='pixel', dims=[BATCH_SIZE, 784])
 label = data_layer(name='label', dims=[BATCH_SIZE])
 fc = fc_layer(net=forward_network, input=images, size=10, act="softmax")
 cost = cross_entropy_layer(net=forward_network, input=fc, label=label)
+
 forward_network.complete_add_op(True)
-print(forward_network)
 backward_net = get_backward_net(forward_network)
-print(backward_net)
 optimize_net.complete_add_op(True)
+
+print(forward_network)
+print(backward_net)
 print(optimize_net)
 
+print_inputs_outputs(forward_network)
+print_inputs_outputs(backward_net)
+print_inputs_outputs(optimize_net)
+
 reader = paddle.batch(
     paddle.reader.shuffle(
         paddle.dataset.mnist.train(), buf_size=8192),
@@ -176,34 +192,17 @@ def print_inputs_outputs(op):
 
 PASS_NUM = 1000
 for pass_id in range(PASS_NUM):
-    print("===========forward==========")
-    # feed_data("pixel", numpy.random.random((BATCH_SIZE, 784)).astype('float32'))
-    # feed_data("label", numpy.ones(BATCH_SIZE).astype("int32"))
     data = reader().next()
+
     image = numpy.array(map(lambda x: x[0], data)).astype("float32")
     label = numpy.array(map(lambda x: x[1], data)).astype("int32")
     feed_data("pixel", image)
     feed_data("label", label)
-    forward_network.infer_shape(scope)
-    print_inputs_outputs(forward_network)
 
-    # print(numpy.array(scope.find_var("label").get_tensor()))
+    forward_network.infer_shape(scope)
     forward_network.run(scope, dev_ctx)
-    # print(numpy.array(scope.find_var("fc_0").get_tensor()))
-
-    print("===========backward==========")
-    cost_data = numpy.array(scope.find_var("cross_entropy_1").get_tensor())
-    print(cost_data.sum() / len(cost_data))
-    cost_grad = scope.find_var(grad_var_name("cross_entropy_1")).get_tensor()
-    cost_grad.set_dims(cost_data.shape)
-    cost_grad.alloc_float(place)
-    cost_grad.set(cost_data, place)
-
+    set_cost()
     backward_net.infer_shape(scope)
-    print_inputs_outputs(backward_net)
-
     backward_net.run(scope, dev_ctx)
 
-    print("===========optimize_net==========")
-    print_inputs_outputs(optimize_net)
     optimize_net.run(scope, dev_ctx)

From d3c65a64dc4ab98af10498cb2eb9327ef1697e5a Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Tue, 22 Aug 2017 20:21:23 -0700
Subject: [PATCH 05/14] fix data reader

---
 python/paddle/v2/framework/tests/mnist.py | 29 ++++++++++++-----------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index 6a3ed0dce0ab5b..1d40fd9a97f3e5 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -52,7 +52,7 @@ def grad_var_name(var_name):
     return var_name + "@GRAD"
 
 
-def sgd_optimizer(net, param_name, learning_rate=0.001):
+def sgd_optimizer(net, param_name, learning_rate=0.01):
     grad_name = grad_var_name(param_name)
     optimize_op = Operator(
         "sgd",
@@ -159,13 +159,13 @@ def print_inputs_outputs(op):
 
 def set_cost():
     cost_data = numpy.array(scope.find_var("cross_entropy_1").get_tensor())
-    # print(cost_data)
     print(cost_data.sum() / len(cost_data))
 
     cost_grad = scope.find_var(grad_var_name("cross_entropy_1")).get_tensor()
+
     cost_grad.set_dims(cost_data.shape)
     cost_grad.alloc_float(place)
-    cost_grad.set(cost_data, place)
+    cost_grad.set(numpy.ones(cost_data.shape).astype("float32"), place)
 
 
 images = data_layer(name='pixel', dims=[BATCH_SIZE, 784])
@@ -192,17 +192,18 @@ def set_cost():
 
 PASS_NUM = 1000
 for pass_id in range(PASS_NUM):
-    data = reader().next()
 
-    image = numpy.array(map(lambda x: x[0], data)).astype("float32")
-    label = numpy.array(map(lambda x: x[1], data)).astype("int32")
-    feed_data("pixel", image)
-    feed_data("label", label)
+    print("pass[" + str(pass_id) + "]")
+    for data in reader():
+        image = numpy.array(map(lambda x: x[0], data)).astype("float32")
+        label = numpy.array(map(lambda x: x[1], data)).astype("int32")
+        feed_data("pixel", image)
+        feed_data("label", label)
 
-    forward_network.infer_shape(scope)
-    forward_network.run(scope, dev_ctx)
-    set_cost()
-    backward_net.infer_shape(scope)
-    backward_net.run(scope, dev_ctx)
+        forward_network.infer_shape(scope)
+        forward_network.run(scope, dev_ctx)
+        set_cost()
+        backward_net.infer_shape(scope)
+        backward_net.run(scope, dev_ctx)
 
-    optimize_net.run(scope, dev_ctx)
+        optimize_net.run(scope, dev_ctx)

From a13798e8f7764239c151864894afc6a543e6c190 Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Tue, 22 Aug 2017 20:41:31 -0700
Subject: [PATCH 06/14] rename add_op to append_op

---
 python/paddle/v2/framework/tests/mnist.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index 1d40fd9a97f3e5..32349b8d4df73b 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -60,7 +60,7 @@ def sgd_optimizer(net, param_name, learning_rate=0.01):
         grad=grad_name,
         param_out=param_name,
         learning_rate=learning_rate)
-    net.add_op(optimize_op)
+    net.append_op(optimize_op)
 
 
 # should use operator and add these to the init_network
@@ -102,7 +102,7 @@ def fc_layer(net, input, size, act="softmax", bias=True, param=None, name=None):
     pre_activation = name + ".mul.out"
     scope.new_var(pre_activation)
     mul_op = Operator("mul", X=input, Y=w_name, Out=pre_activation)
-    net.add_op(mul_op)
+    net.append_op(mul_op)
 
     # create bias variable if needed
     if bias:
@@ -112,13 +112,13 @@ def fc_layer(net, input, size, act="softmax", bias=True, param=None, name=None):
             net=optimize_net, param_name=bias_name, learning_rate=0.01)
         bias_out = name + ".rowwise_add.out"
         scope.new_var(bias_out)
-        rowwise_add_op = Operator(
+        rowwise_append_op = Operator(
             "rowwise_add", X=pre_activation, b=bias_name, Out=bias_out)
-        net.add_op(rowwise_add_op)
+        net.append_op(rowwise_append_op)
         pre_activation = bias_out
 
     activation_op = Operator(act, X=pre_activation, Y=name)
-    net.add_op(activation_op)
+    net.append_op(activation_op)
     scope.new_var(name)
     net.infer_shape(scope)
     return name
@@ -128,7 +128,7 @@ def cross_entropy_layer(net, input, label):
     cost_name = 'cross_entropy_%d' % uniq_id()
     cross_entropy_op = Operator(
         "onehot_cross_entropy", X=input, label=label, Y=cost_name)
-    net.add_op(cross_entropy_op)
+    net.append_op(cross_entropy_op)
     scope.new_var(cost_name)
     net.infer_shape(scope)
     return cost_name

From 6f4b968f5618adce529d12bd2e3b72d4d1b64f61 Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Wed, 23 Aug 2017 00:58:02 -0700
Subject: [PATCH 07/14] can train the parameters

---
 python/paddle/v2/framework/tests/mnist.py | 39 +++++++++++++++++------
 1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index 32349b8d4df73b..ededf767bcab46 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -69,7 +69,7 @@ def init_param(param_name, dims):
     tensor = var.get_tensor()
     tensor.set_dims(dims)
     data = numpy.random.uniform(
-        low=0.0, high=1.0, size=tensor.shape()).astype("float32")
+        low=-0.5, high=0.5, size=tensor.shape()).astype("float32")
     tensor.set(data, place)
 
 
@@ -109,7 +109,7 @@ def fc_layer(net, input, size, act="softmax", bias=True, param=None, name=None):
         bias_name = name + ".b"
         init_param(param_name=bias_name, dims=[size])
         sgd_optimizer(
-            net=optimize_net, param_name=bias_name, learning_rate=0.01)
+            net=optimize_net, param_name=bias_name, learning_rate=0.001)
         bias_out = name + ".rowwise_add.out"
         scope.new_var(bias_out)
         rowwise_append_op = Operator(
@@ -158,20 +158,33 @@ def print_inputs_outputs(op):
 
 
 def set_cost():
-    cost_data = numpy.array(scope.find_var("cross_entropy_1").get_tensor())
+    cost_shape = numpy.array(scope.find_var("cross_entropy_3").get_tensor(
+    )).shape
+    cost_grad = scope.find_var(grad_var_name("cross_entropy_3")).get_tensor()
+    cost_grad.set_dims(cost_shape)
+    cost_grad.alloc_float(place)
+    cost_grad.set(numpy.ones(cost_shape).astype("float32"), place)
+
+
+def print_cost():
+    cost_data = numpy.array(scope.find_var("cross_entropy_3").get_tensor())
     print(cost_data.sum() / len(cost_data))
 
-    cost_grad = scope.find_var(grad_var_name("cross_entropy_1")).get_tensor()
 
-    cost_grad.set_dims(cost_data.shape)
-    cost_grad.alloc_float(place)
-    cost_grad.set(numpy.ones(cost_data.shape).astype("float32"), place)
+def error_rate(predict, label):
+    predict_var = numpy.array(scope.find_var(predict).get_tensor()).argmax(
+        axis=1)
+    label = numpy.array(scope.find_var(label).get_tensor())
+    error_num = numpy.sum(predict_var != label)
+    print(error_num / float(len(label)))
 
 
 images = data_layer(name='pixel', dims=[BATCH_SIZE, 784])
 label = data_layer(name='label', dims=[BATCH_SIZE])
-fc = fc_layer(net=forward_network, input=images, size=10, act="softmax")
-cost = cross_entropy_layer(net=forward_network, input=fc, label=label)
+fc1 = fc_layer(net=forward_network, input=images, size=100, act="sigmoid")
+fc2 = fc_layer(net=forward_network, input=fc1, size=100, act="sigmoid")
+predict = fc_layer(net=forward_network, input=fc2, size=100, act="softmax")
+cost = cross_entropy_layer(net=forward_network, input=predict, label=label)
 
 forward_network.complete_add_op(True)
 backward_net = get_backward_net(forward_network)
@@ -192,8 +205,8 @@ def set_cost():
 
 PASS_NUM = 1000
 for pass_id in range(PASS_NUM):
+    batch_id = 0
 
-    print("pass[" + str(pass_id) + "]")
     for data in reader():
         image = numpy.array(map(lambda x: x[0], data)).astype("float32")
         label = numpy.array(map(lambda x: x[1], data)).astype("int32")
@@ -207,3 +220,9 @@ def set_cost():
         backward_net.run(scope, dev_ctx)
 
         optimize_net.run(scope, dev_ctx)
+        if batch_id % 100 == 0:
+            print("pass[" + str(pass_id) + "] batch_id[" + str(batch_id) + "]")
+            print_cost()
+            error_rate(predict, "label")
+
+        batch_id = batch_id + 1

From 76677f25774a84d9ced011be02e62ae15b03506c Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Wed, 23 Aug 2017 09:12:34 -0700
Subject: [PATCH 08/14] add test

---
 python/paddle/v2/framework/tests/mnist.py | 36 ++++++++++++++++++-----
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index ededf767bcab46..e47de2436ff737 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -52,7 +52,7 @@ def grad_var_name(var_name):
     return var_name + "@GRAD"
 
 
-def sgd_optimizer(net, param_name, learning_rate=0.01):
+def sgd_optimizer(net, param_name, learning_rate=0.005):
     grad_name = grad_var_name(param_name)
     optimize_op = Operator(
         "sgd",
@@ -166,9 +166,9 @@ def set_cost():
     cost_grad.set(numpy.ones(cost_shape).astype("float32"), place)
 
 
-def print_cost():
+def mean_cost():
     cost_data = numpy.array(scope.find_var("cross_entropy_3").get_tensor())
-    print(cost_data.sum() / len(cost_data))
+    return cost_data.sum() / len(cost_data)
 
 
 def error_rate(predict, label):
@@ -176,7 +176,7 @@ def error_rate(predict, label):
         axis=1)
     label = numpy.array(scope.find_var(label).get_tensor())
     error_num = numpy.sum(predict_var != label)
-    print(error_num / float(len(label)))
+    return error_num / float(len(label))
 
 
 images = data_layer(name='pixel', dims=[BATCH_SIZE, 784])
@@ -198,16 +198,35 @@ def error_rate(predict, label):
 print_inputs_outputs(backward_net)
 print_inputs_outputs(optimize_net)
 
-reader = paddle.batch(
+train_reader = paddle.batch(
     paddle.reader.shuffle(
         paddle.dataset.mnist.train(), buf_size=8192),
     batch_size=BATCH_SIZE)
 
+
+def test():
+    test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=128)
+    cost = []
+    error = []
+    for data in test_reader():
+        image = numpy.array(map(lambda x: x[0], data)).astype("float32")
+        label = numpy.array(map(lambda x: x[1], data)).astype("int32")
+        feed_data("pixel", image)
+        feed_data("label", label)
+
+        forward_network.infer_shape(scope)
+        forward_network.run(scope, dev_ctx)
+        cost.append(mean_cost())
+        error.append(error_rate(predict, "label"))
+    print("cost=" + str(sum(cost) / float(len(cost))) + " error_rate=" + str(
+        sum(error) / float(len(error))))
+
+
 PASS_NUM = 1000
 for pass_id in range(PASS_NUM):
     batch_id = 0
 
-    for data in reader():
+    for data in train_reader():
         image = numpy.array(map(lambda x: x[0], data)).astype("float32")
         label = numpy.array(map(lambda x: x[1], data)).astype("int32")
         feed_data("pixel", image)
@@ -222,7 +241,8 @@ def error_rate(predict, label):
         optimize_net.run(scope, dev_ctx)
         if batch_id % 100 == 0:
             print("pass[" + str(pass_id) + "] batch_id[" + str(batch_id) + "]")
-            print_cost()
-            error_rate(predict, "label")
+            test()
+            # print(mean_cost())
+            # print(error_rate(predict, "label"))
 
         batch_id = batch_id + 1

From cf515e4a72f4b02fbbbfdbd79c3b66b1be694e7b Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Wed, 23 Aug 2017 09:39:47 -0700
Subject: [PATCH 09/14] optimize code and name

---
 python/paddle/v2/framework/tests/mnist.py | 56 +++++++++++------------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index e47de2436ff737..886e99610dd380 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -134,7 +134,7 @@ def cross_entropy_layer(net, input, label):
     return cost_name
 
 
-def get_backward_net(forward_net):
+def create_backward_net(forward_net):
     net = core.Operator.backward(forward_net, set())
     for input in net.inputs()["all"]:
         var = scope.new_var(input)
@@ -145,29 +145,29 @@ def get_backward_net(forward_net):
     return net
 
 
-def print_inputs_outputs(op):
+def debug_print_op(op):
     print("===============" + op.type() + "==============")
     print("***inputs:***")
     for input in op.inputs()["all"]:
         print input, scope.find_var(input).get_tensor().get_dims()
-    print("***outputs:***")
+    print("\n***outputs:***")
     for output in op.outputs()["all"]:
         print output, scope.find_var(output).get_tensor().get_dims()
     print("")
     print("")
 
 
-def set_cost():
-    cost_shape = numpy.array(scope.find_var("cross_entropy_3").get_tensor(
-    )).shape
-    cost_grad = scope.find_var(grad_var_name("cross_entropy_3")).get_tensor()
+def set_cost(cost):
+    cost_shape = numpy.array(scope.find_var(cost).get_tensor()).shape
+    cost_grad = \
+        scope.find_var(grad_var_name(cost)).get_tensor()
     cost_grad.set_dims(cost_shape)
     cost_grad.alloc_float(place)
     cost_grad.set(numpy.ones(cost_shape).astype("float32"), place)
 
 
-def mean_cost():
-    cost_data = numpy.array(scope.find_var("cross_entropy_3").get_tensor())
+def mean_cost(cost):
+    cost_data = numpy.array(scope.find_var(cost).get_tensor())
     return cost_data.sum() / len(cost_data)
 
 
@@ -180,23 +180,23 @@ def error_rate(predict, label):
 
 
 images = data_layer(name='pixel', dims=[BATCH_SIZE, 784])
-label = data_layer(name='label', dims=[BATCH_SIZE])
+labels = data_layer(name='label', dims=[BATCH_SIZE])
 fc1 = fc_layer(net=forward_network, input=images, size=100, act="sigmoid")
 fc2 = fc_layer(net=forward_network, input=fc1, size=100, act="sigmoid")
 predict = fc_layer(net=forward_network, input=fc2, size=100, act="softmax")
-cost = cross_entropy_layer(net=forward_network, input=predict, label=label)
+cost = cross_entropy_layer(net=forward_network, input=predict, label=labels)
 
 forward_network.complete_add_op(True)
-backward_net = get_backward_net(forward_network)
+backward_net = create_backward_net(forward_network)
 optimize_net.complete_add_op(True)
 
 print(forward_network)
 print(backward_net)
 print(optimize_net)
 
-print_inputs_outputs(forward_network)
-print_inputs_outputs(backward_net)
-print_inputs_outputs(optimize_net)
+debug_print_op(forward_network)
+debug_print_op(backward_net)
+debug_print_op(optimize_net)
 
 train_reader = paddle.batch(
     paddle.reader.shuffle(
@@ -204,19 +204,19 @@ def error_rate(predict, label):
     batch_size=BATCH_SIZE)
 
 
-def test():
+def test(cost_name):
     test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=128)
     cost = []
     error = []
     for data in test_reader():
-        image = numpy.array(map(lambda x: x[0], data)).astype("float32")
-        label = numpy.array(map(lambda x: x[1], data)).astype("int32")
-        feed_data("pixel", image)
-        feed_data("label", label)
+        image_data = numpy.array(map(lambda x: x[0], data)).astype("float32")
+        label_data = numpy.array(map(lambda x: x[1], data)).astype("int32")
+        feed_data(images, image_data)
+        feed_data(labels, label_data)
 
         forward_network.infer_shape(scope)
         forward_network.run(scope, dev_ctx)
-        cost.append(mean_cost())
+        cost.append(mean_cost(cost_name))
         error.append(error_rate(predict, "label"))
     print("cost=" + str(sum(cost) / float(len(cost))) + " error_rate=" + str(
         sum(error) / float(len(error))))
@@ -227,22 +227,20 @@ def test():
     batch_id = 0
 
     for data in train_reader():
-        image = numpy.array(map(lambda x: x[0], data)).astype("float32")
-        label = numpy.array(map(lambda x: x[1], data)).astype("int32")
-        feed_data("pixel", image)
-        feed_data("label", label)
+        image_data = numpy.array(map(lambda x: x[0], data)).astype("float32")
+        label_data = numpy.array(map(lambda x: x[1], data)).astype("int32")
+        feed_data(images, image_data)
+        feed_data(labels, label_data)
 
         forward_network.infer_shape(scope)
         forward_network.run(scope, dev_ctx)
-        set_cost()
+        set_cost(cost)
         backward_net.infer_shape(scope)
         backward_net.run(scope, dev_ctx)
 
         optimize_net.run(scope, dev_ctx)
         if batch_id % 100 == 0:
             print("pass[" + str(pass_id) + "] batch_id[" + str(batch_id) + "]")
-            test()
-            # print(mean_cost())
-            # print(error_rate(predict, "label"))
+            test(cost)
 
         batch_id = batch_id + 1

From 9db4ad6130d79d72fa150e534b5b54fa723c3240 Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Wed, 23 Aug 2017 09:42:58 -0700
Subject: [PATCH 10/14] reduce pass num to 1

---
 python/paddle/v2/framework/tests/mnist.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index 886e99610dd380..eefd5709a3bd7a 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -222,7 +222,7 @@ def test(cost_name):
         sum(error) / float(len(error))))
 
 
-PASS_NUM = 1000
+PASS_NUM = 1
 for pass_id in range(PASS_NUM):
     batch_id = 0
 

From 37cd8165b3089c8e4a6ce743f5e0ee8c029ba46b Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Wed, 23 Aug 2017 10:56:56 -0700
Subject: [PATCH 11/14] change 128 to BATCH_SIZE

---
 python/paddle/v2/framework/tests/mnist.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index eefd5709a3bd7a..e878bfa4e9b2be 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -205,7 +205,8 @@ def error_rate(predict, label):
 
 
 def test(cost_name):
-    test_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=128)
+    test_reader = paddle.batch(
+        paddle.dataset.mnist.test(), batch_size=BATCH_SIZE)
     cost = []
     error = []
     for data in test_reader():

From 0e300f9bf04ba459dbef93af9537f847cebbcd27 Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Wed, 23 Aug 2017 20:14:54 -0700
Subject: [PATCH 12/14] use init_net and random_op to initialize parameter

---
 python/paddle/v2/framework/tests/mnist.py | 54 +++++++++++------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index e878bfa4e9b2be..0c27ce3e355b6a 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -9,11 +9,8 @@
 place = core.CPUPlace()
 dev_ctx = core.DeviceContext.create(place)
 
-# init_net = core.Net.create()
-forward_network = core.Net.create()
-
-# should be init after forward_op is constructed
-# backward_net = core.Operator.backward(forward_net, set())
+init_net = core.Net.create()
+forward_net = core.Net.create()
 backward_net = None
 optimize_net = core.Net.create()
 
@@ -64,13 +61,12 @@ def sgd_optimizer(net, param_name, learning_rate=0.005):
 
 
 # should use operator and add these to the init_network
-def init_param(param_name, dims):
-    var = scope.new_var(param_name)
-    tensor = var.get_tensor()
-    tensor.set_dims(dims)
-    data = numpy.random.uniform(
-        low=-0.5, high=0.5, size=tensor.shape()).astype("float32")
-    tensor.set(data, place)
+def init_param(net, param_name, dims):
+    scope.new_var(param_name)
+    op = Operator(
+        "uniform_random", Out=param_name, dims=dims, min=-0.5, max=0.5, seed=10)
+    op.infer_shape(scope)
+    net.append_op(op)
 
 
 # fc_layer
@@ -96,7 +92,7 @@ def fc_layer(net, input, size, act="softmax", bias=True, param=None, name=None):
     input_dims = scope.find_var(input).get_tensor().get_dims()
 
     w_name = param or name + ".w"
-    init_param(param_name=w_name, dims=[input_dims[1], size])
+    init_param(net=init_net, param_name=w_name, dims=[input_dims[1], size])
     sgd_optimizer(net=optimize_net, param_name=w_name, learning_rate=0.01)
 
     pre_activation = name + ".mul.out"
@@ -107,7 +103,7 @@ def fc_layer(net, input, size, act="softmax", bias=True, param=None, name=None):
     # create bias variable if needed
     if bias:
         bias_name = name + ".b"
-        init_param(param_name=bias_name, dims=[size])
+        init_param(net=init_net, param_name=bias_name, dims=[size])
         sgd_optimizer(
             net=optimize_net, param_name=bias_name, learning_rate=0.001)
         bias_out = name + ".rowwise_add.out"
@@ -181,20 +177,22 @@ def error_rate(predict, label):
 
 images = data_layer(name='pixel', dims=[BATCH_SIZE, 784])
 labels = data_layer(name='label', dims=[BATCH_SIZE])
-fc1 = fc_layer(net=forward_network, input=images, size=100, act="sigmoid")
-fc2 = fc_layer(net=forward_network, input=fc1, size=100, act="sigmoid")
-predict = fc_layer(net=forward_network, input=fc2, size=100, act="softmax")
-cost = cross_entropy_layer(net=forward_network, input=predict, label=labels)
-
-forward_network.complete_add_op(True)
-backward_net = create_backward_net(forward_network)
+fc1 = fc_layer(net=forward_net, input=images, size=100, act="sigmoid")
+fc2 = fc_layer(net=forward_net, input=fc1, size=100, act="sigmoid")
+predict = fc_layer(net=forward_net, input=fc2, size=100, act="softmax")
+cost = cross_entropy_layer(net=forward_net, input=predict, label=labels)
+
+init_net.complete_add_op(True)
+forward_net.complete_add_op(True)
+backward_net = create_backward_net(forward_net)
 optimize_net.complete_add_op(True)
 
-print(forward_network)
+print(init_net)
+print(forward_net)
 print(backward_net)
 print(optimize_net)
 
-debug_print_op(forward_network)
+debug_print_op(forward_net)
 debug_print_op(backward_net)
 debug_print_op(optimize_net)
 
@@ -215,8 +213,8 @@ def test(cost_name):
         feed_data(images, image_data)
         feed_data(labels, label_data)
 
-        forward_network.infer_shape(scope)
-        forward_network.run(scope, dev_ctx)
+        forward_net.infer_shape(scope)
+        forward_net.run(scope, dev_ctx)
         cost.append(mean_cost(cost_name))
         error.append(error_rate(predict, "label"))
     print("cost=" + str(sum(cost) / float(len(cost))) + " error_rate=" + str(
@@ -224,6 +222,8 @@ def test(cost_name):
 
 
 PASS_NUM = 1
+
+init_net.run(scope, dev_ctx)
 for pass_id in range(PASS_NUM):
     batch_id = 0
 
@@ -233,8 +233,8 @@ def test(cost_name):
         feed_data(images, image_data)
         feed_data(labels, label_data)
 
-        forward_network.infer_shape(scope)
-        forward_network.run(scope, dev_ctx)
+        forward_net.infer_shape(scope)
+        forward_net.run(scope, dev_ctx)
         set_cost(cost)
         backward_net.infer_shape(scope)
         backward_net.run(scope, dev_ctx)

From 3648165b63bd5331d1809cba896176e4af0a9ff2 Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Wed, 23 Aug 2017 23:00:45 -0700
Subject: [PATCH 13/14] add gpu support

---
 python/paddle/v2/framework/tests/mnist.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index 0c27ce3e355b6a..d9941023fe69fc 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -7,6 +7,8 @@
 
 scope = core.Scope()
 place = core.CPUPlace()
+# if you want to test GPU training, you can use gpu place
+# place = core.GPUPlace(0)
 dev_ctx = core.DeviceContext.create(place)
 
 init_net = core.Net.create()

From 625b15355a16fa42476e7dbd166b77e092dcb97f Mon Sep 17 00:00:00 2001
From: qiaolongfei <qiaolongfei@baidu.com>
Date: Wed, 23 Aug 2017 23:56:55 -0700
Subject: [PATCH 14/14] optimize code

---
 python/paddle/v2/framework/tests/mnist.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/python/paddle/v2/framework/tests/mnist.py b/python/paddle/v2/framework/tests/mnist.py
index d9941023fe69fc..9a0b109850e92c 100644
--- a/python/paddle/v2/framework/tests/mnist.py
+++ b/python/paddle/v2/framework/tests/mnist.py
@@ -17,14 +17,14 @@
 optimize_net = core.Net.create()
 
 
-def atom_id():
+def atomic_id():
     id = 0
     while True:
         yield id
         id += 1
 
 
-uniq_id = atom_id().next
+uniq_id = atomic_id().next
 
 
 def data_layer(name, dims):
@@ -164,7 +164,7 @@ def set_cost(cost):
     cost_grad.set(numpy.ones(cost_shape).astype("float32"), place)
 
 
-def mean_cost(cost):
+def get_cost_mean(cost):
     cost_data = numpy.array(scope.find_var(cost).get_tensor())
     return cost_data.sum() / len(cost_data)
 
@@ -217,7 +217,7 @@ def test(cost_name):
 
         forward_net.infer_shape(scope)
         forward_net.run(scope, dev_ctx)
-        cost.append(mean_cost(cost_name))
+        cost.append(get_cost_mean(cost_name))
         error.append(error_rate(predict, "label"))
     print("cost=" + str(sum(cost) / float(len(cost))) + " error_rate=" + str(
         sum(error) / float(len(error))))